├── .gitignore ├── .travis.yml ├── BUILD ├── BUILD-cn ├── LICENSE ├── LICENSE.Apache ├── Makefile ├── README.md ├── benchmark ├── README.md ├── build_ycsb.sh ├── load_data.sh ├── run_test.sh ├── tera_mark.md ├── ycsb4tera.md └── ycsb4tera │ └── tera │ ├── pom.xml │ └── src │ └── main │ └── java │ └── com │ └── yahoo │ └── ycsb │ └── TeraDB.java ├── build.conf.template ├── build.sh ├── build_version.sh ├── conf └── tera.flag ├── depends.mk.template ├── doc ├── README.md ├── cn │ ├── README.md │ ├── cluster_setup.md │ ├── filter.md │ ├── onebox.md │ ├── roadmap.md │ ├── sdk_guide.md │ ├── sdk_guide_http.md │ ├── sdk_guide_java.md │ ├── sdk_guide_python.md │ ├── teracli.md │ └── to_be_a_committer.md ├── data-deletion-in-tera.md ├── data_model.md ├── en │ ├── README.md │ ├── contributor.md │ ├── onebox.md │ ├── performance.md │ ├── roadmap.md │ ├── sdk_guide.md │ ├── sdk_guide_http.md │ ├── sdk_guide_java.md │ ├── sdk_guide_python.md │ └── teracli.md ├── global_txn.md ├── image │ ├── persistent_cache_arch.png │ └── persistent_cache_meta_data_arch.png ├── kpi.md ├── load-balance.md ├── master-ts-client-interactive.md ├── master_design.md ├── multi_tenancy_user_manual.md ├── new_sdk_dev.md ├── newbie.md ├── newcomer │ └── tera_intro.md ├── perf_opz.md ├── persistent_cache.md ├── prop_tree.md ├── read_write_consistency.md ├── release_management.md ├── roles_statement.md ├── sdk_reference │ ├── client.md │ ├── mutation.md │ ├── reader.md │ ├── readme.md │ ├── scan.md │ ├── table.md │ ├── table_descriptor.md │ ├── transaction.md │ └── utils.md ├── se_build_on_tera.md ├── single_row_txn.md ├── tablet-availability.md ├── tablet_state_machine.dot ├── tablet_state_machine.svg ├── tera_design.md ├── tera_flag.md ├── tera_question_answer_hjh.md ├── to_be_a_contributor.md ├── tools │ ├── admincli.md │ ├── benchmark.md │ ├── readme.md │ ├── teracli.md │ ├── terautil.md │ └── ycsb.md ├── txn_user_manual.md ├── ubuntu_install_onebox_and_cluster.md ├── user_manual.md └── work_flow.md ├── example ├── docker │ ├── README.md │ ├── cluster_setup.py │ ├── conf │ ├── dockerfile │ ├── dockerfile.zk │ ├── hdfs.py │ ├── hdfs_setup.py │ ├── install.sh │ ├── master │ ├── readme-cn.md │ ├── tabletnode │ ├── tera.py │ ├── tera_setup.py │ ├── zk.py │ └── zk_setup.py └── onebox │ ├── bin │ ├── config │ ├── kill_tera.sh │ └── launch_tera.sh │ └── conf │ ├── lb.flag │ └── tera.flag ├── ft_test.sh ├── include ├── observer │ ├── notification.h │ ├── observer.h │ ├── scanner.h │ └── scanner_entry.h ├── tera.h ├── tera │ ├── batch_mutation.h │ ├── client.h │ ├── error_code.h │ ├── filter.h │ ├── filter_comparator.h │ ├── filter_list.h │ ├── hash.h │ ├── mutation.h │ ├── reader.h │ ├── scan.h │ ├── table.h │ ├── table_descriptor.h │ ├── tera_entry.h │ ├── transaction.h │ ├── utils.h │ └── value_filter.h └── tera_c.h ├── readme-cn.md ├── resources └── images │ ├── arch.png │ ├── global_txn.png │ ├── tara_arch.png │ ├── tera_master_impl.png │ ├── tera_ts_impl.png │ ├── webtable_datamodel.png │ └── webtable_er.png ├── script ├── README.md └── manual_rebalance.sh ├── so-version-script ├── sql ├── Makefile ├── build.sh ├── script │ ├── init_mysql.sh │ └── start_mysql.sh └── src │ ├── CMakeLists.txt │ ├── ha_tera.cc │ ├── ha_tera.h │ ├── ha_tera_format.cc │ ├── ha_tera_format.h │ ├── ha_tera_util.cc │ └── ha_tera_util.h └── src ├── README.md ├── access ├── access_builder.cc ├── access_builder.h ├── access_entry.cc ├── access_entry.h ├── access_updater.cc ├── access_updater.h ├── authorization │ ├── authorization.cc │ └── authorization.h ├── giano │ ├── giano_identification.cc │ ├── giano_identification.h │ ├── giano_verification.cc │ └── giano_verification.h ├── helpers │ ├── access_utils.cc │ ├── access_utils.h │ ├── permission_builder.cc │ ├── permission_builder.h │ └── version_recorder.h ├── identification │ ├── identification.h │ ├── ugi_identification.cc │ └── ugi_identification.h ├── test │ ├── access_verification_test.cc │ ├── multi_tenancy_test.cc │ └── permission_test.cc └── verification │ ├── ugi_verification.cc │ ├── ugi_verification.h │ └── verification.h ├── admincli.cc ├── benchmark ├── eva │ ├── README.md │ ├── __init__.py │ ├── conf.sample │ ├── eva_utils.py │ ├── eva_var.py │ ├── helper.py │ ├── run.py │ ├── run_sample.sh │ └── setup.sh ├── mark.cc ├── mark.h ├── mark_main.cc └── tpcc │ ├── data_generator.cc │ ├── data_generator.h │ ├── driver.cc │ ├── driver.h │ ├── mock_tpccdb.cc │ ├── mock_tpccdb.h │ ├── random_generator.cc │ ├── random_generator.h │ ├── tera_tpccdb.cc │ ├── tera_tpccdb.h │ ├── tera_txn │ ├── delivery_txn.cc │ ├── new_order_txn.cc │ ├── order_status_txn.cc │ ├── payment_txn.cc │ └── stocklevel_txn.cc │ ├── test │ ├── data_generator_test.cc │ ├── random_generator_test.cc │ └── tpcc_test.cc │ ├── tpcc_flags.cc │ ├── tpcc_main.cc │ ├── tpcc_schemas │ ├── t_customer │ ├── t_customer_last_index │ ├── t_district │ ├── t_history │ ├── t_history_index │ ├── t_item │ ├── t_neworder │ ├── t_order │ ├── t_order_index │ ├── t_orderline │ ├── t_stock │ └── t_warehouse │ ├── tpcc_types.h │ ├── tpccdb.cc │ └── tpccdb.h ├── common ├── atomic.h ├── base │ ├── ascii.h │ ├── bounded_queue.h │ ├── byte_order.h │ ├── preprocess.h │ ├── scoped_ptr.h │ ├── static_assert.h │ ├── stdint.h │ ├── string_ext.cc │ ├── string_ext.h │ ├── string_format.cc │ ├── string_format.h │ ├── string_number.cc │ ├── string_number.h │ └── test │ │ ├── ascii_test.cc │ │ ├── byte_order_test.cc │ │ ├── string_ext_test.cc │ │ └── string_number_test.cc ├── console │ ├── progress_bar.cc │ └── progress_bar.h ├── counter.h ├── cpu_profiler.cc ├── cpu_profiler.h ├── event.h ├── file │ ├── file_flags.cc │ ├── file_path.cc │ ├── file_path.h │ ├── file_stream.cc │ ├── file_stream.h │ ├── file_types.h │ └── recordio │ │ ├── record_io.cc │ │ └── record_io.h ├── func_scope_guard.h ├── heap_profiler.cc ├── heap_profiler.h ├── log │ ├── log_cleaner.cc │ └── log_cleaner.h ├── metric │ ├── cache_collector.h │ ├── collector.h │ ├── collector_report.h │ ├── collector_report_publisher.cc │ ├── collector_report_publisher.h │ ├── counter_collector.h │ ├── hardware_collectors.cc │ ├── hardware_collectors.h │ ├── metric_counter.h │ ├── metric_http_server.cc │ ├── metric_http_server.h │ ├── metric_id.cc │ ├── metric_id.h │ ├── percentile_counter.h │ ├── prometheus_subscriber.cc │ ├── prometheus_subscriber.h │ ├── ratio_collector.h │ ├── ratio_subscriber.h │ ├── subscriber.h │ ├── tcmalloc_collector.cc │ └── tcmalloc_collector.h ├── mutex.h ├── net │ ├── ip_address.cc │ └── ip_address.h ├── request_done_wrapper.h ├── rwmutex.h ├── semaphore.h ├── tera_entry.cc ├── test │ ├── bounded_queue_test.cc │ ├── collector_report_test.cc │ ├── common_test_main.cc │ ├── counter_test.cc │ ├── log_cleaner_test.cc │ ├── metric_counter_test.cc │ ├── metric_http_server_test.cc │ ├── metric_id_test.cc │ ├── metrics_test.cc │ ├── percentile_counter_test.cc │ ├── profiler_test.cc │ ├── progress_bar_test.cc │ └── thread_pool_test.cc ├── this_thread.h ├── thread_attributes.h ├── thread_pool.h └── timer.h ├── io ├── atomic_merge_strategy.cc ├── atomic_merge_strategy.h ├── coding.cc ├── coding.h ├── default_compact_strategy.cc ├── default_compact_strategy.h ├── io_flags.cc ├── io_utils.cc ├── io_utils.h ├── mock_tablet_io.h ├── tablet_io.cc ├── tablet_io.h ├── tablet_scanner.cc ├── tablet_scanner.h ├── tablet_writer.cc ├── tablet_writer.h ├── test │ ├── async_writer_test.cc │ ├── load_test.cc │ ├── tablet_io_test.cc │ └── tablet_scanner_test.cc ├── timekey_comparator.cc ├── timekey_comparator.h ├── ttlkv_compact_strategy.cc ├── ttlkv_compact_strategy.h ├── utils_leveldb.cc └── utils_leveldb.h ├── lbcli_main.cc ├── leveldb ├── .gitignore ├── AUTHORS ├── LICENSE ├── Makefile ├── README ├── README.md ├── TODO ├── bench │ ├── db_bench_sqlite3.cc │ ├── db_bench_tree_db.cc │ └── tera_bench.cc ├── build_detect_platform ├── db │ ├── builder.cc │ ├── builder.h │ ├── c.cc │ ├── c_test.c │ ├── corruption_test.cc │ ├── db.cc │ ├── db_bench.cc │ ├── db_impl.cc │ ├── db_impl.h │ ├── db_impl_test.cc │ ├── db_iter.cc │ ├── db_iter.h │ ├── db_table.cc │ ├── db_table.h │ ├── db_test.cc │ ├── dbformat.cc │ ├── dbformat.h │ ├── dbformat_test.cc │ ├── filename.cc │ ├── filename.h │ ├── filename_test.cc │ ├── import_main.cc │ ├── leveldb_main.cc │ ├── lg_compact_thread.h │ ├── lg_write_thread.h │ ├── log_async_writer.cc │ ├── log_async_writer.h │ ├── log_format.h │ ├── log_reader.cc │ ├── log_reader.h │ ├── log_test.cc │ ├── log_writer.cc │ ├── log_writer.h │ ├── memtable.cc │ ├── memtable.h │ ├── memtable_on_leveldb.cc │ ├── memtable_on_leveldb.h │ ├── repair.cc │ ├── sharded_memtable.cc │ ├── sharded_memtable.h │ ├── skiplist.h │ ├── skiplist_test.cc │ ├── snapshot.h │ ├── table_cache.cc │ ├── table_cache.h │ ├── table_utils.cc │ ├── table_utils_test.cc │ ├── version_edit.cc │ ├── version_edit.h │ ├── version_edit_test.cc │ ├── version_set.cc │ ├── version_set.h │ ├── version_set_test.cc │ ├── write_batch.cc │ ├── write_batch_internal.h │ └── write_batch_test.cc ├── hdfs.c.patch ├── helpers │ └── memenv │ │ ├── memenv.cc │ │ ├── memenv.h │ │ └── memenv_test.cc ├── include │ ├── hdfs.h │ ├── hdfs2.h │ ├── leveldb │ │ ├── c.h │ │ ├── cache.h │ │ ├── compact_strategy.h │ │ ├── comparator.h │ │ ├── config.h │ │ ├── db.h │ │ ├── dfs.h │ │ ├── env.h │ │ ├── env_dfs.h │ │ ├── env_flash.h │ │ ├── env_inmem.h │ │ ├── env_mock.h │ │ ├── filter_policy.h │ │ ├── iterator.h │ │ ├── lg_coding.h │ │ ├── options.h │ │ ├── persistent_cache.h │ │ ├── raw_key_operator.h │ │ ├── slice.h │ │ ├── slog.h │ │ ├── status.h │ │ ├── table.h │ │ ├── table_builder.h │ │ ├── table_utils.h │ │ ├── tera_key.h │ │ └── write_batch.h │ ├── nfs.h │ ├── snappy-stubs-public.h │ └── snappy.h ├── issues │ └── issue178_test.cc ├── persistent_cache │ ├── env_flash_compatibility.cc │ ├── hash_table.h │ ├── hash_table_evictable.h │ ├── lrulist.h │ ├── persistent_cache_file.cc │ ├── persistent_cache_file.h │ ├── persistent_cache_impl.cc │ ├── persistent_cache_impl.h │ ├── persistent_cache_metadata.cc │ ├── persistent_cache_metadata.h │ ├── sharded_persistent_cache_impl.cc │ └── sharded_persistent_cache_impl.h ├── port │ ├── README │ ├── atomic_pointer.h │ ├── port.h │ ├── port_example.h │ ├── port_posix.cc │ ├── port_posix.h │ ├── thread_annotations.h │ └── win │ │ └── stdint.h ├── sample │ ├── Makefile │ └── sample.cpp ├── table │ ├── block.cc │ ├── block.h │ ├── block_builder.cc │ ├── block_builder.h │ ├── filter_block.cc │ ├── filter_block.h │ ├── filter_block_test.cc │ ├── format.cc │ ├── format.h │ ├── iterator.cc │ ├── iterator_wrapper.h │ ├── merger.cc │ ├── merger.h │ ├── persistent_cache_helper.cc │ ├── persistent_cache_helper.h │ ├── table.cc │ ├── table_builder.cc │ ├── table_test.cc │ ├── two_level_iterator.cc │ └── two_level_iterator.h └── util │ ├── arena.cc │ ├── arena.h │ ├── arena_test.cc │ ├── bloom.cc │ ├── bloom_test.cc │ ├── cache.cc │ ├── cache_test.cc │ ├── coding.cc │ ├── coding.h │ ├── coding_test.cc │ ├── comparator.cc │ ├── crc32c.cc │ ├── crc32c.h │ ├── crc32c_test.cc │ ├── dfs.cc │ ├── dfs_read_thread_limiter.h │ ├── env.cc │ ├── env_dfs.cc │ ├── env_flash.cc │ ├── env_inmem.cc │ ├── env_mock.cc │ ├── env_posix.cc │ ├── env_posix.h │ ├── env_test.cc │ ├── filter_policy.cc │ ├── hash.cc │ ├── hash.h │ ├── hdfs.cc │ ├── hdfs.h │ ├── hdfs2.cc │ ├── hdfs_util.h │ ├── histogram.cc │ ├── histogram.h │ ├── lg_coding.cc │ ├── logging.cc │ ├── logging.h │ ├── mutexlock.h │ ├── nfs.cc │ ├── nfs.h │ ├── nfs_version.h │ ├── nfs_wrapper.h │ ├── options.cc │ ├── posix_logger.h │ ├── random.cc │ ├── random.h │ ├── raw_key_operator.cc │ ├── raw_key_operator_test.cc │ ├── slog.cc │ ├── status.cc │ ├── stop_watch.h │ ├── string_ext.cc │ ├── string_ext.h │ ├── tera_key.cc │ ├── tera_key_test.cc │ ├── testharness.cc │ ├── testharness.h │ ├── testutil.cc │ ├── testutil.h │ ├── thread.cc │ ├── thread.h │ ├── thread_pool.cc │ └── thread_pool.h ├── load_balancer ├── action.h ├── action_generator.h ├── action_generators.cc ├── action_generators.h ├── actions.cc ├── actions.h ├── balancer.h ├── cluster.cc ├── cluster.h ├── cost_function.h ├── cost_functions.cc ├── cost_functions.h ├── lb_entry.cc ├── lb_entry.h ├── lb_flags.cc ├── lb_impl.cc ├── lb_impl.h ├── lb_node.h ├── lb_service_impl.cc ├── lb_service_impl.h ├── meta_balancer.cc ├── meta_balancer.h ├── options.h ├── plan.h ├── random.h ├── test │ ├── action_generators_test.cc │ ├── actions_test.cc │ ├── balancer_test_main.cc │ ├── cluster_test.cc │ ├── cost_functions_test.cc │ └── random_test.cc ├── unity_balancer.cc └── unity_balancer.h ├── master ├── abnormal_node_mgr.cc ├── abnormal_node_mgr.h ├── availability.cc ├── availability.h ├── create_table_procedure.cc ├── create_table_procedure.h ├── delete_table_procedure.cc ├── delete_table_procedure.h ├── disable_table_procedure.cc ├── disable_table_procedure.h ├── enable_table_procedure.cc ├── enable_table_procedure.h ├── load_tablet_procedure.cc ├── load_tablet_procedure.h ├── master_entry.cc ├── master_entry.h ├── master_env.cc ├── master_env.h ├── master_flags.cc ├── master_impl.cc ├── master_impl.h ├── master_state_machine.cc ├── master_state_machine.h ├── master_zk_adapter.cc ├── master_zk_adapter.h ├── merge_tablet_procedure.cc ├── merge_tablet_procedure.h ├── move_tablet_procedure.cc ├── move_tablet_procedure.h ├── multi_tenancy_service_impl.cc ├── multi_tenancy_service_impl.h ├── procedure.h ├── procedure_executor.cc ├── procedure_executor.h ├── procedure_limiter.h ├── remote_master.cc ├── remote_master.h ├── remote_multi_tenancy_service.cc ├── remote_multi_tenancy_service.h ├── scheduler.h ├── set_quota_procedure.cc ├── set_quota_procedure.h ├── split_tablet_procedure.cc ├── split_tablet_procedure.h ├── state_machine.h ├── table_state_machine.cc ├── table_state_machine.h ├── tablet_manager.cc ├── tablet_manager.h ├── tablet_state_machine.cc ├── tablet_state_machine.h ├── tabletnode_manager.cc ├── tabletnode_manager.h ├── test │ ├── abnormal_node_test.cc │ ├── create_table_procedure_test.cc │ ├── load_tablet_procedure_test.cc │ ├── master_impl_test.cc │ ├── master_state_machine_test.cc │ ├── master_test.cc │ ├── merge_tablet_procedure_test.cc │ ├── mock_master_zk_adapter.h │ ├── move_tablet_procedure_test.cc │ ├── procedure_executor_test.cc │ ├── procedure_limiter_test.cc │ ├── set_quota_procedure_test.cc │ ├── split_tablet_procedure_test.cc │ ├── table_state_machine_test.cc │ ├── tablet_state_machine_test.cc │ ├── tabletnode_state_transition.cc │ ├── trackable_gc_test.cc │ ├── unload_tablet_procedure_test.cc │ ├── update_auth_procedure_test.cc │ └── update_table_procedure_test.cc ├── unload_tablet_procedure.cc ├── unload_tablet_procedure.h ├── update_auth_procedure.cc ├── update_auth_procedure.h ├── update_table_procedure.cc ├── update_table_procedure.h ├── user_manager.cc ├── user_manager.h ├── workload_scheduler.cc └── workload_scheduler.h ├── monitor └── teramo_main.cc ├── observer ├── executor │ ├── key_selector.h │ ├── notification_impl.cc │ ├── notification_impl.h │ ├── notify_cell.h │ ├── random_key_selector.cc │ ├── random_key_selector.h │ ├── scanner_entry.cc │ ├── scanner_impl.cc │ ├── scanner_impl.h │ ├── tablet_bucket_key_selector.cc │ └── tablet_bucket_key_selector.h ├── observer_demo │ ├── demo_entry.cc │ ├── demo_entry.h │ ├── demo_observer.cc │ ├── demo_observer.h │ └── observe_demo_main.cc ├── rowlocknode │ ├── fake_rowlocknode_zk_adapter.cc │ ├── fake_rowlocknode_zk_adapter.h │ ├── ins_rowlocknode_zk_adapter.cc │ ├── ins_rowlocknode_zk_adapter.h │ ├── remote_rowlocknode.cc │ ├── remote_rowlocknode.h │ ├── rowlock_db.h │ ├── rowlocknode_entry.cc │ ├── rowlocknode_entry.h │ ├── rowlocknode_impl.cc │ ├── rowlocknode_impl.h │ ├── rowlocknode_zk_adapter.cc │ ├── rowlocknode_zk_adapter.h │ └── rowlocknode_zk_adapter_base.h ├── rowlockproxy │ ├── remote_rowlock_proxy.cc │ ├── remote_rowlock_proxy.h │ ├── rowlock_proxy_entry.cc │ ├── rowlock_proxy_entry.h │ ├── rowlock_proxy_impl.cc │ ├── rowlock_proxy_impl.h │ ├── rowlock_proxy_zk_adapter.cc │ └── rowlock_proxy_zk_adapter.h └── test │ ├── notification_test.cc │ ├── observer_test.cc │ ├── rowlock_proxy_test.cc │ ├── rowlock_test.cc │ ├── scanner_test.cc │ └── tablet_bucket_key_selector_test.cc ├── proto ├── access_control.proto ├── filter.proto ├── http.proto ├── kv_helper.cc ├── kv_helper.h ├── lb_client.cc ├── lb_client.h ├── load_balancer_rpc.proto ├── master_client.cc ├── master_client.h ├── master_mutli_tenancy_client.cc ├── master_mutli_tenancy_client.h ├── master_rpc.proto ├── mock_master_client.h ├── monitor.proto ├── proto_helper.cc ├── proto_helper.h ├── quota.proto ├── rowlocknode_rpc.proto ├── rpc_client.cc ├── rpc_client.h ├── stat_table.proto ├── status_code.proto ├── table_meta.proto ├── table_schema.proto ├── tabletnode.proto ├── tabletnode_client.cc ├── tabletnode_client.h ├── tabletnode_rpc.proto ├── test_helper.cc ├── test_helper.h └── timeoracle_rpc.proto ├── quota ├── flow_controller.h ├── helpers │ ├── master_quota_helper.cc │ ├── master_quota_helper.h │ ├── quota_utils.cc │ └── quota_utils.h ├── limiter │ ├── general_quota_limiter.cc │ ├── general_quota_limiter.h │ ├── general_rate_limiter.cc │ ├── general_rate_limiter.h │ ├── limiter_factory.h │ ├── quota_limiter.h │ └── rate_limiter.h ├── master_quota_entry.cc ├── master_quota_entry.h ├── quota_entry.cc ├── quota_entry.h ├── quota_limiter_container.cc ├── quota_limiter_container.h ├── test │ ├── master_quota_entry_test.cc │ ├── master_quota_helper_test.cc │ ├── quota_entry_test.cc │ └── ts_write_flow_controller_test.cc ├── ts_write_flow_controller.cc └── ts_write_flow_controller.h ├── sample ├── Makefile ├── atomic_sample.cc ├── global_txn_async_sample.cc ├── global_txn_sync_sample.cc ├── tera_batch_mutation_sample.cc ├── tera_row_txn_sample.cc └── tera_sample.cc ├── sdk ├── batch_mutation_impl.cc ├── batch_mutation_impl.h ├── client_impl.cc ├── client_impl.h ├── cookie.cc ├── cookie.h ├── filter.cc ├── filter_comparator.cc ├── filter_list.cc ├── filter_list_base.cc ├── filter_list_base.h ├── filter_list_with_and.cc ├── filter_list_with_and.h ├── filter_list_with_or.cc ├── filter_list_with_or.h ├── filter_utils.cc ├── filter_utils.h ├── global_txn.cc ├── global_txn.h ├── global_txn_internal.cc ├── global_txn_internal.h ├── http │ ├── http.cc │ └── http_sample.py ├── java │ ├── native-src │ │ ├── gen_jni_headers.sh │ │ ├── jni.h │ │ ├── jni_md.h │ │ ├── jni_tera_base.cc │ │ ├── jni_tera_base.h │ │ ├── jni_tera_client.cc │ │ ├── jni_tera_client.h │ │ ├── jni_tera_common.cc │ │ ├── jni_tera_common.h │ │ ├── jni_tera_mutation.cc │ │ ├── jni_tera_mutation.h │ │ ├── jni_tera_reader.cc │ │ ├── jni_tera_reader.h │ │ ├── jni_tera_result.cc │ │ ├── jni_tera_result.h │ │ ├── jni_tera_result_stream.cc │ │ ├── jni_tera_result_stream.h │ │ ├── jni_tera_scan.cc │ │ ├── jni_tera_scan.h │ │ ├── jni_tera_table.cc │ │ └── jni_tera_table.h │ ├── pom.xml │ └── tera │ │ ├── pom.xml │ │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── baidu │ │ │ └── tera │ │ │ └── client │ │ │ ├── ScanDescriptor.java │ │ │ ├── ScanResultStreamImpl.java │ │ │ ├── TeraBase.java │ │ │ ├── TeraCell.java │ │ │ ├── TeraClientImpl.java │ │ │ ├── TeraMutationImpl.java │ │ │ ├── TeraReaderImpl.java │ │ │ ├── TeraResultImpl.java │ │ │ ├── TeraScanImpl.java │ │ │ └── TeraTableImpl.java │ │ └── test │ │ └── java │ │ └── com │ │ └── baidu │ │ └── tera │ │ └── client │ │ ├── TeraBaseTest.java │ │ ├── TeraClientImplTest.java │ │ └── TeraTableImplTest.java ├── murmur_hash_impl.cc ├── mutate_impl.cc ├── mutate_impl.h ├── python │ ├── TeraSdk.py │ ├── checker.sh │ └── sample.py ├── read_impl.cc ├── read_impl.h ├── rowlock_client.cc ├── rowlock_client.h ├── rowlock_client_zk_adapter.cc ├── rowlock_client_zk_adapter.h ├── scan.cc ├── scan_impl.cc ├── scan_impl.h ├── schema.cc ├── schema_impl.cc ├── schema_impl.h ├── sdk_flags.cc ├── sdk_metric_name.h ├── sdk_perf.cc ├── sdk_perf.h ├── sdk_task.cc ├── sdk_task.h ├── sdk_utils.cc ├── sdk_utils.h ├── sdk_zk.cc ├── sdk_zk.h ├── single_row_txn.cc ├── single_row_txn.h ├── stat_table.cc ├── stat_table.h ├── table_impl.cc ├── table_impl.h ├── tera.cc ├── tera_easy.cc ├── tera_easy.h ├── tera_replication.cc ├── tera_replication.h ├── test │ ├── batch_muation_test.cc │ ├── filter_list_test.cc │ ├── filter_utils_test.cc │ ├── global_txn_batch_op.cc │ ├── global_txn_internal_test.cc │ ├── global_txn_test.cc │ ├── global_txn_test_tool.cc │ ├── global_txn_test_tool.h │ ├── global_txn_testutils.cc │ ├── global_txn_testutils.h │ ├── hash_table_test.cc │ ├── mock_table.h │ ├── scan_impl_test.cc │ ├── sdk_client_test.cc │ ├── sdk_table_test.cc │ ├── sdk_test.cc │ ├── sdk_timeout_manager_test.cc │ ├── sdk_utils_test.cc │ └── value_filter_test.cc ├── timeoracle_client_impl.cc ├── timeoracle_client_impl.h ├── transaction_wrapper.h └── value_filter.cc ├── tabletnode ├── remote_tabletnode.cc ├── remote_tabletnode.h ├── rpc_compactor.h ├── rpc_schedule.cc ├── rpc_schedule.h ├── rpc_schedule_policy.cc ├── rpc_schedule_policy.h ├── tablet_manager.cc ├── tablet_manager.h ├── tabletnode_entry.cc ├── tabletnode_entry.h ├── tabletnode_flags.cc ├── tabletnode_impl.cc ├── tabletnode_impl.h ├── tabletnode_metric_name.h ├── tabletnode_sysinfo.cc ├── tabletnode_sysinfo.h ├── tabletnode_zk_adapter.cc ├── tabletnode_zk_adapter.h └── test │ ├── mock_tablet_manager.h │ ├── mock_zk_adapter.h │ ├── readtablet_test.cc │ ├── remote_tabletnode_test.cc │ ├── tablet_manager_test.cc │ ├── tabletnode_entry_test.cc │ ├── tabletnode_impl_test.cc │ └── tabletnode_sysinfo_test.cc ├── tera_c.cc ├── tera_flags.cc ├── tera_main.cc ├── tera_main_wrapper.cc ├── tera_test_main.cc ├── teracli_main.cc ├── terautil.cc ├── timeoracle ├── bench │ └── timeoracle_bench.cc ├── remote_timeoracle.h ├── test │ └── timeoracle_test.cc ├── timeoracle.cc ├── timeoracle.h ├── timeoracle_entry.cc ├── timeoracle_entry.h ├── timeoracle_zk_adapter.cc └── timeoracle_zk_adapter.h ├── timeoracle_main.cc ├── types.h ├── utils ├── cli_mt ├── config_utils.cc ├── config_utils.h ├── crypt.cc ├── crypt.h ├── fragment.cc ├── fragment.h ├── libzookeeper_mt.so ├── network_utils.cc ├── network_utils.h ├── prop_tree.cc ├── prop_tree.h ├── rpc_timer_list.cc ├── rpc_timer_list.h ├── schema_utils.cc ├── schema_utils.h ├── string_util.cc ├── string_util.h ├── test │ ├── fragment_test.cc │ ├── prop_tree_test.cc │ ├── string_util_test.cc │ └── tprinter_test.cc ├── tprinter.cc ├── tprinter.h ├── utils_cmd.cc └── utils_cmd.h ├── version.h └── zk ├── dummy_zk_adapter.h ├── zk_adapter.cc ├── zk_adapter.h ├── zk_lock.cc ├── zk_lock.h ├── zk_util.cc └── zk_util.h /.gitignore: -------------------------------------------------------------------------------- 1 | *.o 2 | *.a 3 | *.pyc 4 | 5 | #protobuf auto-gen code 6 | src/proto/*.pb.* 7 | 8 | #version.cc is auto generated 9 | src/version.cc 10 | 11 | #leveldb binary 12 | src/leveldb/db_import 13 | src/leveldb/leveldbutil 14 | src/leveldb/tera_bench 15 | 16 | depends.mk 17 | build.conf 18 | 19 | .build/ 20 | build/ 21 | thirdsrc/ 22 | thirdparty/ 23 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | env: 2 | global: 3 | - secure: "EBGwhqHaPbERmOAPA7a1IprZZdFjEZqnuekgkNTBtzmGTaIYuh1BbSNGmVtnj3DuXuqAusiYN6olW2lMax15Fqw3Mwh++vh6DJFQ4wePImCzot7D4fTcopmNS2yoPl0IeyL/sLyQrxjflBfoTzw6DUZAXiU55gGB1faqCAfM5sQ=" 4 | - CC=gcc-4.8 5 | - CXX=g++-4.8 6 | addons: 7 | apt: 8 | sources: 9 | - ubuntu-toolchain-r-test 10 | packages: 11 | - gcc-4.8 12 | - g++-4.8 13 | - gdb 14 | - apport 15 | coverity_scan: 16 | project: 17 | name: "baidu/tera" 18 | description: "An Internet-Scale Database." 19 | notification_email: tera@users.noreply.github.com 20 | build_command_prepend: "make clean" 21 | build_command: "make -j4" 22 | branch_pattern: coverity_scan 23 | 24 | before_install: 25 | - ulimit -c unlimited -S 26 | - ulimit -a 27 | - echo -n | openssl s_client -connect scan.coverity.com:443 | 28 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' | 29 | sudo tee -a /etc/ssl/certs/ca- 30 | 31 | install: 32 | - bash -x build.sh origin 33 | 34 | script: 35 | - make check > /dev/null 2>&1 36 | 37 | -------------------------------------------------------------------------------- /benchmark/README.md: -------------------------------------------------------------------------------- 1 | 做个总体介绍吧 2 | @李康 3 | -------------------------------------------------------------------------------- /benchmark/build_ycsb.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [[ ! -e ycsb-0.12.0.tar.gz ]]; then 4 | wget --no-check-certificate https://github.com/brianfrankcooper/YCSB/archive/0.12.0.tar.gz -O ycsb-0.12.0.tar.gz 5 | fi 6 | 7 | rm -rf YCSB-0.12.0 8 | tar zxvf ycsb-0.12.0.tar.gz 9 | 10 | cp ycsb4tera/tera/ YCSB-0.12.0/ -fr 11 | sed -i '/tarantool/a"tera":"com.yahoo.ycsb.TeraDB",' YCSB-0.12.0/bin/ycsb 12 | sed -i '/tarantool<\/module>/atera' YCSB-0.12.0/pom.xml 13 | sed -i '/tarantool.version/a0.6.1' YCSB-0.12.0/pom.xml 14 | cd YCSB-0.12.0 && mvn -pl com.yahoo.ycsb:tera-binding -am clean package 15 | -------------------------------------------------------------------------------- /benchmark/load_data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ $# != 4 ]; then 4 | echo "$0 ROW_NUM VALUE_SIZE COLUMN_NUM TABLE_NAME" 5 | exit 0 6 | fi 7 | 8 | ROW_NUM=$1 9 | VALUE_SIZE=$2 10 | COLUMN_NUM=$3 11 | TABLE_NAME=$4 12 | 13 | cd YCSB-0.12.0 14 | ln -s ../tera_mark tera_mark 15 | ln -s ../tera.flag tera.flag 16 | 17 | bin/ycsb load tera -p workload=com.yahoo.ycsb.workloads.CoreWorkload \ 18 | -p recordcount=$ROW_NUM \ 19 | -p fieldlength=$VALUE_SIZE \ 20 | -p fieldcount=$COLUMN_NUM \ 21 | -p exportfile=ycsb.out \ 22 | | ./tera_mark --mode=m --tablename=$TABLE_NAME --type=async --verify=false 23 | -------------------------------------------------------------------------------- /benchmark/run_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [[ $# != 9 || $6 -lt 0 || $6 -gt 100 ]]; then 4 | echo "$0 DIST[zipfian, uniform, latest] ROW_NUM OP_NUM VALUE_SIZE COLUMN_NUM UPDATE_PROPORTION[0~100] OP_SPEED THREAD_NUM TABLE_NAME" 5 | exit 1 6 | fi 7 | 8 | DIST=$1 9 | ROW_NUM=$2 10 | OP_NUM=$3 11 | VALUE_SIZE=$4 12 | COLUMN_NUM=$5 13 | UPDATE_PROPORTION=$6 14 | OP_SPEED=$7 15 | THREAD_NUM=$8 16 | TABLE_NAME=$9 17 | 18 | UPDATE_PROPORTION=`echo $6 | awk '{printf("%.2f",$1/100)}'` 19 | READ_PROPORTION=`echo $6 | awk '{printf("%.2f",(100-$1)/100)}'` 20 | 21 | echo "$UPDATE_PROPORTION" 22 | echo "$READ_PROPORTION" 23 | 24 | cd YCSB-0.12.0 25 | ln -s ../tera_mark tera_mark 26 | ln -s ../tera.flag tera.flag 27 | 28 | bin/ycsb run tera -p workload=com.yahoo.ycsb.workloads.CoreWorkload \ 29 | -p requestdistribution=$DIST \ 30 | -p recordcount=$ROW_NUM \ 31 | -p operationcount=$OP_NUM \ 32 | -p fieldlength=$VALUE_SIZE \ 33 | -p fieldcount=$COLUMN_NUM \ 34 | -p updateproportion=$UPDATE_PROPORTION \ 35 | -p readproportion=$READ_PROPORTION \ 36 | -p target=$OP_SPEED \ 37 | -p thread=$THREAD_NUM \ 38 | -p exportfile=ycsb.out \ 39 | | ./tera_mark --mode=m --tablename=$TABLE_NAME --type=async --verify=false 40 | 41 | exit $? 42 | 43 | -------------------------------------------------------------------------------- /benchmark/ycsb4tera/tera/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.yahoo.ycsb 7 | root 8 | 0.12.0 9 | 10 | 11 | tera-binding 12 | Tera Basic Binding 13 | jar 14 | 15 | 16 | false 17 | 1.9.4 18 | 19 | 20 | 21 | 22 | org.codehaus.jackson 23 | jackson-mapper-asl 24 | ${jackson.api.version} 25 | 26 | 27 | org.codehaus.jackson 28 | jackson-core-asl 29 | ${jackson.api.version} 30 | 31 | 32 | com.yahoo.ycsb 33 | core 34 | ${project.version} 35 | 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /conf/tera.flag: -------------------------------------------------------------------------------- 1 | # common 2 | --log_dir=../log 3 | --v=6 4 | --logbufsecs=0 5 | 6 | # master 7 | --tera_master_port=1100 8 | --tera_master_split_tablet_size=2048 9 | 10 | # tablet node 11 | --tera_tabletnode_port=2200 12 | --tera_tabletnode_path_prefix=${dfs_prefix} 13 | --tera_tabletnode_cache_paths=${local_path} 14 | 15 | # zk 16 | --tera_zk_addr_list=${zk_addr} 17 | --tera_zk_root_path=${zk_root} 18 | --tera_zk_timeout=10000 19 | -------------------------------------------------------------------------------- /doc/cn/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Tera文档专区 3 | ## 简介 4 | 5 | [系统设计](../tera_design.md) 6 | 7 | [Tera表格数据模型及实现](../data_model.md) 8 | 9 | ## 使用帮助 10 | 11 | [体验单机Tera](onebox.md) 12 | 13 | [命令行工具teracli使用方法](../tools/teracli.md) 14 | 15 | [集群间数据迁移的dump工具terautil使用方法](../tools/terautil.md) 16 | 17 | [造数据的工具 & 读写数据使用方法](../tools/benchmark.md) 18 | 19 | [性能测试工具ycsb的使用方法](../tools/ycsb.md) 20 | 21 | [主要api使用方法](../sdk_reference/readme.md) 22 | 23 | [搭建tera集群](cluster_setup.md) 24 | 25 | [各flag配置项含义介绍](../tera_flag.md) 26 | 27 | ## 性能优化 28 | 29 | [针对表格不同读写特性的性能优化建议](../perf_opz.md) 30 | 31 | ## 功能实现 32 | 33 | [动态负载均衡](../load-balance.md) 34 | 35 | [数据删除的实现](../data-deletion-in-tera.md) 36 | 37 | [tablet的可用性统计实现](../tablet-availability.md) 38 | 39 | [master、ts、client的交互](../master-ts-client-interactive.md) 40 | 41 | ## 版本发布 42 | 43 | [版本发布及管理](../release_management.md) 44 | 45 | -------------------------------------------------------------------------------- /doc/cn/roadmap.md: -------------------------------------------------------------------------------- 1 | # Roadmap 2 | 3 | ## 1.0版 4 | * 完善同步读写接口 5 | * 完善一键编译 6 | * 常见Linux发行版的二进制发布 7 | * docker发布 8 | * 与hbase的性能对比报告 9 | 10 | ## 1.1版 11 | * 支持mapreduce 12 | * 兼容hbase接口 13 | 14 | ## 1.2版 15 | * 去zk/nexus依赖 16 | * 去boost依赖 17 | * 用户管理，权限和quota 18 | * 记录级cache 19 | * 灵活的compact算法 20 | * 快照 21 | 22 | ## 1.3版 23 | * 多级索引 24 | * 分布式事务 25 | * 冷备 26 | 27 | ## 1.4版 28 | * 热备 29 | * 跨数据中心备份 30 | -------------------------------------------------------------------------------- /doc/cn/sdk_guide_java.md: -------------------------------------------------------------------------------- 1 | # Tera Java SDK 2 | 3 | # 背景 4 | 5 | * Tera Java SDK通过JNI（libjni_tera.so）调用native接口实现。 6 | * 使用Maven进行编译。 7 | 8 | # 体验方法 9 | 10 | 1. 编译Tera，参考Build[../../build.md] 11 | 1. 部署Tera运行环境，可以参考[onebox](../onebox-cn.md)进行简单搭建。 12 | 1. 将编译出的libjni_tera.so的路径加入环境变量**LD_LIBRARY_PATH**。 13 | 1. 将环境变量**TERA_CONF_PATH**设为tera.flag的路径。 14 | 1. 进入src/sdk/java目录，执行命令**mvn package**运行单测（可以参考单测使用sdk接口），并产出JAR文件。 15 | 1. 将JAR文件安装到Maven库，以便将来依赖使用。 16 | -------------------------------------------------------------------------------- /doc/cn/to_be_a_committer.md: -------------------------------------------------------------------------------- 1 | # Tera Committer 2 | 为什么你需要成为tera的committer？因为你一旦成为了tera的committer，就意味着你的代码将有机会运行在一个实际的大规模生产环境中。同时，你的代码也能得到足够的应用场景验证，对实践大规模分布式编程，非常有帮助。因此期待你的加入！:-) 3 | 4 | 5 | # 如何成为Tera Committer 6 | 作为一个tera的commiter，首要条件是需要对tera代码有足够的熟悉度。因此，除了熟读代码之外，要将tera真正run起来，这样能加深系统的理解。一旦发现问题，可以通过提issue和pull request，获得问题反馈和解决（当然，我们更欢迎自己解决问题的同学）。 7 | 8 | 提交issue时，如果是提交bug，需要描述复现方法、具体日志记录等；如果是新功能，则需要描述思路、附带设计文档，方便其他人review你的想法。 9 | 10 | 提交pull request时，需要按照一定的[规则](../en/contributor.md)填写commit log。 11 | 12 | 当你给tera开发了一到两个核心功能之后，就表明你对tera有了一定的把控能力，此时我们会将你列入Tera Committer的候选列表，经过内部讨论通过，就能正式成为tera的committer！:-) 13 | 14 | # 如何进行Code Review 15 | 对他人的代码进行Code Review时，需要本着认真负责的原则，并按一定规则进行： 16 | 17 | 1、每个pr惯例是需要得到两个LGTM，然后由最后一个LGTM的人执行merge； 18 | 2、merge pr时，提pr的人，不能merge自己的pr； 19 | 3、code review时，避免两个都是新的commiter，完成LGTM就自行merge代码； 20 | 4、push代码时，禁止rebase，否则难以看出代码diff; 21 | 5、merge代码时，需要将commit log进行合并，每个pr保证只有一个commint log。 22 | 23 | ps：每个commiter都应自觉遵守上述规则。 24 | -------------------------------------------------------------------------------- /doc/en/README.md: -------------------------------------------------------------------------------- 1 | # Tera Docs 2 | 3 | ## User Manual 4 | 5 | [Pseudo Distributed Mode](onebox.md) 6 | 7 | [API Manual](sdk_guide.md) 8 | 9 | [Teracli Manual](teracli.md) 10 | 11 | ## Benchmark 12 | 13 | [Performance](performance.md) 14 | 15 | ## Join Us 16 | 17 | [How to contribute](contributor.md) 18 | 19 | # More Docs (Todo List) 20 | 21 | ## Introduction 22 | 23 | 1. Tera Design 24 | 1. Data Model and Implementation 25 | 26 | ## User Manual 27 | 28 | 1. Distributed Mode 29 | 1. Flag Manual 30 | 31 | ## Performance Profiling 32 | 33 | 1. Read/Write 34 | 35 | ## Inside Tera 36 | 37 | 1. Design and Implementation of Load Balance 38 | 1. How Delete Data Works 39 | 1. How Tablet Availability Check Works 40 | 1. Interaction Between Master, Tablet Server and Client 41 | 1. Design and implementation of master 42 | 1. Design and implementation of tablet server 43 | 1. SDK manual(user friendly: e.g. searchable, foldable) 44 | 1. Performance benchmark on various environment 45 | 1. Design trade-off about performance(how to implement high perfromance random read/write) 46 | 1. Monitor manual 47 | 1. How fast and low cost split & merge works 48 | 49 | ## Version Management 50 | 51 | 1. Release Management 52 | 53 | 54 | -------------------------------------------------------------------------------- /doc/en/performance.md: -------------------------------------------------------------------------------- 1 | # Performance 2 | 3 | ## Hardware info 4 | ``` 5 | CPU: 2X Intel Xeon 16 Core E5-2450 v2 2.50GHz 6 | RAM: 8X 16GB 1333 MHz DDR3 7 | DISK: 5X 3TB SATA 8 | FLASH: 2X 480G SSD MLC 9 | ``` 10 | 11 | ## Software info 12 | ``` 13 | OS: CentOS 4.3 14 | KERNEL: 2.6.32 15 | ``` 16 | 17 | ## Benckmark 18 | ``` 19 | SCHEMA: { 20 | lg0 { 21 | cf0 22 | }, 23 | lg1 { 24 | cf1 25 | } 26 | } 27 | DATA: key-size=24, value-size=1000, key-count=300000000 28 | RESULT: 29 | BENCHMARK Throughput(MB/s) Queries/s Latency(ms) 30 | Write 9 - 30 31 | Random-Read - 32000 2 32 | Seq-Read 20 - - 33 | ``` 34 | -------------------------------------------------------------------------------- /doc/en/roadmap.md: -------------------------------------------------------------------------------- 1 | # Roadmap 2 | 3 | ## Version 1.0 4 | * Low latency synchronous API 5 | * Powerful one-key build script 6 | * Binary releases for common Linux distributions 7 | * Docker image release 8 | * Performance report vs. HBase 9 | 10 | ## Version 1.1 11 | * Support MapReduce 12 | * Compliance with HBase API 13 | 14 | ## Version 1.2 15 | * Remove ZooKeeper/Nexus dependency 16 | * Remove Boost dependency 17 | * User management, including permission and quota 18 | * Record(Row/Key) cache 19 | * Pluggable compact algorithms 20 | * Snapshot 21 | 22 | ## Version 1.3 23 | * Secondary indexes 24 | * Distribute transaction 25 | * Cold backup 26 | 27 | ## Version 1.4 28 | * Tablet standby 29 | * Global replication 30 | -------------------------------------------------------------------------------- /doc/en/sdk_guide_java.md: -------------------------------------------------------------------------------- 1 | # Tera Java SDK Guide 2 | 3 | ## Implementation 4 | 5 | * Java-SDK invokes [native-SDK](sdk_guide.md) through JNI library (libjni_tera.so). 6 | * Java-SDK uses [Maven](https://maven.apache.org/) to automate the build of projects. 7 | 8 | ## Have a quick try 9 | 10 | 1. Build Tera. See [BUILD](../../build.md). 11 | 1. Deploy Tera. See [pseudo-distributed mode](../onebox.md). 12 | 1. Add the path of *libjni_tera.so* to environment variable **LD_LIBRARY_PATH**. 13 | 1. Set environment variable **TERA_CONF_PATH** to the path of *tera.flag*. 14 | 1. Under the directory src/sdk/java, execute command **mvn package** to run the unittest and build JAR files. 15 | 1. Install the output JAR files into the Maven repository to use Tera Java SDK in the future. 16 | -------------------------------------------------------------------------------- /doc/global_txn.md: -------------------------------------------------------------------------------- 1 | # Tera全局事务的原理及实现 2 | 3 | [image-1]: ../resources/images/global_txn.png 4 | -------------------------------------------------------------------------------- /doc/image/persistent_cache_arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidu/tera/dbcd28af792d879d961bf9fc7eb60de81b437646/doc/image/persistent_cache_arch.png -------------------------------------------------------------------------------- /doc/image/persistent_cache_meta_data_arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidu/tera/dbcd28af792d879d961bf9fc7eb60de81b437646/doc/image/persistent_cache_meta_data_arch.png -------------------------------------------------------------------------------- /doc/kpi.md: -------------------------------------------------------------------------------- 1 | # tera指标建设 2 | 3 | ## 建设哪些指标？ 4 | 1. 性能 5 | 4个维度(随机写\随机读\顺序写\顺序读) * 3种介质(磁盘\SSD\内存) 6 | 12种场景(KV\少列\多列 * 1k/100k * uniform\zipfian) 7 | 2. IO放大 8 | 写放大&读放大&存储空间放大 9 | 3. 可用性 10 | tablet可服务比例、单tablet连续不可服务时间 11 | 4. 扩展性 12 | 集群规模对吞吐的影响 = 集群吞吐/(单机吞吐*机器数） 13 | 5. 可靠性 14 | 用户数据&meta数据 15 | -------------------------------------------------------------------------------- /doc/master-ts-client-interactive.md: -------------------------------------------------------------------------------- 1 | # 通信协议 2 | 使用protobuf定义通信协议，参见`src/proto/`. 3 | 4 | # meta表 5 | meta表的加载： 6 | master找到meta表(meta表持久化存储的位置是约定好的，大家都知道的)， 7 | 选择某个ts加载meta表，然后将载有meta表的ts位置[ip:port]记录在zk下约定好的文件中。 8 | 9 | client如何获取meta表？ 10 | 参考[meta表的加载]，从zk读取某个特定文件就可以知道哪个ts加载了meta表，进而访问meta表。 11 | 12 | # 系统启动(和三者交互有关的部分) 13 | **ts的启动** 14 | * 在zk上注册特定文件 15 | * 在该文件中写入自身[ip:port] 16 | 17 | **master的启动** 18 | * 通过zk获取master锁 19 | * 扫描特定目录发现活跃状态的ts 20 | * 与每个ts通信获取已经被载入的tablet列表 21 | * 如果meta表未载入，则master命令某个ts载入meta表 22 | * 读取meta表，获知所有tablet的信息以及未被载入的tablet集合 23 | 24 | # 寻址 25 | **client如何获知某个tablet在哪个ts上？** 26 | 先访问meta表，然后根据meta表的内容知道某个tablet在哪个ts上。 27 | 28 | **client如何获知master位置？** 29 | master启动时会在zk上的master锁中记下自己的位置[ip:port]. 30 | 31 | **master如何获知ts的信息？** 32 | 参考master的启动，可以获知master初始化时各个ts的状态(例如位置[ip:port])， 33 | 34 | 通过zk获知ts的加入或退出。 35 | -------------------------------------------------------------------------------- /doc/new_sdk_dev.md: -------------------------------------------------------------------------------- 1 | # C++ SDK接口重构 2 | 3 | # 目标 4 | 5 | * 提升接口可读性，降低找接口时间 6 | * 精简接口，提升注释 7 | * 降低用户误用接口的概率 8 | * 内部与接口代码分离 9 | * 保证与旧代码的兼容 10 | 11 | # 原SDK 12 | 13 | * 截至0.5版本 14 | * 所有接口全部在一个头文件（tera.h）里 15 | * 代码过长，很难找到相要的接口 16 | * 不同类的使用频度不同，放在一起会让更重要的类被淹没（比如Client/Table等） 17 | * 代码层次感弱一点，文件名也是代码的重要一部分 18 | 19 | # 新SDK结构 20 | 21 | * 最外层增加一个include目录，表示对外接口 22 | * tera.h移至此目录 23 | * tera_easy.h/tera_ha.h等移至此目录 24 | * 编译时，生效至build目录 25 | * tera.h 26 | * 拆分成若干文件，按对象、功能划分文件 27 | * tera.h自身变成一个入口，使用时依然只需要include一个文件 28 | * 每个类内部（Table/Client/RowMutation等） 29 | * 按接口使用频率重新排序 30 | * 接口分类 31 | * 默认为已发布接口 32 | * DEVELOPING 开发中接口，鼓励完善 33 | * EXPERIMENTAL 正在测试中，鼓励试用 34 | * DEPRECATED 未实现或不建议用户使用，保证兼容性 35 | * 重写接口注释 36 | -------------------------------------------------------------------------------- /doc/newbie.md: -------------------------------------------------------------------------------- 1 | # 怎么学习tera？ 2 | 3 | 4 | 我这里给出个建议的范围，欢迎大家补充、修改。 5 | 6 | 1. 明确Tera产生的背景与设计目标: 7 | 高性能的结构化数据存储，用来存储搜索引擎万亿量级的超链和网页数据。 8 | 1. 理解Tera的数据模型: 9 | Tera数据模型主要参考bigtable设计，可以通过阅读[bigtable论文](http://static.googleusercontent.com/media/research.google.com/zh-CN//archive/bigtable-osdi06.pdf), 重点是理解其中的数据模型和Localitygroup的设计。 10 | 1. 了解tera架构和各模块的功能定位 11 | 这个通过阅读tera设计文档实现，只需要大致了解包含哪些模块，每个模块的职责是什么。 12 | 1. tera都实现了哪些特性，这些特性都是怎么实现的 13 | 自动分片、自动负载均衡、多版本、稀疏表、强一致、行级事务等等 14 | 1. tera的表格是怎么实现的 15 | 怎样将列族、qualifier、时间戳这些映射到底层kv存储中的 16 | 1. 学习底层kv存储Leveldb的实现 17 | 阅读leveldb相关资料，了解leveldb本身的特点，怎样实现读和写的均衡、怎样将随机写转化为顺序写的，同时了解tera中leveldb的分裂和合并是怎么实现。 18 | 1. tera都依赖哪些系统，借助这些系统实现了什么功能 19 | 可以查看Makefile里的编译依赖，看tera都依赖了哪些东西 20 | 1. 表格管理流程 21 | 创建、删除、更新流程，这部分需要自己学习代码 22 | 1. 数据读写流程 23 | 数据读写依赖哪些模块，SDK的meta表是怎么更新和缓存的 24 | 1. tablet分裂流程 25 | tablet分裂过程是怎么产生垃圾的，垃圾收集是怎么做的，分裂失败是怎么回滚的 26 | 1. tablet合并流程 27 | * Master在什么情况下会调度tablet的分裂、合并及迁移 28 | * Localitygroup怎么实现一致性的 29 | 1. Tera的cache 30 | 有哪些cache类型，内存表和flash表是怎么实现的 31 | 1. Tera的性能 32 | Tera的读写性能大概是多少，瓶颈点在哪 33 | 34 | 强烈建议将了解到的内容形成成文档，放在这供后来人学习。 35 | 36 | 学习完后可以尝试回答以下问题： 37 | 1. Tera和HBase的异同，有哪些优势和不足 38 | 2. 除了链接和网页存储，tera还适用于哪些场景 39 | -------------------------------------------------------------------------------- /doc/perf_opz.md: -------------------------------------------------------------------------------- 1 | # 读写性能优化建议 2 | 3 | #### 1. 何时使用kv存储，何时使用表格模型存储？ 4 | 5 | 能使用kv的场景下，尽量使用kv存储。 6 | 7 | 随机读、扫描等简单操作的性能会明显优于表格模式，cpu的利用效率也会更高。表格存储提供了更高级的功能，应该将更核心、更复杂的逻辑应用在其上。 8 | 9 | #### 2. 刚建的表为什么无法支持高吞吐的读写？集群资源够的情况下。 10 | 11 | 刚建的表只有一个tablet，只能使用一台机器，吞吐必然很低。随着读写的持续进行，tera会进行分裂，将负载分散。 12 | 13 | 也可以在建表时**预分多个tablet**，这样就可以直接利用更多的机器资源。 14 | 15 | #### 3. 某些行更新频率很高的表，该行读性能会下降？ 16 | 17 | 有可能，如果存在单行更新频率很高的情况，可以将该表格配置为内存compact模式，可以大幅提升此情况下的读性能。 18 | 19 | #### 4. 表格存在明显的读写热点情况如何处理？ 20 | 21 | 如果热点不是非常集中的情况下，可以将对应tablet进行拆分，分散压力。 22 | 23 | 如果非常集中，极端情况下，可能读写热点全部集中在某一行上（比如dns中，80%的读写可能全集中在少数几个站点上）： 24 | 25 | * 读热点，极端情况下，可能一行的读取压力（每秒数万以上）将cpu占满，此时需要客户端进行适当缓存。 26 | * 写热点，开启内存compact功能，可大幅提升写性能。 27 | 28 | #### 5. 按某一列进行扫描时，cpu消耗很高，而且速度很慢？ 29 | 30 | 当某表列很多时，只扫描其中一列，会将所有列数据扫描一遍，速度会很慢，cpu也会消耗很高。 31 | 32 | 此时可以针对不同列的扫描需求，将不同列拆分至不同Locality Groups，进行IO隔离，扫描时访问所需列数据，节省资源，提高性能。 33 | -------------------------------------------------------------------------------- /doc/prop_tree.md: -------------------------------------------------------------------------------- 1 | # Tera表格字符串描述 —— PropTree 2 | 3 | Tera中的表格结构较复杂，存在多种元素和多种属性可以配置，可配置的选项可能还会增加。 4 | 5 | 针对tera表格结构的特殊情况，我们使用一种带可配置属性的树型结构字符串（PropTree）进行描述。 6 | 7 | 树中的节点语法描述为： 8 | 9 | node_name[][{child1,...}] 10 | 11 | 其中，node_name是节点名称，对应到tera表格中可能是表格名称、locality group、column family名称等；“<>”中是节点的属性集合，可支持0个或多个属性；“{}”中是节点的孩子节点集合，支持递归嵌套（描述多层树结构），如果没有孩子节点可以不写。 12 | 13 | PropTree支持通过 ' ', '\n', '\t' 等字符进行排版，增加可读性。 14 | 15 | PropTree支持通过‘#’进行行注释（类shell）。 16 | 17 | 比如一个典型的tera表格描述文件： 18 | 19 | table_hello { 20 | # 索引lg，小体积，全内存存储，高性能 21 | lg_index { 22 | update_flag 23 | }, 24 | # 属性lg，flash存储，高性能 25 | lg_props { 26 | level , 27 | weight # 权重 28 | }, 29 | # 原始数据 30 | lg_raw { 31 | data 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /doc/read_write_consistency.md: -------------------------------------------------------------------------------- 1 | # Tera中读写一致性的保证 2 | 3 | ## 一、读写时序 - “实读实写” 4 | * 以写操作返回成功为准，自此刻发起的所有读取操作会返回最新更新成功的数据。 5 | * 写操作返回成功前，可能读到旧数据，也可能读到新数据。 6 | * 保证读到全新或全旧的数据，不会出现“残行”。 7 | 8 | ## 二、语义 9 | 10 | tera在kv模型的基础上实现了多LocalityGroup及多列数据模型。读写一致性上，tera实现了两种语义。 11 | 12 | ### 1. 原子单行更新 13 | #### 语义 14 | * 单行中某些列（包含跨LG的列）更新的过程中，要么全部成功，要么全部失败 15 | 16 | #### 实现 17 | * 更新过程包含sync Write-Ahead-Log(WAL)和更新内存 18 | * 更新内存不会失败，所以更新的成功与否取决于sync WAL 19 | * tera实现了同一行的多个LocalityGroup/多列共享同一个WAL 20 | * 更新过程中此WAL sync成功，所有列更新成功；sync失败保证所有列更新同时失败 21 | 22 | ### 2. 并发更新读取隔离 23 | #### 语义 24 | * 读取过程中发生的更新，对本次读取不生效，返回更新前数据 25 | * 在整行（多列）更新完成之前，新数据不可见，旧数据可读；避免出现更新至一半，“残行”出现。 26 | 27 | #### 实现 28 | * 更新过程： 29 | * 更新开始时创建一个临时快照，保证新版本数据生效前旧数据不被删除。 30 | * 自此发生的所有读取操作基于此快照，保证新写入的数据不被读到。 31 | * 所有列更新完成后，删除快照，新发生的读取操作作用至新版本数据上。 32 | * 读取过程： 33 | * 读取开始时持有一个临时快照 34 | * 此快照保护所需数据在读取过程中不被删除 35 | * 以此快照为准进行读取，忽略正在更新的数据 36 | -------------------------------------------------------------------------------- /doc/release_management.md: -------------------------------------------------------------------------------- 1 | # Tera 版本管理 2 | 3 | ## 版本号 4 | 5 | 版本号采用3位版本： 6 | 7 | ``0.1.2`` 8 | 9 | * 1位，重大更新时升级。 10 | * 2位，定期更新，从master拉出。2位版本相同的版本可以稳定升级，不包含新功能。 11 | * 3位，从master中cherry-pick出重要commit merge进已发布版本中。用于已发布版本bug修复及稳定性提升。 12 | 13 | ## 发布规则 14 | 15 | * 定期从master拉出分支，如0.2，0.3等作为版本迭代，此分支上不再更新新功能。 16 | * 基于分支进行版本发布，如0.2.1，0.2.2等，通过小版本进行稳定性提升。 17 | * 将分支中的build_version.sh中对应版本号更新至待发布版本号、地址更新为版本存档地址。 18 | * 已发布版本不再修改，通过发布日志标注版本之间的关系。 19 | 20 | ## 发布日志更新 21 | 22 | 每一个版本对应自己的发布日志，日志中包含： 23 | 24 | * 是否为稳定版本 25 | * 新增功能列表 26 | * 新修复问题列表 27 | * 用户反馈存在问题列表及升级路径 28 | 29 | 刚刚发布时，将新功能及问题修复记录在日志中。 30 | 31 | 当有用户反馈出此版本问题时，随时记录，待问题修复后，更新日志，标注对应升级的版本。 32 | 33 | 经过用户及时间验证后，可以将对应版本标记为稳定版本，支持大规模升级。 34 | 35 | ## Q&A 36 | 37 | 1. **如何发布稳定版？** 38 | 39 | 稳定版需要用户及时间验证，待某个版本被验证可用性达到较高程度时，将此版本标记为稳定版本。 40 | 41 | 1. **新版本在迭代，旧版本如何保证更新、淘汰？旧版本会不会过多，难以维护？** 42 | 43 | 新版本在迭代，但并不是每一个版本都会有用户使用，采用反馈更新的方式维护旧版本。如果某个版本的用户并没有反馈问题，则此版本会被自然淘汰。 44 | 45 | -------------------------------------------------------------------------------- /doc/roles_statement.md: -------------------------------------------------------------------------------- 1 | # 角色和责任 2 | 3 | * 项目管理委员会（PMC） 4 | * 项目管理委员会成员（PMCer） 5 | * 项目开发委员会（Committers） 6 | * 项目开发委员会成员（Committer） 7 | * 项目贡献者（Contributor） 8 | 9 | ## 项目管理委员会（PMC） 10 | PMC负责对项目的重大发展问题进行决策，指导Committers的工作。 11 | 12 | ### 责任 13 | 1. 指导Committer进行项目的开发管理 14 | 1. 决定PMCer资格、Committer资格 15 | 1. 推选PMC主席 16 | 1. 维护项目章程 17 | 18 | ### 成为PMCer 19 | 1. 由一名PMCer提名，成为候选人 20 | 1. PMC征得候选人同意 21 | 1. PMC投票决定候选人是否成为PMCer 22 | 23 | ## 项目开发委员会（Committers） 24 | Committers负责项目的开发管理。 25 | 26 | ### 责任 27 | 1. 负责评审issue和pull request，与提交者沟通 28 | 1. 负责pull request的拒绝和接受 29 | 1. 负责分支管理、新版本发布 30 | 1. 能够持续投入项目开发 31 | 32 | ### 成为Committer 33 | 1. Contributor连续3个月每月至少有1个pull request被接受，或者1个月内至少有3个pull request被接受，自动成为候选人 34 | 1. PMC征得候选人同意 35 | 1. PMC投票决定候选人是否成为Committer 36 | -------------------------------------------------------------------------------- /doc/sdk_reference/readme.md: -------------------------------------------------------------------------------- 1 | # Tera SDK主要api接口说明 2 | 3 | 4 | ### 主要数据结构 5 | 6 | * tera::[client](../sdk_reference/client.md) 7 | * tera::[table](../sdk_reference/table.md) 8 | * tera::[mutation](../sdk_reference/mutation.md) 9 | * tera::[reader](../sdk_reference/reader.md) 10 | * tera::[table_descriptor](../sdk_reference/table_descriptor.md) 11 | * tera::[transaction](../sdk_reference/transaction.md) 12 | * tera::[scan](../sdk_reference/scan.md) 13 | * tera::[utils](../sdk_reference/utils.md) 14 | 15 | 16 | ### 介绍 17 | #### (1) tera::client 访问tera服务主结构，所有对tera的访问或操作全部由此发起。 18 | 一个集群对应一个client即可，如需访问多个client，需要创建多个 19 | ##### 主要功能包括： 20 | * 表格操作：建、删、加载、卸载、打开、关闭、更新表结构、获取表格信息、快照等 21 | * 用户管理：建、删、修改密码、组管理等 22 | * 集群信息获取：获取全部表格列表、状态等 23 | 24 | #### (2) tera::table 表格主结构，对表格的所有增删查改操作由此发起。 25 | 由tera::Client::OpenTable产生，tera::Client::CloseTable关闭，不可析构。 26 | 27 | #### (3) tera::error_code 错误码，很多操作会返回，注意检查。 28 | 29 | #### (4) tera::mutation 30 | 31 | #### (5) tera::scan 扫描操作，并获取返回数据。 32 | 33 | #### (6) tera::reader 读取操作，并获取返回数据。 34 | 35 | #### (7) tera::table_descriptor 表格描述符主体 36 | 37 | #### (8) tera::transaction 单行事务 38 | 39 | 40 | #### (9) tera::scan 扫描 41 | 42 | #### (10) tera::utils 编码解码 43 | -------------------------------------------------------------------------------- /doc/sdk_reference/transaction.md: -------------------------------------------------------------------------------- 1 | 2 | # 单行事务transaction接口说明 3 | 4 | ## 主要功能 5 | 6 | 7 | ##### (1) 提交一个修改操作 Transaction::ApplyMutation 8 | ``` 9 | void ApplyMutation(RowMutation* row_mu) = 0 10 | ``` 11 | 12 | ##### (2) 读取操作 Transaction::Get 13 | ``` 14 | ErrorCode Get(RowReader* row_reader) = 0 15 | ``` 16 | ##### (3) 回调函数原型 Transaction::Callback 17 | ``` 18 | typedef void (*Callback)(Transaction* transaction) 19 | ``` 20 | 21 | ##### (4) 设置提交回调, 提交操作会异步返回 Transaction::SetCommitCallback 22 | 23 | ``` 24 | void SetCommitCallback(Callback callback) = 0; 25 | ``` 26 | 27 | ##### (5) 获取提交回调 Transaction::GetCommit 28 | 29 | ``` 30 | Callback GetCommitCallback() = 0; 31 | ``` 32 | 33 | ##### (6) 设置用户上下文，可在回调函数中获取 Transaction::SetContext 34 | 35 | ``` 36 | void SetContext(void* context) = 0; 37 | ``` 38 | 39 | ##### (7) 获取用户上下文 Transaction::GetContext 40 | 41 | ``` 42 | void* GetContext() = 0 43 | ``` 44 | 45 | ##### (8) 获得结果错误码 Transaction::GetError 46 | 47 | ``` 48 | const ErrorCode& GetError() = 0; // 异步模式下，通过GetError()获取提交结果 49 | ``` 50 | 51 | ##### (9) 同步模式下，获得提交的结果 Transaction::Commit 52 | ``` 53 | ErrorCode Commit() = 0 // 同步模式下，Commit()的返回值代表了提交操作的结果(成功 或者 失败及其原因) 54 | ``` 55 | 56 | ##### (10) 获取事务开始时间戳 Transaction::GetStartTimestamp 57 | ``` 58 | int64_t GetStartTimestamp() = 0 //仅在全局事务场景下有效 59 | ``` 60 | 61 | -------------------------------------------------------------------------------- /doc/sdk_reference/utils.md: -------------------------------------------------------------------------------- 1 | 2 | # utils接口说明 3 | tera中utils操作主要用来编码和解码counter cell 4 | ##### (1) 编码 5 | ``` 6 | static std::string EncodeCounter(int64_t counter); 7 | ``` 8 | 9 | ##### (2) 解码 10 | 11 | ``` 12 | static bool DecodeCounter(const std::string& buf, int64_t* counter); 13 | ``` 14 | 15 | -------------------------------------------------------------------------------- /doc/se_build_on_tera.md: -------------------------------------------------------------------------------- 1 | # 基于Tera构建搜索引擎 2 | 暂时没时间写，有空的同学可以试试~ 3 | -------------------------------------------------------------------------------- /doc/to_be_a_contributor.md: -------------------------------------------------------------------------------- 1 | # 怎样成为一个贡献者 2 | 这是一个既简单又复杂的问题，Tera需要一个文档引导有兴趣的人加入。 3 | 借鉴linux内核的方式，我设计了5个小任务，顺利完成这5个小任务后，就能成为一个Tera的committer（至少是贡献者）。 4 | 5 | # 第一个任务 6 | 了解tera的数据模型和表格组织，回答以下两个问题： 7 | 8 | 1. Tera中数据可以存在多个版本，建表时通过Schema指定保留策略，这个多版本概念是什么维度的？ 9 | a) 表格维度，每个表保留若干版本 10 | b) 行维度，每个行保留若干版本 11 | c) 单元格维度，每个单元格保留若干版本 12 | 13 | 2. 在Tera中，一个表既可以划分为多个Tablet，也可以划分为多个LocalityGroup，这两者的关系是什么？ 14 | a) Tablet包含LocalityGroup，表格先被划分为若干个Tablet， 每个Tabelt又被划分为若干个LocalityGroup 15 | b) Localitygroup包含Tablet，表格先被划分为若干个LocalityGroup，每个LocalityGroup又被换分为若干个Tablet 16 | c) 两者互不包含 17 | 18 | # 后续的任务 19 | 在你完成第一个任务后，请将解答通过邮件发至tera-user@baidu.com， 20 | 我会将第二个任务通过邮件回复给你，后面任务可能会逐渐加大难度，至少是需要动下手了， 21 | 不过相信我，总共不超过一两个小时，你就能成为一个tera的贡献者了！ 22 | -------------------------------------------------------------------------------- /doc/tools/benchmark.md: -------------------------------------------------------------------------------- 1 | 2 | ## 1. tera_bench 3 | 造数据的工具 4 | ### (1) 用法 5 | ``` 6 | ./tera_bench --compression_ratio=1 --key_seed=1 --value_seed=20 --value_size=1000 --num=200000 --benchmarks=random --key_size=24 --key_step=1 7 | ``` 8 | 9 | ## 2. tera_mark 10 | 读写数据,支持异步读写scan 11 | 12 | ### (1) 用法 13 | ``` 14 | #示例： 15 | ./tera_mark --mode=w --tablename=test --type=async --verify=false --entry_limit=1000 16 | ``` 17 | 18 | ### (2) 参数列表 19 | 20 | 参数名 | 意义 | 有效取值 | 单位 | 默认值 | 其它说明 21 | --- | --- | --- | --- | --- | --- 22 | table | 表名 | - | - | "" | 23 | mode | 模式 | "w"/"r"/"s"/"m" | - | "w" | - 24 | type | 类型 | "sync"/"async" | - | "async" | - 25 | pend_size | 最大pending大小 | - | - | 100 | - 26 | pend_count | 最大pending数 | - | - | 100000 | - 27 | start_key | scan的开始key | - | - | "" | - 28 | end_key | scan的结束key | - | - | "" | - 29 | cf_list | scan的列簇 | - | - | "" | - 30 | print | scan的结果是否需要打印 | true/false | - | false | - 31 | buf_size | scan的buffer_size | >0 | - | 65536 | - 32 | verify | md5 verify(writer&read) | true/false | - | true | - 33 | max_outflow | max_outflow | - | - | -1 | - 34 | max_rate | max_rate | - | - | -1 | - 35 | scan_streaming | enable streaming scan | true/false | - | false | - 36 | batch_count | batch_count(sync) | - | - | 1 | - 37 | entry_limit | writing/reading speed limit | - | - | 0 | - 38 | 39 | -------------------------------------------------------------------------------- /doc/tools/readme.md: -------------------------------------------------------------------------------- 1 | 2 | # Tera 主要工具说明 3 | 4 | ## 主要工具 5 | * 操作tera的工具: [teracli](../tools/teracli.md) 6 | * 集群间数据迁移的dump工具: [terautil](../tools/terautil.md) 7 | * 造数据 & 读写数据的工具: [tera_bench & tera_mark](../tools/benchmark.md) 8 | * 业界通用NoSQL测试的基准测试工具: [YCSB](../tools/ycsb.md) 9 | 10 | 11 | -------------------------------------------------------------------------------- /example/docker/conf: -------------------------------------------------------------------------------- 1 | {"hdfs": 1, "ip": "ip_address1:ip_address2:ip_address3", "tera": 1, "zk": 1, "log_prefix": "your_log_path"} 2 | -------------------------------------------------------------------------------- /example/docker/dockerfile.zk: -------------------------------------------------------------------------------- 1 | FROM debian:jessie 2 | 3 | RUN apt-get update && apt-get install -y openjdk-7-jre-headless wget 4 | RUN wget -q -O - http://apache.mirrors.pair.com/zookeeper/zookeeper-3.4.6/zookeeper-3.4.6.tar.gz | tar -xzf - -C /opt \ 5 | && mv /opt/zookeeper-3.4.6 /opt/zookeeper \ 6 | && cp /opt/zookeeper/conf/zoo_sample.cfg /opt/zookeeper/conf/zoo.cfg \ 7 | && mkdir -p /tmp/zookeeper 8 | 9 | ENV JAVA_HOME /usr/lib/jvm/java-7-openjdk-amd64 10 | 11 | RUN apt-get -y install python 12 | RUN apt-get -y install python-dev 13 | RUN apt-get -y install gcc 14 | RUN apt-get -y install make 15 | RUN cd /opt/zookeeper/src/c && ./configure && make && make install 16 | 17 | ADD workspace /opt/workspace 18 | 19 | RUN wget https://bootstrap.pypa.io/ez_setup.py -O - | python 20 | RUN cd /opt/workspace/argparse-1.3.0/ && python setup.py install 21 | RUN cd /opt/workspace/zkpython-0.4/ && python setup.py install 22 | RUN cp /opt/workspace/zk_setup.py /opt/ 23 | RUN mkdir /opt/share 24 | 25 | ENV LD_LIBRARY_PATH /usr/local/lib/ 26 | 27 | WORKDIR /opt/ 28 | 29 | -------------------------------------------------------------------------------- /example/docker/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # python related 4 | sudo apt-get install -y python python-dev python-pip 5 | sudo pip install argparse 6 | sudo pip install paramiko 7 | 8 | # docker images preparation 9 | echo 'pulling lylei/tera:latest' 10 | docker pull lylei/tera:latest 11 | 12 | echo 'saving lylei/tera:latest to tera.tar' 13 | docker save lylei/tera:latest > ./tera.tar 14 | 15 | while [ $# -gt 0 ]; do 16 | echo $1 17 | scp tera.tar $1:$HOME 18 | echo 'loading lylei/tera:latest' 19 | ssh $1 "cd $HOME && docker load < tera.tar && rm tera.tar" 20 | shift 21 | done 22 | 23 | -------------------------------------------------------------------------------- /example/docker/master: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export CLASSPATH=.:$CLASSPATH:$JAVA_HOME/lib:$JAVA_HOME/jre/lib:$JAVA_HOME/jre/lib/amd64/:/home/ubuntu/leiliyuan/hadoop-1.2.1/lib:$(hadoop classpath) 4 | cd /opt/tera/bin 5 | 6 | nohup ./tera_master --flagfile=../conf/tera.flag --tera_local_addr $1 &> ../../share/log/master.stderr & 7 | -------------------------------------------------------------------------------- /example/docker/tabletnode: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export CLASSPATH=.:$CLASSPATH:$JAVA_HOME/lib:$JAVA_HOME/jre/lib:$JAVA_HOME/jre/lib/amd64/:/home/ubuntu/leiliyuan/hadoop-1.2.1/lib:$(hadoop classpath) 4 | cd /opt/tera/bin 5 | nohup ./tabletserver --flagfile=../conf/tera.flag --tera_local_addr $1 &> ../../share/log/tabletserver.stderr & 6 | -------------------------------------------------------------------------------- /example/onebox/bin/config: -------------------------------------------------------------------------------- 1 | # master 和 tabletnode 绑定的端口，以此开始 2 | PORT=7700 3 | 4 | # 启动的 tabletnode 个数 5 | TABLETNODE_NUM=3 6 | -------------------------------------------------------------------------------- /example/onebox/bin/kill_tera.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | source ./config 3 | 4 | PID=`ps x | grep tera_master | grep $PORT | awk '{print $1}'`; 5 | if [ ${PID}"x" != "x" ]; then 6 | kill -9 $PID; 7 | fi 8 | 9 | for ((i=1; i<=${TABLETNODE_NUM}; i++)); do 10 | PID=`ps x | grep tabletserver | grep $((PORT+i)) | awk '{print $1}'`; 11 | if [ ${PID}"x" != "x" ]; then 12 | kill -9 $PID; 13 | fi 14 | done 15 | -------------------------------------------------------------------------------- /example/onebox/conf/lb.flag: -------------------------------------------------------------------------------- 1 | --tera_lb_server_port=31000 2 | 3 | --log_dir=../log 4 | --logbufsecs=0 5 | --v=10 6 | -------------------------------------------------------------------------------- /example/onebox/conf/tera.flag: -------------------------------------------------------------------------------- 1 | # 指定各种路径，onebox中无需修改 2 | --log_dir=../log 3 | --tera_tabletnode_path_prefix=../data/ 4 | --tera_zk_lib_log_path=../log/zk.log 5 | --logbufsecs=0 6 | --v=6 7 | 8 | # 指定使用本地文件系统 9 | --tera_leveldb_env_type=local 10 | 11 | # 指定使用fake_zk模式, 只能本机访问tera 12 | --tera_coord_type=fake_zk 13 | --tera_master_query_tabletnode_period=1000 14 | --tera_local_addr=127.0.0.1 15 | 16 | # sdk 17 | --tera_sdk_write_timeout=20000 18 | --tera_sdk_read_timeout=20000 19 | --tera_sdk_scan_timeout=30000 20 | 21 | # others 22 | --online_schema_update_enabled=true 23 | --tera_metric_http_server_enable=false 24 | -------------------------------------------------------------------------------- /ft_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | exit 0 4 | -------------------------------------------------------------------------------- /include/observer/scanner_entry.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2017, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #pragma once 6 | 7 | #include "observer/observer.h" 8 | #include "observer/scanner.h" 9 | #include "tera/tera_entry.h" 10 | 11 | namespace tera { 12 | namespace observer { 13 | 14 | class Scanner; 15 | 16 | class ScannerEntry : public TeraEntry { 17 | public: 18 | ScannerEntry(); 19 | virtual ~ScannerEntry(); 20 | 21 | virtual bool StartServer(); 22 | virtual bool Run(); 23 | virtual void ShutdownServer(); 24 | 25 | virtual ErrorCode Observe(); 26 | virtual void SetOptions(const ScannerOptions& options); 27 | Scanner* GetScanner() const; 28 | 29 | private: 30 | ScannerOptions options_; 31 | }; 32 | 33 | } // namespace observer 34 | } // namespace tera 35 | -------------------------------------------------------------------------------- /include/tera.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | // 5 | // See more api documents: 6 | // https://github.com/baidu/tera/blob/master/doc/sdk_dev_guide.md 7 | // or local documents: 8 | // doc/sdk_dev_guide.md 9 | // 10 | 11 | #ifndef TERA_TERA_H_ 12 | #define TERA_TERA_H_ 13 | 14 | #include "tera/client.h" 15 | #include "tera/error_code.h" 16 | #include "tera/mutation.h" 17 | #include "tera/batch_mutation.h" 18 | #include "tera/reader.h" 19 | #include "tera/scan.h" 20 | #include "tera/table.h" 21 | #include "tera/table_descriptor.h" 22 | #include "tera/tera_entry.h" 23 | #include "tera/transaction.h" 24 | #include "tera/utils.h" 25 | #include "tera/filter.h" 26 | #include "tera/value_filter.h" 27 | #include "tera/filter_list.h" 28 | 29 | #include "observer/notification.h" 30 | #include "observer/observer.h" 31 | #include "observer/scanner.h" 32 | #include "observer/scanner_entry.h" 33 | 34 | #endif // TERA_TERA_H_ 35 | -------------------------------------------------------------------------------- /include/tera/hash.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | #pragma once 5 | #include 6 | 7 | #pragma GCC visibility push(default) 8 | 9 | namespace tera { 10 | std::string MurmurHash(const std::string& user_key); 11 | } 12 | -------------------------------------------------------------------------------- /include/tera/tera_entry.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #pragma once 6 | 7 | #include 8 | 9 | #pragma GCC visibility push(default) 10 | namespace tera { 11 | 12 | class TeraEntry { 13 | public: 14 | TeraEntry(); 15 | virtual ~TeraEntry(); 16 | 17 | virtual bool Start(); 18 | virtual bool Run(); 19 | virtual bool Shutdown(); 20 | 21 | protected: 22 | virtual bool StartServer() = 0; 23 | virtual void ShutdownServer() = 0; 24 | 25 | private: 26 | bool ShouldStart(); 27 | bool ShouldShutdown(); 28 | 29 | private: 30 | std::atomic started_; 31 | }; 32 | 33 | } // namespace tera 34 | #pragma GCC visibility pop 35 | -------------------------------------------------------------------------------- /include/tera/utils.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | // 5 | // Some tools are defined in this file. 6 | 7 | #ifndef TERA_UTILS_H_ 8 | #define TERA_UTILS_H_ 9 | 10 | #include 11 | #include 12 | 13 | #pragma GCC visibility push(default) 14 | namespace tera { 15 | 16 | class CounterCoding { 17 | public: 18 | // encode an int64 to a string for a counter cell 19 | static std::string EncodeCounter(int64_t counter); 20 | // decode an int64 from a string for a counter cell 21 | static bool DecodeCounter(const std::string& buf, int64_t* counter); 22 | }; 23 | } // namespace tera 24 | #pragma GCC visibility pop 25 | 26 | #endif // TERA_READER_H_ 27 | -------------------------------------------------------------------------------- /resources/images/arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidu/tera/dbcd28af792d879d961bf9fc7eb60de81b437646/resources/images/arch.png -------------------------------------------------------------------------------- /resources/images/global_txn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidu/tera/dbcd28af792d879d961bf9fc7eb60de81b437646/resources/images/global_txn.png -------------------------------------------------------------------------------- /resources/images/tara_arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidu/tera/dbcd28af792d879d961bf9fc7eb60de81b437646/resources/images/tara_arch.png -------------------------------------------------------------------------------- /resources/images/tera_master_impl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidu/tera/dbcd28af792d879d961bf9fc7eb60de81b437646/resources/images/tera_master_impl.png -------------------------------------------------------------------------------- /resources/images/tera_ts_impl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidu/tera/dbcd28af792d879d961bf9fc7eb60de81b437646/resources/images/tera_ts_impl.png -------------------------------------------------------------------------------- /resources/images/webtable_datamodel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidu/tera/dbcd28af792d879d961bf9fc7eb60de81b437646/resources/images/webtable_datamodel.png -------------------------------------------------------------------------------- /resources/images/webtable_er.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidu/tera/dbcd28af792d879d961bf9fc7eb60de81b437646/resources/images/webtable_er.png -------------------------------------------------------------------------------- /script/README.md: -------------------------------------------------------------------------------- 1 | # 通用操作脚本 2 | 3 | ## manual_rebalance.sh 4 | 5 | 手动负载均衡，将指定key范围的数据分片移至负载最低的服务器上。 6 | 7 | ### 用法 8 | 9 | ``` 10 | sh manual_rebalance.sh 11 | ``` 12 | 13 | 其中key-range-list-file 文件结构为： 14 | * 每一行分为两列，为对应key range的start/end 15 | * 支持多个range同时操作 16 | 17 | ### 配置 18 | 19 | ``` 20 | max_tablet_num=10 # 最大均衡并发数，若超过此值，放弃执行，控制风险 21 | teracli="./teracli" # 对应访问集群的客户端程序 22 | table="hello" # 目标表名 23 | ``` 24 | 25 | ### 其他 26 | 27 | #### 低负载ts选择规则 28 | 29 | * 先按lread指标排序 30 | * 选出其中2 * max_tablet_num个最小的ts做为候选 31 | * 保证读和扫描负载相对低 32 | * 将侯选ts再按cpu消耗排序 33 | * 选出其中最小的max_tablet_num个做为最终目标ts 34 | 35 | #### 输入文件格式 36 | 37 | * 不支持空、含有空格等字符串key，为了保证输入文件格式简单 38 | -------------------------------------------------------------------------------- /so-version-script: -------------------------------------------------------------------------------- 1 | lang_cpp { 2 | global: 3 | extern "C++" { 4 | tera::*; 5 | teraeasy::*; 6 | }; 7 | local: 8 | *; 9 | }; 10 | 11 | lang_c { 12 | global: 13 | extern "C" { 14 | tera_*; 15 | Java_com_baidu_tera_*; 16 | }; 17 | local: 18 | *; 19 | }; 20 | 21 | 22 | -------------------------------------------------------------------------------- /sql/Makefile: -------------------------------------------------------------------------------- 1 | include ../depends.mk 2 | 3 | OUTPUT = $(shell pwd)/build 4 | MYSQL_PATH = mysql-5.6.31 5 | TERA_PLUGIN_PATH = $(MYSQL_PATH)/storage/tera 6 | 7 | .PHONY: all clean cleanall 8 | 9 | all: $(MYSQL_PATH) FORCE 10 | rm -rf $(TERA_PLUGIN_PATH) 11 | cp -a src $(TERA_PLUGIN_PATH) 12 | cp -a ../build/lib/libtera.so $(TERA_PLUGIN_PATH) 13 | cp -a ../build/include/tera.h $(TERA_PLUGIN_PATH) 14 | cd $(MYSQL_PATH) && cmake -DCMAKE_INSTALL_PREFIX=$(OUTPUT) \ 15 | -DCMAKE_BUILD_TYPE=Debug -DMYSQL_MAINTAINER_MODE=OFF \ 16 | -DWITH_TERA_STORAGE_ENGINE=ON 17 | $(MAKE) -C $(MYSQL_PATH) 18 | $(MAKE) install -C $(MYSQL_PATH) 19 | cp -af $(TERA_PLUGIN_PATH)/libtera.so $(OUTPUT)/lib/plugin 20 | cp -af script/* $(OUTPUT)/bin 21 | echo 'Done' 22 | 23 | clean: 24 | $(MAKE) clean -C $(MYSQL_PATH) 25 | 26 | cleanall: 27 | $(MAKE) clean 28 | rm -rf $(OUTPUT) 29 | 30 | .PHONY: FORCE 31 | FORCE: 32 | 33 | -------------------------------------------------------------------------------- /sql/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e -u -E # this script will exit if any sub-command fails 4 | 5 | ######################################## 6 | # download mysql source 7 | ######################################## 8 | 9 | if [ ! -d "mysql-5.6.31" ]; then 10 | wget -O mysql-5.6.31.tar.gz http://dev.mysql.com/get/Downloads/MySQL-5.6/mysql-5.6.31.tar.gz 11 | rm -rf mysql-5.6.31 12 | tar zxf mysql-5.6.31.tar.gz 13 | fi 14 | 15 | ######################################## 16 | # build 17 | ######################################## 18 | 19 | make -j4 20 | -------------------------------------------------------------------------------- /sql/script/init_mysql.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | BIN_DIR=`pwd` 4 | ROOT_DIR=$BIN_DIR/.. 5 | 6 | mkdir -p $ROOT_DIR/log 7 | mkdir -p $ROOT_DIR/run 8 | mkdir -p $ROOT_DIR/var 9 | mkdir -p $ROOT_DIR/tmp 10 | mkdir -p $ROOT_DIR/mysql-files 11 | 12 | ../scripts/mysql_install_db --user=$USER --basedir=$ROOT_DIR --datadir=$ROOT_DIR/data 13 | 14 | -------------------------------------------------------------------------------- /sql/script/start_mysql.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | BIN_DIR=`pwd` 4 | ROOT_DIR=$BIN_DIR/.. 5 | 6 | export LD_LIBRARY_PATH=$ROOT_DIR/lib/plugin:$LD_LIBRARY_PATH 7 | cp -f $ROOT_DIR/lib/plugin/tera.flag /tmp/tera.flag 8 | nohup \ 9 | ./mysqld --port=8806 --user=$USER \ 10 | --datadir=$ROOT_DIR/data \ 11 | --tmpdir=$ROOT_DIR/tmp \ 12 | --socket=$ROOT_DIR/var/mysql.sock \ 13 | --secure-file-priv=$ROOT_DIR/mysql-files \ 14 | --pid-file=$ROOT_DIR/run/mysqld.pid \ 15 | --log-error=$ROOT_DIR/log/mysqld.log \ 16 | --lc-messages-dir=$ROOT_DIR/share \ 17 | --plugin-dir=$ROOT_DIR/lib/plugin \ 18 | --plugin-load="ha_tera.so;libtera.so" \ 19 | --old_passwords=2 \ 20 | --gdb --core-file \ 21 | --debug=d:t:i:o,$ROOT_DIR/log/mysqld.trace \ 22 | &> $ROOT_DIR/log/stderr & 23 | 24 | -------------------------------------------------------------------------------- /sql/src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2006, 2014, Oracle and/or its affiliates. All rights reserved. 2 | # 3 | # This program is free software; you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation; version 2 of the License. 6 | # 7 | # This program is distributed in the hope that it will be useful, 8 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | # GNU General Public License for more details. 11 | # 12 | # You should have received a copy of the GNU General Public License 13 | # along with this program; if not, write to the Free Software 14 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 15 | 16 | ADD_DEFINITIONS(-DMYSQL_SERVER) 17 | SET(CMAKE_CXX_FLAGS "-L. -ltera") 18 | SET(TERA_PLUGIN_DYNAMIC "ha_tera") 19 | SET(TERA_SOURCES ha_tera.cc ha_tera.h ha_tera_util.cc ha_tera_util.h ha_tera_format.cc ha_tera_format.h) 20 | MYSQL_ADD_PLUGIN(tera ${TERA_SOURCES} STORAGE_ENGINE DEFAULT MODULE_ONLY RECOMPILE_FOR_EMBEDDED) 21 | -------------------------------------------------------------------------------- /sql/src/ha_tera_util.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | /* Copyright (c) 2004, 2014, Oracle and/or its affiliates. All rights reserved. 6 | 7 | This program is free software; you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation; version 2 of the License. 10 | 11 | This program is distributed in the hope that it will be useful, 12 | but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | GNU General Public License for more details. 15 | 16 | You should have received a copy of the GNU General Public License 17 | along with this program; if not, write to the Free Software 18 | Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ 19 | 20 | #include 21 | 22 | class ha_tera_util { 23 | public: 24 | static void path_to_dbname(const char *path_name, char *dbname); 25 | static void path_to_tabname(const char *path_name, char *tabname); 26 | static void name_to_tera_tabname(const char* dbname, const char* tabname, char* tera_tabname); 27 | static bool tera_tabname_to_name(const char* tera_tabname, char* dbname, char* tabname); 28 | }; 29 | 30 | std::string escape_string(const std::string& src); 31 | -------------------------------------------------------------------------------- /src/README.md: -------------------------------------------------------------------------------- 1 | tera源码目录结构 2 | ---- 3 | #####common 4 | 公共库的实现。 5 | #####io 6 | Tabletserver的数据存储相关逻辑的实现。 7 | #####leveldb 8 | key-value存储的实现，基于开源系统leveldb。 9 | #####master 10 | Master的实现。 11 | #####proto 12 | Master、Tabletserver和SDK三者之间的通信协议。 13 | #####sample 14 | Tera使用样例。 15 | #####sdk 16 | SDK的实现。 17 | #####tabletnode 18 | Tabletserver的tablet管理相关逻辑的实现。 19 | #####utils 20 | 常用函数的实现，便于上层模块使用。 21 | #####zk 22 | Tera对zookeeper的封装，主要实现选主、存活性检测等逻辑。 23 | #####tera_flags.cc 24 | Tera所有flags的定义及默认值。 25 | #####tera_main.cc tera_entry.cc 26 | Tera的服务端主程序入口。 27 | #####teracli_main.cc 28 | Tera的命令行工具的实现，用于表格创建、schema更新和执行管理命令。 29 | 30 | -------------------------------------------------------------------------------- /src/access/access_entry.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #include "access_entry.h" 6 | 7 | namespace tera { 8 | namespace auth { 9 | 10 | AccessEntry::AccessEntry(const std::string& auth_policy) : auth_policy_(auth_policy) { 11 | if (!AccessUtils::GetAuthPolicyType(auth_policy, &auth_policy_type_)) { 12 | LOG(ERROR) << "Unimplemented auth policy " << auth_policy 13 | << ", AccessEntry construct failed, exit!"; 14 | _Exit(EXIT_FAILURE); 15 | } 16 | access_updater_.reset(new AccessUpdater(auth_policy_type_)); 17 | } 18 | 19 | bool AccessEntry::Verify(const IdentityInfo& identity_info, RoleList* roles) { 20 | if (auth_policy_type_ == AuthPolicyType::kNoneAuthPolicy) { 21 | return true; 22 | } 23 | VerificationPtr verification = access_updater_->GetVerification(); 24 | return verification->Verify(identity_info, roles); 25 | } 26 | 27 | bool AccessEntry::Authorize(const RoleList& roles) { 28 | if (auth_policy_type_ == AuthPolicyType::kNoneAuthPolicy) { 29 | return true; 30 | } 31 | AuthorizationPtr authorization = access_updater_->GetAuthorization(); 32 | return authorization->Authorize(roles); 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/access/authorization/authorization.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #include "access/authorization/authorization.h" 6 | 7 | namespace tera { 8 | namespace auth { 9 | 10 | void Authorization::Update(const std::string& role_name) { 11 | MutexLock l(&mutex_); 12 | roles_.emplace(role_name); 13 | } 14 | 15 | void Authorization::Delete(const std::string& role_name) { 16 | MutexLock l(&mutex_); 17 | auto it = roles_.find(role_name); 18 | if (it != roles_.end()) { 19 | roles_.erase(it); 20 | } 21 | } 22 | 23 | void Authorization::GetAll(std::set* role_list) { 24 | MutexLock l(&mutex_); 25 | for (auto& role : roles_) { 26 | role_list->emplace(role); 27 | } 28 | } 29 | 30 | bool Authorization::Authorize(const std::set& role_list) { 31 | MutexLock l(&mutex_); 32 | for (const auto& role : role_list) { 33 | auto it = roles_.find(role); 34 | if (it != roles_.end()) { 35 | return true; 36 | } 37 | } 38 | return false; 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/access/authorization/authorization.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | #include 10 | #include "common/mutex.h" 11 | 12 | namespace tera { 13 | namespace auth { 14 | 15 | class Authorization { 16 | public: 17 | explicit Authorization() {} 18 | virtual ~Authorization() {} 19 | void Update(const std::string& role_name); 20 | void Delete(const std::string& role_name); 21 | void GetAll(std::set* role_list); 22 | bool Authorize(const std::set& role_list); 23 | 24 | private: 25 | std::set roles_; 26 | mutable Mutex mutex_; 27 | }; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/access/giano/giano_identification.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #include "access/giano/giano_identification.h" 6 | 7 | namespace tera { 8 | namespace auth { 9 | 10 | GianoIdentification::GianoIdentification() { 11 | } 12 | 13 | bool GianoIdentification::Login(IdentityInfo* const identity_info, ErrorCode* const error_code) { 14 | return false; 15 | } 16 | 17 | } // auth 18 | } // tera 19 | -------------------------------------------------------------------------------- /src/access/giano/giano_identification.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #pragma once 6 | 7 | #include "access/identification/identification.h" 8 | 9 | namespace tera { 10 | namespace auth { 11 | 12 | class GianoIdentification : public Identification { 13 | public: 14 | GianoIdentification(); 15 | virtual ~GianoIdentification() {} 16 | virtual bool Login(IdentityInfo* const identity_info, ErrorCode* const error_code) override; 17 | }; 18 | 19 | } // auth 20 | } // tera 21 | -------------------------------------------------------------------------------- /src/access/giano/giano_verification.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #include "access/giano/giano_verification.h" 6 | 7 | namespace tera { 8 | namespace auth { 9 | 10 | GianoVerification::GianoVerification() { 11 | } 12 | 13 | bool GianoVerification::Verify(const IdentityInfo& identity_info, RoleList* roles) { 14 | return false; 15 | } 16 | 17 | } // auth 18 | } // tera 19 | -------------------------------------------------------------------------------- /src/access/giano/giano_verification.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #pragma once 6 | 7 | #include "access/verification/verification.h" 8 | 9 | namespace tera { 10 | namespace auth { 11 | 12 | class GianoVerification : public Verification { 13 | public: 14 | GianoVerification(); 15 | virtual ~GianoVerification() {} 16 | virtual bool Verify(const IdentityInfo& identity_info, RoleList* roles) override; 17 | virtual bool Update(const std::string& user_name, 18 | const VerificationInfoPtr& verification_info_ptr) override { 19 | return false; 20 | } 21 | virtual bool Get(const std::string& user_name, 22 | VerificationInfoPtr* verification_info_ptr) override { 23 | return false; 24 | } 25 | virtual void GetAll(UserVerificationInfoList* user_verification_info) override { 26 | } 27 | virtual bool Delete(const std::string& user_name) override { 28 | return false; 29 | } 30 | }; 31 | 32 | } // auth 33 | } // tera 34 | -------------------------------------------------------------------------------- /src/access/helpers/access_utils.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | #include "access/verification/verification.h" 10 | 11 | namespace tera { 12 | 13 | namespace master { 14 | struct MetaWriteRecord; 15 | } 16 | 17 | namespace auth { 18 | 19 | class AccessEntry; 20 | 21 | static const std::string kInternalGroup("internal_group"); 22 | 23 | class AccessUtils { 24 | public: 25 | static bool GetAuthPolicy(const AuthPolicyType& auth_policy_type, std::string* const auth_policy); 26 | static bool GetAuthPolicyType(const std::string& auth_policy, 27 | AuthPolicyType* const auth_policy_type); 28 | static master::MetaWriteRecord* NewMetaRecord(const std::shared_ptr& access_entry, 29 | const UpdateAuthInfo& update_auth_info); 30 | static std::string GetNameFromMetaKey(const std::string& key); 31 | static VerificationInfoPtr GetVerificationInfoFromMetaValue(const std::string& value); 32 | }; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/access/helpers/permission_builder.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #pragma once 6 | 7 | #include "proto/access_control.pb.h" 8 | #include 9 | 10 | namespace tera { 11 | namespace auth { 12 | 13 | class PermissionBuilder { 14 | public: 15 | static Permission* NewPermission(Permission::Action action); 16 | static Permission* NewPermission(Permission::Action action, const std::string& namespace_name); 17 | static Permission* NewPermission(Permission::Action action, const std::string& namespace_name, 18 | const std::string& table_name); 19 | static Permission* NewPermission(Permission::Action action, const std::string& namespace_name, 20 | const std::string& table_name, const std::string& family, 21 | const std::string& qualifier); 22 | }; 23 | 24 | } // namespace auth 25 | } // namespace tera -------------------------------------------------------------------------------- /src/access/helpers/version_recorder.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #pragma once 6 | 7 | #include 8 | 9 | namespace tera { 10 | namespace auth { 11 | 12 | class VersionRecorder { 13 | public: 14 | explicit VersionRecorder() : version_(0), need_update_(false) {} 15 | ~VersionRecorder() {} 16 | 17 | VersionRecorder(VersionRecorder&) = delete; 18 | VersionRecorder& operator==(const VersionRecorder&) = delete; 19 | 20 | bool NeedUpdate() { return need_update_; } 21 | 22 | void SetNeedUpdate(bool need_update) { need_update_.store(need_update); } 23 | 24 | void IncVersion() { version_.fetch_add(1); } 25 | 26 | void SetVersion(uint64_t version) { version_.store(version); } 27 | 28 | uint64_t GetVersion() { return version_.load(); } 29 | 30 | bool IsSameVersion(uint64_t version) { return version_.load() == version; } 31 | 32 | private: 33 | std::atomic version_; 34 | std::atomic need_update_; 35 | }; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/access/identification/identification.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #pragma once 6 | 7 | #include "tera/error_code.h" 8 | #include "access/helpers/access_utils.h" 9 | 10 | namespace tera { 11 | namespace auth { 12 | 13 | class Identification { 14 | public: 15 | // Login to get the cred/password 16 | // Success will return true 17 | // otherwise return false 18 | virtual bool Login(IdentityInfo* const identity_info, ErrorCode* const error_code) = 0; 19 | }; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/access/identification/ugi_identification.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #include "access/identification/ugi_identification.h" 6 | #include 7 | 8 | namespace tera { 9 | namespace auth { 10 | 11 | UgiIdentification::UgiIdentification() {} 12 | 13 | bool UgiIdentification::Login(IdentityInfo* const identity_info, ErrorCode* const error_code) { 14 | if (identity_info->name() == "" || identity_info->token() == "") { 15 | LOG(ERROR) << "Make sure set --tera_auth_name=xxx & --tera_auth_token=yyy " 16 | << "in tera.flag when use kUgiAuthPolicy, [name = " << identity_info->name() 17 | << ", token = " << identity_info->token(); 18 | if (error_code != nullptr) { 19 | std::string reason = 20 | "Login argument absent(--tera_auth_name=xxx && --tera_auth_token=yyy)\ 21 | when use kUgiAuthPolicy"; 22 | error_code->SetFailed(ErrorCode::kAuthBadParam, reason); 23 | } 24 | return false; 25 | } 26 | return true; 27 | } 28 | } 29 | } -------------------------------------------------------------------------------- /src/access/identification/ugi_identification.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #pragma once 6 | 7 | #include "access/identification/identification.h" 8 | 9 | namespace tera { 10 | namespace auth { 11 | class UgiIdentification : public Identification { 12 | public: 13 | UgiIdentification(); 14 | virtual ~UgiIdentification() {} 15 | virtual bool Login(IdentityInfo* const identity_info, ErrorCode* const error_code) override; 16 | }; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/access/test/multi_tenancy_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #include 6 | #include 7 | #include 8 | 9 | DECLARE_bool(tera_quota_enabled); 10 | DECLARE_string(tera_quota_limiter_type); 11 | DECLARE_int64(tera_quota_normal_estimate_value); 12 | 13 | int main(int argc, char** argv) { 14 | ::google::ParseCommandLineFlags(&argc, &argv, true); 15 | FLAGS_tera_quota_enabled = true; 16 | FLAGS_tera_quota_limiter_type = "general_quota_limiter"; 17 | FLAGS_tera_quota_normal_estimate_value = 1; 18 | ::testing::InitGoogleTest(&argc, argv); 19 | return RUN_ALL_TESTS(); 20 | } 21 | -------------------------------------------------------------------------------- /src/access/verification/ugi_verification.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | #include "access/verification/verification.h" 10 | #include "common/mutex.h" 11 | 12 | namespace tera { 13 | 14 | namespace auth { 15 | 16 | class UgiVerification : public Verification { 17 | public: 18 | UgiVerification(); 19 | virtual ~UgiVerification() {} 20 | virtual bool Verify(const IdentityInfo& identity_info, RoleList* roles) override; 21 | 22 | virtual bool Update(const std::string& user_name, 23 | const VerificationInfoPtr& verification_info) override; 24 | virtual bool Get(const std::string& user_name, VerificationInfoPtr* verification_info) override; 25 | virtual void GetAll(UserVerificationInfoList* user_verification_info) override; 26 | virtual bool Delete(const std::string& user_name) override; 27 | 28 | private: 29 | UserVerificationInfoList user_verification_info_list_; 30 | mutable Mutex ugi_mutex_; 31 | }; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/access/verification/verification.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include "proto/access_control.pb.h" 13 | 14 | namespace tera { 15 | namespace auth { 16 | 17 | using RoleList = std::set; 18 | using VerificationInfo = std::pair; 19 | using VerificationInfoPtr = std::shared_ptr; 20 | using UserVerificationInfoList = std::map; 21 | 22 | class Verification { 23 | public: 24 | // Verify ideantity 25 | // Return false means to fake ideantity 26 | virtual bool Verify(const IdentityInfo& identity_info, RoleList* roles) = 0; 27 | 28 | // Update verification infos 29 | virtual bool Update(const std::string& user_name, 30 | const VerificationInfoPtr& verification_info_ptr) = 0; 31 | virtual bool Get(const std::string& user_name, VerificationInfoPtr* verification_info_ptr) = 0; 32 | virtual void GetAll(UserVerificationInfoList* user_verification_info) = 0; 33 | 34 | // Delete verification's user 35 | virtual bool Delete(const std::string& user_name) = 0; 36 | }; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/benchmark/eva/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidu/tera/dbcd28af792d879d961bf9fc7eb60de81b437646/src/benchmark/eva/__init__.py -------------------------------------------------------------------------------- /src/benchmark/eva/conf.sample: -------------------------------------------------------------------------------- 1 | { 2 | "table_name": "test", 3 | "read_speed_limit(Qps)": "0", 4 | "value_sizeB(B)": "102400", 5 | "mode": "rw", 6 | "key_seed": "10", 7 | "value_seed": "10", 8 | "step": "False", 9 | "scan_buffer(M)": "0", 10 | "tablet_number": "1", 11 | "ts_number": "1", 12 | "key_size(B)": "15", 13 | "write_speed_limit(M)": "3", 14 | "entry_number(M)": "0.1", 15 | "split_size": 1024000, 16 | "table_schema": {"lg0": {"storage": "disk", "cf": "cf0:q"}} 17 | } 18 | -------------------------------------------------------------------------------- /src/benchmark/eva/helper.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | # Use of this source code is governed by a BSD-style license that can be 3 | # found in the LICENSE file. 4 | 5 | #!/usr/bin/env pythons 6 | 7 | from bin import run 8 | 9 | 10 | def main(): 11 | run.parse_input() 12 | 13 | 14 | if __name__ == '__main__': 15 | main() -------------------------------------------------------------------------------- /src/benchmark/eva/run_sample.sh: -------------------------------------------------------------------------------- 1 | set -x 2 | 3 | sh some_script_to_start_tera_cluster 4 | 5 | python run.py my_conf_file 6 | 7 | sh some_script_to_send_report --report_file ../tmp/mail_report 8 | -------------------------------------------------------------------------------- /src/benchmark/eva/setup.sh: -------------------------------------------------------------------------------- 1 | set -x 2 | 3 | mkdir bin 4 | mkdir -p log/eva 5 | mkdir tmp 6 | mkdir conf 7 | 8 | mv *.py bin/ 9 | mv run_sample.sh bin/ 10 | -------------------------------------------------------------------------------- /src/benchmark/tpcc/mock_tpccdb.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2017, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | // 5 | // Author: baorenyi@baidu.com 6 | 7 | #include "benchmark/tpcc/mock_tpccdb.h" 8 | 9 | #include 10 | #include 11 | 12 | namespace tera { 13 | namespace tpcc { 14 | 15 | MockTpccDb::MockTpccDb() : flag_(true) {} 16 | 17 | } // namespace tpcc 18 | } // namespace tera 19 | -------------------------------------------------------------------------------- /src/benchmark/tpcc/test/tpcc_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2017, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | // 5 | // Author: baorenyi@baidu.com 6 | 7 | #include "gflags/gflags.h" 8 | #include "glog/logging.h" 9 | #include "gtest/gtest.h" 10 | 11 | namespace tera { 12 | namespace tpcc { 13 | 14 | int main(int argc, char* argv[]) { 15 | ::testing::InitGoogleTest(&argc, argv); 16 | return RUN_ALL_TESTS(); 17 | } 18 | 19 | } // namespace tpcc 20 | } // namespace tera 21 | -------------------------------------------------------------------------------- /src/benchmark/tpcc/tpcc_flags.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2017, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | // 5 | // Author: baorenyi@baidu.com 6 | 7 | #include "gflags/gflags.h" 8 | 9 | DEFINE_int64(transactions_count, 200, "the count of transactions"); 10 | DEFINE_int32(warehouses_count, 2, "the count of warsehouses"); 11 | DEFINE_int32(tpcc_thread_pool_size, 20, "size of tpcc thread pool"); 12 | DEFINE_int32(tpcc_run_gtxn_thread_pool_size, 20, 13 | "size of tpcc run global transactions thread pool"); 14 | DEFINE_string(db_type, "tera", "test db type"); 15 | DEFINE_string(tera_client_flagfile, "./tera.flag", "the flag file path of tera client"); 16 | DEFINE_string(tera_table_schema_dir, "./tpcc_schemas/", "table schema directory"); 17 | DEFINE_int32(generate_data_wait_times, 3600000, "(ms) generate data wait times, default 1h"); 18 | DEFINE_int32(driver_wait_times, 3600000, "(ms) driver wait times, default 1h"); 19 | -------------------------------------------------------------------------------- /src/benchmark/tpcc/tpcc_schemas/t_customer: -------------------------------------------------------------------------------- 1 | t_customer { 2 | lg0 { 3 | cf0 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /src/benchmark/tpcc/tpcc_schemas/t_customer_last_index: -------------------------------------------------------------------------------- 1 | t_customer_last_index { 2 | lg0 { 3 | cf0 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /src/benchmark/tpcc/tpcc_schemas/t_district: -------------------------------------------------------------------------------- 1 | t_district { 2 | lg0 { 3 | cf0 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /src/benchmark/tpcc/tpcc_schemas/t_history: -------------------------------------------------------------------------------- 1 | t_history { 2 | lg0 { 3 | cf0 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /src/benchmark/tpcc/tpcc_schemas/t_history_index: -------------------------------------------------------------------------------- 1 | t_history_index { 2 | lg0 { 3 | cf0 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /src/benchmark/tpcc/tpcc_schemas/t_item: -------------------------------------------------------------------------------- 1 | t_item { 2 | lg0 { 3 | cf0 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /src/benchmark/tpcc/tpcc_schemas/t_neworder: -------------------------------------------------------------------------------- 1 | t_neworder { 2 | lg0 { 3 | cf0 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /src/benchmark/tpcc/tpcc_schemas/t_order: -------------------------------------------------------------------------------- 1 | t_order { 2 | lg0 { 3 | cf0 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /src/benchmark/tpcc/tpcc_schemas/t_order_index: -------------------------------------------------------------------------------- 1 | t_order_index { 2 | lg0 { 3 | cf0 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /src/benchmark/tpcc/tpcc_schemas/t_orderline: -------------------------------------------------------------------------------- 1 | t_orderline { 2 | lg0 { 3 | cf0 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /src/benchmark/tpcc/tpcc_schemas/t_stock: -------------------------------------------------------------------------------- 1 | t_stock { 2 | lg0 { 3 | cf0 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /src/benchmark/tpcc/tpcc_schemas/t_warehouse: -------------------------------------------------------------------------------- 1 | t_warehouse { 2 | lg0 { 3 | cf0 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /src/common/base/static_assert.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef COMMON_BASE_STATIC_ASSERT_H 6 | #define COMMON_BASE_STATIC_ASSERT_H 7 | 8 | #if __GXX_EXPERIMENTAL_CXX0X__ || defined __MSC_VER && __MSC_VER >= 1600 9 | #define STATIC_ASSERT(e, ...) static_assert(e, "" __VA_ARGS__) 10 | #else 11 | 12 | #include "common/base/preprocess.h" 13 | 14 | // namespace common { 15 | 16 | template 17 | struct static_assertion_failure; 18 | 19 | template <> 20 | struct static_assertion_failure { 21 | enum { value = 1 }; 22 | }; 23 | 24 | template 25 | struct static_assert_test {}; 26 | 27 | /// 编译期间的静态断言 28 | /// @param e 常量表达式 29 | /// 用于编译期间检查错误的场合，示例： 30 | /// @code 31 | /// STATIC_ASSERT(sizeof(Foo) == 48, "Foo 的大小必须为 48"); 32 | /// @endcode 33 | #define STATIC_ASSERT(e, ...) \ 34 | typedef static_assert_test)> PP_JOIN( \ 35 | static_assert_failed, __LINE__) 36 | 37 | #endif 38 | 39 | // } // namespace common 40 | 41 | #endif // COMMON_BASE_STATIC_ASSERT_H 42 | -------------------------------------------------------------------------------- /src/common/base/string_ext.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_COMMON_STRING_EXT_H_ 6 | #define TERA_COMMON_STRING_EXT_H_ 7 | 8 | #include 9 | #include 10 | 11 | void SplitString(const std::string& full, const std::string& delim, 12 | std::vector* result); 13 | 14 | void SplitStringEnd(const std::string& full, std::string* begin_part, std::string* end_part, 15 | std::string delim = "."); 16 | 17 | std::string ReplaceString(const std::string& str, const std::string& src, const std::string& dest); 18 | 19 | std::string TrimString(const std::string& str, const std::string& trim = " "); 20 | 21 | bool StringEndsWith(const std::string& str, const std::string& sub_str); 22 | 23 | bool StringStartWith(const std::string& str, const std::string& sub_str); 24 | 25 | char* StringAsArray(std::string* str); 26 | 27 | #endif // TERA_COMMON_STRING_EXT_H_ 28 | -------------------------------------------------------------------------------- /src/common/base/string_format.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_COMMON_BASE_STRING_FORMAT_H_ 6 | #define TERA_COMMON_BASE_STRING_FORMAT_H_ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | size_t StringFormatAppendVA(std::string* dst, const char* format, va_list ap); 14 | 15 | size_t StringFormatAppend(std::string* dst, const char* format, ...); 16 | 17 | size_t StringFormatTo(std::string* dst, const char* format, ...); 18 | 19 | std::string StringFormat(const char* format, ...); 20 | 21 | #endif // TERA_COMMON_BASE_STRING_FORMAT_H_ 22 | -------------------------------------------------------------------------------- /src/common/base/test/string_ext_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #include "bvs/common/base/string_ext.h" 6 | 7 | int main(int argc, char* argv[]) { 8 | cout << strtool::trim(" nihao ") << "\n"; 9 | 10 | vector vt; 11 | strtool::split(",o h,,,nice,,,,,,,", vt); 12 | for (size_t i = 0; i < vt.size(); ++i) { 13 | cout << "out:" << vt[i] << "\n"; 14 | } 15 | 16 | string ret = strtool::replace("xxAxxxAxxAxx", "A", "B"); 17 | cout << "replace:" << ret << "\n"; 18 | return 0; 19 | } 20 | -------------------------------------------------------------------------------- /src/common/file/file_flags.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #include 6 | 7 | DEFINE_int32(file_op_retry_times, 3, "the max retry times when file operation occurred error"); 8 | -------------------------------------------------------------------------------- /src/common/file/file_stream.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_COMMON_FILE_FILE_STREAM_H_ 6 | #define TERA_COMMON_FILE_FILE_STREAM_H_ 7 | 8 | #include 9 | #include 10 | 11 | #include "common/file/file_types.h" 12 | 13 | class FileStream { 14 | public: 15 | FileStream(); 16 | ~FileStream() {} 17 | 18 | bool Open(const std::string& file_path, FileOpenMode flag, FileErrorCode* error_code = NULL); 19 | bool Close(FileErrorCode* error_code = NULL); 20 | 21 | int64_t Write(const void* buffer, int64_t buffer_size, FileErrorCode* error_code = NULL); 22 | 23 | int64_t Read(void* buffer, int64_t buffer_size, FileErrorCode* error_code = NULL); 24 | 25 | bool Flush(); 26 | 27 | int64_t Seek(int64_t offset, int32_t origin, FileErrorCode* error_code = NULL); 28 | 29 | int64_t Tell(FileErrorCode* error_code = NULL); 30 | 31 | int64_t GetSize(const std::string& file_path, FileErrorCode* error_code = NULL); 32 | 33 | int32_t ReadLine(void* buffer, int32_t max_size); 34 | int32_t ReadLine(std::string* result); 35 | 36 | private: 37 | void SetErrorCode(FileErrorCode* error_code, FileErrorCode code); 38 | std::string FileOpenModeToString(uint32_t flag); 39 | 40 | private: 41 | FILE* fp_; 42 | }; 43 | 44 | #endif // TERA_COMMON_FILE_FILE_STREAM_H_ 45 | -------------------------------------------------------------------------------- /src/common/file/file_types.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_COMMON_FILE_FILE_DEF_H_ 6 | #define TERA_COMMON_FILE_FILE_DEF_H 7 | 8 | #include 9 | 10 | enum FileOpenMode { FILE_READ = 0x01, FILE_WRITE = 0x02, FILE_APPEND = 0x04 }; 11 | 12 | enum FileErrorCode { 13 | kFileSuccess, 14 | kFileErrParameter, 15 | kFileErrOpenFail, 16 | kFileErrNotOpen, 17 | kFileErrWrite, 18 | kFileErrRead, 19 | kFileErrClose, 20 | kFileErrNotExit 21 | }; 22 | 23 | #endif // TERA_COMMON_FILE_FILE_DEF_H_ 24 | -------------------------------------------------------------------------------- /src/common/func_scope_guard.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #pragma once 6 | 7 | #include 8 | 9 | namespace common { 10 | 11 | class FuncScopeGuard { 12 | public: 13 | explicit FuncScopeGuard(std::function on_exit_scope) 14 | : on_exit_scope_(on_exit_scope), dismissed_(false) {} 15 | 16 | FuncScopeGuard(FuncScopeGuard const&) = delete; 17 | FuncScopeGuard& operator=(const FuncScopeGuard&) = delete; 18 | 19 | virtual ~FuncScopeGuard() { 20 | if (!dismissed_) { 21 | on_exit_scope_(); 22 | } 23 | } 24 | 25 | void Dismiss() { dismissed_ = true; } 26 | 27 | private: 28 | std::function on_exit_scope_; 29 | bool dismissed_; 30 | }; 31 | } 32 | 33 | using common::FuncScopeGuard; 34 | -------------------------------------------------------------------------------- /src/common/metric/collector.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | // Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved 3 | // Use of this source code is governed by a BSD-style license that can be 4 | // found in the LICENSE file. 5 | #include 6 | #include 7 | 8 | namespace tera { 9 | class Collector { 10 | public: 11 | virtual ~Collector() {} 12 | // return a instant value of the metric for tera to dump log and other usage 13 | virtual int64_t Collect() = 0; 14 | }; 15 | } 16 | -------------------------------------------------------------------------------- /src/common/metric/counter_collector.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_COMMON_METRIC_COUNTER_COLLECTOR_H_ 6 | #define TERA_COMMON_METRIC_COUNTER_COLLECTOR_H_ 7 | 8 | #include "common/metric/collector.h" 9 | #include "common/counter.h" 10 | 11 | namespace tera { 12 | 13 | class CounterCollector : public Collector { 14 | public: 15 | /// if is_periodic is true, the counter will be cleared when collect 16 | /// this parameter is usually true, but it's false with some instantaneous 17 | /// value 18 | /// Eg: read_pending_count, scan_pending_count, which can't be clear during 19 | /// collect. 20 | explicit CounterCollector(Counter* counter, bool is_periodic = true) 21 | : counter_(counter), is_periodic_(is_periodic) {} 22 | 23 | ~CounterCollector() override {} 24 | 25 | int64_t Collect() override { 26 | if (counter_ == NULL) { 27 | return -1; 28 | } else { 29 | return is_periodic_ ? counter_->Clear() : counter_->Get(); 30 | } 31 | } 32 | 33 | private: 34 | Counter* const counter_; 35 | const bool is_periodic_; 36 | }; 37 | } // end namespace tera 38 | 39 | #endif // TERA_COMMON_METRIC_COUNTER_COLLECTOR_H_ 40 | 41 | /* vim: set expandtab ts=4 sw=4 sts=4 tw=100: */ -------------------------------------------------------------------------------- /src/common/metric/ratio_collector.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_COMMOM_METRIC_RATIO_COLLECTOR_H_ 6 | #define TERA_COMMOM_METRIC_RATIO_COLLECTOR_H_ 7 | 8 | #include 9 | #include "common/metric/collector_report_publisher.h" 10 | 11 | namespace tera { 12 | 13 | class RatioCollector : public Collector { 14 | public: 15 | explicit RatioCollector(Counter* first_counter, Counter* second_counter, bool is_periodic = true) 16 | : first_counter_(first_counter), second_counter_(second_counter), is_periodic_(is_periodic) {} 17 | 18 | int64_t Collect() override { 19 | if (NULL == first_counter_ || NULL == second_counter_) { 20 | return 0; 21 | } else { 22 | double ratio = (double)first_counter_->Get() / second_counter_->Get(); 23 | if (is_periodic_) { 24 | first_counter_->Clear(); 25 | second_counter_->Clear(); 26 | } 27 | return isnan(ratio) ? -1 : static_cast(ratio * 100); 28 | } 29 | } 30 | 31 | private: 32 | Counter* const first_counter_; 33 | Counter* const second_counter_; 34 | const bool is_periodic_; 35 | }; 36 | 37 | } // end namespace tera 38 | 39 | #endif // TERA_COMMOM_METRIC_RATIO_COLLECTOR_H_ 40 | 41 | /* vim: set expandtab ts=4 sw=4 sts=4 tw=100: */ 42 | -------------------------------------------------------------------------------- /src/common/net/ip_address.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_COMMON_NET_IP_ADDRESS_H_ 6 | #define TERA_COMMON_NET_IP_ADDRESS_H_ 7 | 8 | #include 9 | 10 | #include 11 | 12 | class IpAddress { 13 | public: 14 | IpAddress(); 15 | IpAddress(const std::string& ip_port); 16 | IpAddress(const std::string& ip, const std::string& port); 17 | IpAddress(const std::string& ip, uint16_t port); 18 | 19 | ~IpAddress() {} 20 | 21 | std::string ToString() const; 22 | std::string GetIp() const; 23 | uint16_t GetPort() const; 24 | std::string GetPortString() const; 25 | 26 | bool IsValid() const { return valid_address_; } 27 | 28 | bool Assign(const std::string& ip_port); 29 | bool Assign(const std::string& ip, const std::string& port); 30 | bool Assign(const std::string& ip, uint16_t port); 31 | 32 | private: 33 | std::string ip_; 34 | uint16_t port_; 35 | 36 | bool valid_address_; 37 | }; 38 | 39 | #endif // TERA_COMMON_NET_IP_ADDRESS_H_ 40 | -------------------------------------------------------------------------------- /src/common/request_done_wrapper.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | namespace tera { 5 | class RequestDoneWrapper : public google::protobuf::Closure { 6 | public: 7 | static google::protobuf::Closure* NewInstance(google::protobuf::Closure* done) { 8 | return new RequestDoneWrapper(done); 9 | } 10 | 11 | // Self-Deleted, never access it after Run(); 12 | // Default do nothing; 13 | virtual void Run() override { delete this; } 14 | 15 | virtual ~RequestDoneWrapper() { done_->Run(); } 16 | 17 | protected: 18 | // Can Only Create on Heap; 19 | RequestDoneWrapper(google::protobuf::Closure* done) : done_(done) {} 20 | 21 | private: 22 | google::protobuf::Closure* done_; 23 | }; 24 | } -------------------------------------------------------------------------------- /src/common/semaphore.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | #pragma once 5 | 6 | #include "mutex.h" 7 | 8 | namespace common { 9 | 10 | class Semaphore { 11 | public: 12 | Semaphore(const Semaphore&) = delete; 13 | Semaphore& operator=(const Semaphore&) = delete; 14 | Semaphore(Semaphore&&) = delete; 15 | Semaphore& operator=(Semaphore&&) = delete; 16 | 17 | explicit Semaphore(int64_t counter) : cv_(&mutex_), counter_(counter) {} 18 | ~Semaphore() {} 19 | 20 | bool TryAcquire() { 21 | MutexLock lock(&mutex_); 22 | if (counter_ <= 0) { 23 | return false; 24 | } 25 | --counter_; 26 | return true; 27 | } 28 | 29 | void Acquire() { 30 | MutexLock lock(&mutex_); 31 | while (counter_ <= 0) { 32 | cv_.Wait(); 33 | } 34 | --counter_; 35 | } 36 | void Release() { 37 | MutexLock lock(&mutex_); 38 | ++counter_; 39 | cv_.Signal(); 40 | } 41 | 42 | private: 43 | Mutex mutex_; 44 | CondVar cv_; 45 | int64_t counter_; 46 | }; 47 | 48 | } // namespace common 49 | -------------------------------------------------------------------------------- /src/common/tera_entry.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #include "tera/tera_entry.h" 6 | #include "common/this_thread.h" 7 | 8 | namespace tera { 9 | 10 | TeraEntry::TeraEntry() : started_(false) {} 11 | 12 | TeraEntry::~TeraEntry() {} 13 | 14 | bool TeraEntry::Start() { 15 | if (ShouldStart()) { 16 | return StartServer(); 17 | } 18 | return false; 19 | } 20 | 21 | bool TeraEntry::Run() { 22 | ThisThread::Sleep(2000); 23 | return true; 24 | } 25 | 26 | bool TeraEntry::Shutdown() { 27 | if (ShouldShutdown()) { 28 | ShutdownServer(); 29 | return true; 30 | } 31 | return false; 32 | } 33 | 34 | bool TeraEntry::ShouldStart() { 35 | bool has_started = false; 36 | return started_.compare_exchange_strong(has_started, true); 37 | } 38 | 39 | bool TeraEntry::ShouldShutdown() { 40 | bool has_shutdown = true; 41 | return started_.compare_exchange_strong(has_shutdown, false); 42 | } 43 | 44 | } // namespace tera 45 | -------------------------------------------------------------------------------- /src/common/test/bounded_queue_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #include 6 | #include "gtest/gtest.h" 7 | #include "common/base/bounded_queue.h" 8 | 9 | namespace tera { 10 | 11 | class BoundedQueueTest : public ::testing::Test { 12 | public: 13 | BoundedQueueTest() {} 14 | 15 | virtual void SetUp() {} 16 | 17 | virtual void TearDown() {} 18 | 19 | virtual void Reset(int64_t limit) { bq_.reset(new common::BoundedQueue{limit}); } 20 | 21 | private: 22 | std::unique_ptr> bq_; 23 | }; 24 | 25 | TEST_F(BoundedQueueTest, BaseTest) { 26 | Reset(5); 27 | EXPECT_TRUE(bq_.get()); 28 | EXPECT_EQ(bq_->Average(), 0); 29 | EXPECT_EQ(bq_->Sum(), 0); 30 | bq_->Push(1); 31 | bq_->Push(2); 32 | bq_->Push(3); 33 | bq_->Push(4); 34 | bq_->Push(5); 35 | EXPECT_EQ(bq_->Average(), 3); 36 | EXPECT_EQ(bq_->Sum(), 15); 37 | bq_->Push(2); 38 | bq_->Push(2); 39 | bq_->Push(2); 40 | bq_->Push(2); 41 | bq_->Push(2); 42 | EXPECT_EQ(bq_->Average(), 2); 43 | EXPECT_EQ(bq_->Sum(), 10); 44 | EXPECT_EQ(bq_->qu_.size(), bq_->Size()); 45 | EXPECT_EQ(bq_->qu_.size(), 5); 46 | } 47 | } // end namespace tera 48 | 49 | /* vim: set expandtab ts=4 sw=4 sts=4 tw=100: */ 50 | -------------------------------------------------------------------------------- /src/common/test/common_test_main.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #include 6 | 7 | #include "gflags/gflags.h" 8 | #include "glog/logging.h" 9 | #include "gtest/gtest.h" 10 | 11 | #include "utils/utils_cmd.h" 12 | 13 | int main(int argc, char** argv) { 14 | ::google::InitGoogleLogging(argv[0]); 15 | FLAGS_v = 16; 16 | FLAGS_minloglevel = 0; 17 | FLAGS_log_dir = "./log"; 18 | if (access(FLAGS_log_dir.c_str(), F_OK)) { 19 | mkdir(FLAGS_log_dir.c_str(), 0777); 20 | } 21 | std::string pragram_name("tera"); 22 | tera::utils::SetupLog(pragram_name); 23 | ::google::ParseCommandLineFlags(&argc, &argv, true); 24 | ::testing::InitGoogleTest(&argc, argv); 25 | 26 | return RUN_ALL_TESTS(); 27 | } 28 | 29 | /* vim: set expandtab ts=4 sw=4 sts=4 tw=100: */ 30 | -------------------------------------------------------------------------------- /src/common/test/progress_bar_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | // 5 | // Author: xupeilin@baidu.com 6 | 7 | #include "common/console/progress_bar.h" 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | using common::ProgressBar; 14 | 15 | TEST(ProgressBarTest, Test) { 16 | int cur_size = 0; 17 | int total_size = 100000000; 18 | 19 | ProgressBar progress_bar(ProgressBar::ENHANCED, total_size, 100, "B"); 20 | 21 | srand((uint32_t)time(NULL)); 22 | timespec interval = {0, 1000}; 23 | while (cur_size < total_size) { 24 | cur_size += rand() % 10000; 25 | progress_bar.Refresh(cur_size); 26 | nanosleep(&interval, &interval); 27 | } 28 | progress_bar.Done(); 29 | } 30 | -------------------------------------------------------------------------------- /src/common/this_thread.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | // 5 | // Author: yanshiguang02@baidu.com 6 | 7 | #ifndef TERA_COMMON_THIS_THREAD_H_ 8 | #define TERA_COMMON_THIS_THREAD_H_ 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | namespace common { 18 | 19 | class ThisThread { 20 | public: 21 | /// Sleep in ms 22 | static void Sleep(int64_t time_ms) { 23 | if (time_ms > 0) { 24 | timespec ts = {time_ms / 1000, (time_ms % 1000) * 1000000}; 25 | nanosleep(&ts, &ts); 26 | } 27 | } 28 | /// Get thread id 29 | static int GetId() { return syscall(__NR_gettid); } 30 | 31 | /// Yield cpu 32 | static void Yield() { sched_yield(); } 33 | 34 | /// Thread-safe random generator 35 | template 36 | static T GetRandomValue(T min, T max) { 37 | static thread_local std::random_device rd; 38 | static thread_local std::mt19937 gen(rd()); 39 | std::uniform_int_distribution dist(min, max); 40 | return dist(gen); 41 | } 42 | }; 43 | 44 | } // namespace common 45 | 46 | using common::ThisThread; 47 | 48 | #endif // TERA_COMMON_THIS_THREAD_H_ 49 | -------------------------------------------------------------------------------- /src/io/atomic_merge_strategy.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_IO_ATOMIC_MERGE_STRATEGY_H_ 6 | #define TERA_IO_ATOMIC_MERGE_STRATEGY_H_ 7 | 8 | #include "leveldb/raw_key_operator.h" 9 | #include "leveldb/slice.h" 10 | 11 | namespace tera { 12 | namespace io { 13 | 14 | bool IsAtomicOP(leveldb::TeraKeyType keyType); 15 | 16 | class AtomicMergeStrategy { 17 | public: 18 | AtomicMergeStrategy(); 19 | 20 | void Init(std::string* merged_key, std::string* merged_value, const leveldb::Slice& latest_key, 21 | const leveldb::Slice& latest_value, leveldb::TeraKeyType latest_key_type); 22 | 23 | void MergeStep(const leveldb::Slice& key, const leveldb::Slice& value, 24 | leveldb::TeraKeyType key_type); 25 | 26 | bool Finish(); 27 | 28 | private: 29 | std::string* merged_key_; 30 | std::string* merged_value_; 31 | leveldb::TeraKeyType latest_key_type_; 32 | int64_t counter_; // for ADD 33 | int64_t int64_; // for int64(add) 34 | std::string append_buffer_; // for Append 35 | }; 36 | 37 | } // namespace io 38 | } // namespace tera 39 | 40 | #endif // TERA_IO_ATOMIC_MERGE_STRATEGY_H_ 41 | -------------------------------------------------------------------------------- /src/io/io_utils.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_IO_IO_UTILS_H_ 6 | #define TERA_IO_IO_UTILS_H_ 7 | 8 | #include 9 | 10 | #include "common/semaphore.h" 11 | #include "common/rwmutex.h" 12 | #include "leveldb/raw_key_operator.h" 13 | #include "leveldb/status.h" 14 | #include "io/tablet_io.h" 15 | #include "proto/status_code.pb.h" 16 | #include "proto/table_schema.pb.h" 17 | 18 | namespace tera { 19 | 20 | StatusCode LeveldbCodeToTeraCode(const leveldb::Status& status); 21 | 22 | void SetStatusCode(const leveldb::Status& db_status, StatusCode* tera_status); 23 | 24 | void SetStatusCode(const io::TabletIO::TabletStatus& tablet_status, StatusCode* tera_status); 25 | 26 | const leveldb::RawKeyOperator* GetRawKeyOperatorFromSchema(TableSchema& schema); 27 | } // namespace tera 28 | 29 | #endif // TERA_IO_IO_UTILS_H_ 30 | -------------------------------------------------------------------------------- /src/io/timekey_comparator.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_IO_TIMEKEY_COMARATOR_H_ 6 | #define TERA_IO_TIMEKEY_COMARATOR_H_ 7 | 8 | #include "leveldb/comparator.h" 9 | #include "leveldb/slice.h" 10 | 11 | namespace tera { 12 | namespace io { 13 | 14 | class TimekeyComparator : public leveldb::Comparator { 15 | public: 16 | TimekeyComparator(const leveldb::Comparator* comparator); 17 | ~TimekeyComparator(); 18 | 19 | int Compare(const leveldb::Slice& a, const leveldb::Slice& b) const; 20 | 21 | const char* Name() const; 22 | 23 | void FindShortestSeparator(std::string* start, const leveldb::Slice& limit) const; 24 | 25 | void FindShortSuccessor(std::string* key) const; 26 | 27 | private: 28 | const leveldb::Comparator* comparator_; 29 | }; 30 | 31 | const TimekeyComparator* NewTimekeyComparator(const leveldb::Comparator* comparator); 32 | 33 | } // namespace io 34 | } // namespace tera 35 | 36 | #endif // TERA_IO_TIMEKEY_COMARATOR_H_ 37 | -------------------------------------------------------------------------------- /src/leveldb/.gitignore: -------------------------------------------------------------------------------- 1 | build_config.mk 2 | *.a 3 | *.o 4 | *.dylib* 5 | *.so 6 | *.so.* 7 | *_test 8 | db_bench 9 | -------------------------------------------------------------------------------- /src/leveldb/AUTHORS: -------------------------------------------------------------------------------- 1 | # Names should be added to this file like so: 2 | # Name or Organization 3 | 4 | Google Inc. 5 | 6 | # Initial version authors: 7 | Jeffrey Dean 8 | Sanjay Ghemawat 9 | 10 | # Partial list of contributors: 11 | Kevin Regan 12 | -------------------------------------------------------------------------------- /src/leveldb/README.md: -------------------------------------------------------------------------------- 1 | ####Tera基于leveldb实现数据持久化存储，在原版leveldb的基础上开发了以下组件 2 | ######DBTable 3 | 支持按列存储优化(Localitygroup)，通过封装多个DBImpl，并使用WAL&MVCC保证行级原子性读写 4 | ######DFSEnv 5 | 支持分布式文件系统，可以对接HDFS，NFS等文件系统 6 | ######FlashEnv 7 | 支持SSD盘缓存，将一个数据副本置于本地SSD，加速随机读，并使用DFS持久化数据 8 | ######InMemoryEnv 9 | 支持内存数据库，将一个副本置于本机内存中，加速读访问，并使用DFS持久化数据 10 | ######Split&Merge 11 | 支持将一个leveldb分裂成两个或将两个leveldb合并成为一个，分裂合并只修改元数据，所以非常高效 12 | ######ThreadPool 13 | 支持多个线程进行后台compact 14 | ######MemTableOnLevelDB 15 | 支持LSM-tree版本的内存表，替代了跳表实现，解决表格展开为kv时，内存表性能不足的问题 16 | ######AsyncWriter 17 | 支持在log写入阻塞的情况下，切换新的文件写入，降低写延时 18 | -------------------------------------------------------------------------------- /src/leveldb/TODO: -------------------------------------------------------------------------------- 1 | ss 2 | - Stats 3 | 4 | db 5 | - Maybe implement DB::BulkDeleteForRange(start_key, end_key) 6 | that would blow away files whose ranges are entirely contained 7 | within [start_key..end_key]? For Chrome, deletion of obsolete 8 | object stores, etc. can be done in the background anyway, so 9 | probably not that important. 10 | - There have been requests for MultiGet. 11 | 12 | After a range is completely deleted, what gets rid of the 13 | corresponding files if we do no future changes to that range. Make 14 | the conditions for triggering compactions fire in more situations? 15 | -------------------------------------------------------------------------------- /src/leveldb/db/builder.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | // 5 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 6 | // Use of this source code is governed by a BSD-style license that can be 7 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 8 | 9 | #ifndef STORAGE_LEVELDB_DB_BUILDER_H_ 10 | #define STORAGE_LEVELDB_DB_BUILDER_H_ 11 | 12 | #include 13 | 14 | #include "leveldb/status.h" 15 | 16 | namespace leveldb { 17 | 18 | struct Options; 19 | struct FileMetaData; 20 | 21 | class Env; 22 | class Iterator; 23 | class TableCache; 24 | class VersionEdit; 25 | 26 | // Build a Table file from the contents of *iter. The generated file 27 | // will be named according to meta->number. On success, the rest of 28 | // *meta will be filled with metadata about the generated table. 29 | // If no data is present in *iter, meta->file_size will be set to 30 | // zero, and no Table file will be produced. 31 | extern Status BuildTable(const std::string& dbname, Env* env, const Options& options, 32 | TableCache* table_cache, Iterator* iter, FileMetaData* meta, 33 | uint64_t* saved_size, uint64_t smallest_snapshot); 34 | 35 | } // namespace leveldb 36 | 37 | #endif // STORAGE_LEVELDB_DB_BUILDER_H_ 38 | -------------------------------------------------------------------------------- /src/leveldb/db/db_iter.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | // 5 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 6 | // Use of this source code is governed by a BSD-style license that can be 7 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 8 | 9 | #ifndef STORAGE_LEVELDB_DB_DB_ITER_H_ 10 | #define STORAGE_LEVELDB_DB_DB_ITER_H_ 11 | 12 | #include 13 | #include "leveldb/db.h" 14 | #include "db/dbformat.h" 15 | 16 | namespace leveldb { 17 | 18 | // Return a new iterator that converts internal keys (yielded by 19 | // "*internal_iter") that were live at the specified "sequence" number 20 | // into appropriate user keys. 21 | extern Iterator* NewDBIterator(const std::string* dbname, Env* env, 22 | const Comparator* user_key_comparator, Iterator* internal_iter, 23 | const SequenceNumber& sequence, 24 | const std::map& rollbacks); 25 | 26 | } // namespace leveldb 27 | 28 | #endif // STORAGE_LEVELDB_DB_DB_ITER_H_ 29 | -------------------------------------------------------------------------------- /src/leveldb/db/leveldb_main.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | // 5 | // Copyright (c) 2012 The LevelDB Authors. All rights reserved. 6 | // Use of this source code is governed by a BSD-style license that can be 7 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 8 | 9 | #include 10 | #include "leveldb/env.h" 11 | #include "leveldb/table_utils.h" 12 | 13 | static void Usage() { 14 | fprintf(stderr, 15 | "Usage: leveldbutil command...\n" 16 | " dump files... -- dump contents of specified files\n"); 17 | } 18 | 19 | int main(int argc, char** argv) { 20 | leveldb::Env* env = leveldb::Env::Default(); 21 | bool ok = true; 22 | if (argc < 2) { 23 | Usage(); 24 | ok = false; 25 | } else { 26 | std::string command = argv[1]; 27 | if (command == "dump") { 28 | ok = leveldb::HandleDumpCommand(env, argv + 2, argc - 2); 29 | } else { 30 | Usage(); 31 | ok = false; 32 | } 33 | } 34 | return (ok ? 0 : 1); 35 | } 36 | -------------------------------------------------------------------------------- /src/leveldb/db/lg_compact_thread.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef LEVELDB_DB_LG_COMPACT_THREAD_H_ 6 | #define LEVELDB_DB_LG_COMPACT_THREAD_H_ 7 | 8 | #include "leveldb/slice.h" 9 | #include "util/thread.h" 10 | #include "db/db_impl.h" 11 | 12 | namespace leveldb { 13 | 14 | class LGCompactThread : public Thread { 15 | public: 16 | LGCompactThread(uint32_t lg_id, DBImpl* lg_impl, const Slice* begin = NULL, 17 | const Slice* end = NULL) 18 | : lg_id_(lg_id), lg_impl_(lg_impl), begin_(begin), end_(end) {} 19 | virtual ~LGCompactThread() {} 20 | 21 | virtual void Run(void* params) { lg_impl_->CompactRange(begin_, end_); } 22 | 23 | private: 24 | uint32_t lg_id_; 25 | DBImpl* lg_impl_; 26 | const Slice* begin_; 27 | const Slice* end_; 28 | }; 29 | 30 | } // namespace leveldb 31 | 32 | #endif // LEVELDB_DB_LG_COMPACT_THREAD_H_ 33 | -------------------------------------------------------------------------------- /src/leveldb/db/lg_write_thread.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef LEVELDB_DB_LG_WRITE_THREAD_H_ 6 | #define LEVELDB_DB_LG_WRITE_THREAD_H_ 7 | 8 | #include "db/db_impl.h" 9 | #include "leveldb/write_batch.h" 10 | #include "util/thread.h" 11 | 12 | namespace leveldb { 13 | 14 | class LGWriteThread : public Thread { 15 | public: 16 | LGWriteThread(uint32_t lg_id, DBImpl* impl, const WriteOptions& opts, WriteBatch* bench) 17 | : wopts_(opts), wbench_(bench); 18 | virtual ~LGWriteThread() {} 19 | 20 | virtual void Run(void* params) { 21 | std::cout << "LG Thread #" << lg_id_ << ": Write()" << std::endl; 22 | lg_impl_->Write(wopts_, wbatch_); 23 | } 24 | 25 | Status GetResult() { return ret_; } 26 | 27 | private: 28 | uint32_t lg_id_; 29 | DBImpl* lg_impl_; 30 | const WriteOptions& wopts_; 31 | WriteBatch* wbatch_; 32 | Status ret_; 33 | }; 34 | 35 | } // namespace leveldb 36 | 37 | #endif // LEVELDB_DB_LG_WRITE_THREAD_H_ 38 | -------------------------------------------------------------------------------- /src/leveldb/db/log_format.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | // 5 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 6 | // Use of this source code is governed by a BSD-style license that can be 7 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 8 | // 9 | // Log format information shared by reader and writer. 10 | // See ../doc/log_format.txt for more detail. 11 | 12 | #ifndef STORAGE_LEVELDB_DB_LOG_FORMAT_H_ 13 | #define STORAGE_LEVELDB_DB_LOG_FORMAT_H_ 14 | 15 | namespace leveldb { 16 | namespace log { 17 | 18 | enum RecordType { 19 | // Zero is reserved for preallocated files 20 | kZeroType = 0, 21 | 22 | kFullType = 1, 23 | 24 | // For fragments 25 | kFirstType = 2, 26 | kMiddleType = 3, 27 | kLastType = 4 28 | }; 29 | static const int kMaxRecordType = kLastType; 30 | 31 | static const uint32_t kBlockSize = 32768; 32 | 33 | // Header is checksum (4 bytes), type (1 byte), length (2 bytes). 34 | static const uint32_t kHeaderSize = 4 + 1 + 2; 35 | 36 | } // namespace log 37 | } // namespace leveldb 38 | 39 | #endif // STORAGE_LEVELDB_DB_LOG_FORMAT_H_ 40 | -------------------------------------------------------------------------------- /src/leveldb/db/table_utils_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #include "leveldb/table_utils.h" 6 | 7 | #include "util/testharness.h" 8 | 9 | namespace leveldb { 10 | 11 | class TableUtilsTest {}; 12 | 13 | TEST(TableUtilsTest, HeadAndDumpManifest) { 14 | // the hex content of the manifest 15 | uint8_t content[] = { 16 | 0x6e, 0x11, 0x5f, 0x7f, 0x2d, 0x00, 0x01, 0x9a, 0x80, 0x40, 0x01, 0x19, 0x74, 0x65, 17 | 0x72, 0x61, 0x2e, 0x54, 0x65, 0x72, 0x61, 0x42, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x43, 18 | 0x6f, 0x6d, 0x70, 0x61, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x81, 0x80, 0x40, 0x02, 0x00, 19 | 0x81, 0x80, 0x40, 0x03, 0x02, 0x81, 0x80, 0x40, 0x04, 0x00, 0x0a, 20 | }; 21 | 22 | std::string manifest_file("./MANIFEST-000001"); 23 | FILE* file = fopen(manifest_file.c_str(), "wb"); 24 | size_t len = sizeof(content) / sizeof(uint8_t); 25 | fwrite(content, 1, len, file); 26 | fclose(file); 27 | 28 | leveldb::Env* env = leveldb::Env::Default(); 29 | 30 | bool ret = false; 31 | ret = DumpFile(env, manifest_file); 32 | ASSERT_TRUE(ret); 33 | remove(manifest_file.c_str()); 34 | } 35 | } 36 | 37 | int main(int argc, char** argv) { return leveldb::test::RunAllTests(); } 38 | -------------------------------------------------------------------------------- /src/leveldb/hdfs.c.patch: -------------------------------------------------------------------------------- 1 | int hdfsSync(hdfsFS fs, hdfsFile f) 2 | { 3 | // JAVA EQUIVALENT 4 | // FSDataOutputStream.sync 5 | //Get the JNIEnv* corresponding to current thread 6 | JNIEnv* env = getJNIEnv(); 7 | if (env == NULL) { 8 | errno = EINTERNAL; 9 | return -1; 10 | } 11 | //Parameters 12 | jobject jOutputStream = (jobject)(f ? f->file : 0); 13 | //Caught exception 14 | jthrowable jExc = NULL; 15 | //Sanity check 16 | if (!f || f->type != OUTPUT) { 17 | errno = EBADF; 18 | return -1; 19 | } 20 | if (invokeMethod(env, NULL, &jExc, INSTANCE, jOutputStream, 21 | HADOOP_OSTRM, "sync", "()V") != 0) { 22 | errno = errnoFromException(jExc, env, "org.apache.hadoop.fs." 23 | "FSDataOutputStream::sync"); 24 | return -1; 25 | } 26 | return 0; 27 | } 28 | -------------------------------------------------------------------------------- /src/leveldb/helpers/memenv/memenv.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | // 5 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 6 | // Use of this source code is governed by a BSD-style license that can be 7 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 8 | 9 | #ifndef STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_ 10 | #define STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_ 11 | 12 | namespace leveldb { 13 | 14 | class Env; 15 | 16 | // Returns a new environment that stores its data in memory and delegates 17 | // all non-file-storage tasks to base_env. The caller must delete the result 18 | // when it is no longer needed. 19 | // *base_env must remain live while the result is in use. 20 | Env* NewMemEnv(Env* base_env); 21 | 22 | } // namespace leveldb 23 | 24 | #endif // STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_ 25 | -------------------------------------------------------------------------------- /src/leveldb/include/leveldb/lg_coding.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_LG_CODING_H_ 6 | #define STORAGE_LEVELDB_UTIL_LG_CODING_H_ 7 | 8 | #include 9 | 10 | #include "leveldb/slice.h" 11 | 12 | namespace leveldb { 13 | 14 | extern void PutFixed32LGId(std::string* dst, uint32_t lg_id); 15 | 16 | extern bool GetFixed32LGId(Slice* input, uint32_t* lg_id); 17 | 18 | } // namespace leveldb 19 | 20 | #endif // STORAGE_LEVELDB_UTIL_LG_CODING_H_ 21 | -------------------------------------------------------------------------------- /src/leveldb/include/leveldb/raw_key_operator.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_LEVELDB_UTILS_RAW_KEY_OPERATOR_H 6 | #define TERA_LEVELDB_UTILS_RAW_KEY_OPERATOR_H 7 | 8 | #include 9 | 10 | #include "leveldb/slice.h" 11 | #include "leveldb/tera_key.h" 12 | 13 | namespace leveldb { 14 | 15 | class RawKeyOperator { 16 | public: 17 | virtual void EncodeTeraKey(const std::string& row_key, const std::string& family, 18 | const std::string& qualifier, int64_t timestamp, TeraKeyType type, 19 | std::string* tera_key) const = 0; 20 | 21 | virtual bool ExtractTeraKey(const Slice& tera_key, Slice* row_key, Slice* family, 22 | Slice* qualifier, int64_t* timestamp, TeraKeyType* type) const = 0; 23 | virtual int Compare(const Slice& key1, const Slice& key2) const = 0; 24 | virtual const char* Name() const = 0; 25 | }; 26 | 27 | const RawKeyOperator* ReadableRawKeyOperator(); 28 | const RawKeyOperator* BinaryRawKeyOperator(); 29 | const RawKeyOperator* KvRawKeyOperator(); 30 | 31 | } // namespace leveldb 32 | #endif // TERA_LEVELDB_UTILS_RAW_KEY_OPERATOR_H 33 | -------------------------------------------------------------------------------- /src/leveldb/include/leveldb/table_utils.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_TABLE_UTILS_H 6 | #define STORAGE_LEVELDB_DB_TABLE_UTILS_H 7 | 8 | #include 9 | #include 10 | 11 | #include "leveldb/env.h" 12 | #include "leveldb/comparator.h" 13 | 14 | namespace leveldb { 15 | 16 | void ArchiveFile(Env* env, const std::string& fname); 17 | 18 | bool HandleDumpCommand(Env* env, char** files, int num); 19 | 20 | bool DumpFile(Env* env, const std::string& fname); 21 | } // namespace leveldb 22 | 23 | #endif // STORAGE_LEVELDB_DB_TABLE_UTILS_H 24 | -------------------------------------------------------------------------------- /src/leveldb/persistent_cache/sharded_persistent_cache_impl.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | #include "sharded_persistent_cache_impl.h" 5 | #include "leveldb/persistent_cache.h" 6 | 7 | namespace leveldb { 8 | Status NewShardedPersistentCache(const std::vector &configs, 9 | std::shared_ptr *cache) { 10 | if (!cache) { 11 | return Status::IOError("invalid argument cache"); 12 | } 13 | 14 | auto pcache = std::make_shared(configs); 15 | Status s = pcache->Open(); 16 | 17 | if (!s.ok()) { 18 | return s; 19 | } 20 | 21 | *cache = pcache; 22 | return s; 23 | } 24 | } // namespace leveldb 25 | -------------------------------------------------------------------------------- /src/leveldb/port/README: -------------------------------------------------------------------------------- 1 | This directory contains interfaces and implementations that isolate the 2 | rest of the package from platform details. 3 | 4 | Code in the rest of the package includes "port.h" from this directory. 5 | "port.h" in turn includes a platform specific "port_.h" file 6 | that provides the platform specific implementation. 7 | 8 | See port_posix.h for an example of what must be provided in a platform 9 | specific header file. 10 | 11 | -------------------------------------------------------------------------------- /src/leveldb/port/port.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | // 5 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 6 | // Use of this source code is governed by a BSD-style license that can be 7 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 8 | 9 | #ifndef STORAGE_LEVELDB_PORT_PORT_H_ 10 | #define STORAGE_LEVELDB_PORT_PORT_H_ 11 | 12 | #include 13 | 14 | // Include the appropriate platform specific file below. If you are 15 | // porting to a new platform, see "port_example.h" for documentation 16 | // of what the new port_.h file must provide. 17 | // #if defined(LEVELDB_PLATFORM_POSIX) 18 | // # include "port/port_posix.h" 19 | // #elif defined(LEVELDB_PLATFORM_CHROMIUM) 20 | // # include "port/port_chromium.h" 21 | // #endif 22 | 23 | // we only support posix in tera 24 | #include "port/port_posix.h" 25 | 26 | #endif // STORAGE_LEVELDB_PORT_PORT_H_ 27 | -------------------------------------------------------------------------------- /src/leveldb/port/win/stdint.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | // 5 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 6 | // Use of this source code is governed by a BSD-style license that can be 7 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 8 | 9 | // MSVC didn't ship with this file until the 2010 version. 10 | 11 | #ifndef STORAGE_LEVELDB_PORT_WIN_STDINT_H_ 12 | #define STORAGE_LEVELDB_PORT_WIN_STDINT_H_ 13 | 14 | #if !defined(_MSC_VER) 15 | #error This file should only be included when compiling with MSVC. 16 | #endif 17 | 18 | // Define C99 equivalent types. 19 | typedef signed char int8_t; 20 | typedef signed short int16_t; 21 | typedef signed int int32_t; 22 | typedef signed long long int64_t; 23 | typedef unsigned char uint8_t; 24 | typedef unsigned short uint16_t; 25 | typedef unsigned int uint32_t; 26 | typedef unsigned long long uint64_t; 27 | 28 | #endif // STORAGE_LEVELDB_PORT_WIN_STDINT_H_ 29 | -------------------------------------------------------------------------------- /src/leveldb/sample/Makefile: -------------------------------------------------------------------------------- 1 | CXX=g++ 2 | CXX_FLAGS=-g -fPIC -Wall -Wextra -pipe -Wno-unused 3 | CXX_INC= -I../include 4 | CXX_LINK=-Xlinker "-(" \ 5 | ../lib/libleveldb.a \ 6 | -lpthread \ 7 | -lcrypto \ 8 | -lrt \ 9 | -Xlinker "-)" 10 | 11 | OBJS=sample 12 | 13 | all: $(OBJS) 14 | 15 | sample: sample.cpp 16 | $(CXX) $^ $(CXX_INC) $(CXX_LINK) $(CXX_FLAGS) -o $@ 17 | 18 | clean: 19 | -rm -f *.o *.a $(OBJS) 20 | -------------------------------------------------------------------------------- /src/leveldb/table/block.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | // 5 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 6 | // Use of this source code is governed by a BSD-style license that can be 7 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 8 | 9 | #ifndef STORAGE_LEVELDB_TABLE_BLOCK_H_ 10 | #define STORAGE_LEVELDB_TABLE_BLOCK_H_ 11 | 12 | #include 13 | #include 14 | #include "leveldb/iterator.h" 15 | 16 | namespace leveldb { 17 | 18 | struct BlockContents; 19 | class Comparator; 20 | 21 | class Block { 22 | public: 23 | // Initialize the block with the specified contents. 24 | explicit Block(const BlockContents& contents); 25 | 26 | ~Block(); 27 | 28 | size_t size() const { return size_; } 29 | Iterator* NewIterator(const Comparator* comparator); 30 | 31 | private: 32 | uint32_t NumRestarts() const; 33 | 34 | const char* data_; 35 | size_t size_; 36 | uint32_t restart_offset_; // Offset in data_ of restart array 37 | bool owned_; // Block owns data_[] 38 | 39 | // No copying allowed 40 | Block(const Block&); 41 | void operator=(const Block&); 42 | 43 | class Iter; 44 | }; 45 | 46 | } // namespace leveldb 47 | 48 | #endif // STORAGE_LEVELDB_TABLE_BLOCK_H_ 49 | -------------------------------------------------------------------------------- /src/leveldb/table/merger.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | // 5 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 6 | // Use of this source code is governed by a BSD-style license that can be 7 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 8 | 9 | #ifndef STORAGE_LEVELDB_TABLE_MERGER_H_ 10 | #define STORAGE_LEVELDB_TABLE_MERGER_H_ 11 | 12 | namespace leveldb { 13 | 14 | class Comparator; 15 | class Iterator; 16 | 17 | // Return an iterator that provided the union of the data in 18 | // children[0,n-1]. Takes ownership of the child iterators and 19 | // will delete them when the result iterator is deleted. 20 | // 21 | // The result does no duplicate suppression. I.e., if a particular 22 | // key is present in K child iterators, it will be yielded K times. 23 | // 24 | // REQUIRES: n >= 0 25 | extern Iterator* NewMergingIterator(const Comparator* comparator, Iterator** children, int n); 26 | 27 | } // namespace leveldb 28 | 29 | #endif // STORAGE_LEVELDB_TABLE_MERGER_H_ 30 | -------------------------------------------------------------------------------- /src/leveldb/util/dfs.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | // 5 | // Author: yanshiguang02@baidu.com 6 | 7 | #include "leveldb/dfs.h" 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | namespace leveldb { 14 | 15 | static void* handle = NULL; 16 | 17 | Dfs* Dfs::NewDfs(const std::string& so_path, const std::string& conf) { 18 | dlerror(); 19 | fprintf(stderr, "Open %s\n", so_path.c_str()); 20 | handle = dlopen(so_path.c_str(), RTLD_LAZY | RTLD_LOCAL | RTLD_DEEPBIND); 21 | const char* err = dlerror(); 22 | if (handle == NULL) { 23 | fprintf(stderr, "Open %s fail: %s\n", so_path.c_str(), err); 24 | return NULL; 25 | } 26 | 27 | DfsCreator creator = (DfsCreator)dlsym(handle, "NewDfs"); 28 | err = dlerror(); 29 | if (err != NULL) { 30 | fprintf(stderr, "Load NewDfs from %s fail: %s\n", so_path.c_str(), err); 31 | return NULL; 32 | } 33 | return (*creator)(conf.c_str()); 34 | } 35 | } 36 | 37 | /* vim: set expandtab ts=4 sw=4 sts=4 tw=100: */ 38 | -------------------------------------------------------------------------------- /src/leveldb/util/filter_policy.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | // 5 | // Copyright (c) 2012 The LevelDB Authors. All rights reserved. 6 | // Use of this source code is governed by a BSD-style license that can be 7 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 8 | 9 | #include "leveldb/filter_policy.h" 10 | 11 | namespace leveldb { 12 | 13 | FilterPolicy::~FilterPolicy() {} 14 | 15 | } // namespace leveldb 16 | -------------------------------------------------------------------------------- /src/leveldb/util/hash.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | // 5 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 6 | // Use of this source code is governed by a BSD-style license that can be 7 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 8 | // 9 | // Simple hash function used for internal data structures 10 | 11 | #ifndef STORAGE_LEVELDB_UTIL_HASH_H_ 12 | #define STORAGE_LEVELDB_UTIL_HASH_H_ 13 | 14 | #include 15 | #include 16 | 17 | namespace leveldb { 18 | 19 | extern uint32_t Hash(const char* data, size_t n, uint32_t seed); 20 | } 21 | 22 | #endif // STORAGE_LEVELDB_UTIL_HASH_H_ 23 | -------------------------------------------------------------------------------- /src/leveldb/util/histogram.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | // 5 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 6 | // Use of this source code is governed by a BSD-style license that can be 7 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 8 | 9 | #ifndef STORAGE_LEVELDB_UTIL_HISTOGRAM_H_ 10 | #define STORAGE_LEVELDB_UTIL_HISTOGRAM_H_ 11 | 12 | #include 13 | #include "port/port_posix.h" 14 | #include "util/mutexlock.h" 15 | 16 | namespace leveldb { 17 | 18 | class Histogram { 19 | public: 20 | Histogram() { Clear(); } 21 | ~Histogram() {} 22 | 23 | void Clear(); 24 | void Add(double value); 25 | void Merge(const Histogram& other); 26 | 27 | std::string ToString() const; 28 | 29 | private: 30 | mutable port::Mutex mutex_; 31 | double min_; 32 | double max_; 33 | double num_; 34 | double sum_; 35 | double sum_squares_; 36 | 37 | enum { kNumBuckets = 154 }; 38 | static const double kBucketLimit[kNumBuckets]; 39 | double buckets_[kNumBuckets]; 40 | 41 | public: 42 | double Median() const; 43 | double Percentile(double p) const; 44 | double Average() const; 45 | double StandardDeviation() const; 46 | }; 47 | 48 | } // namespace leveldb 49 | 50 | #endif // STORAGE_LEVELDB_UTIL_HISTOGRAM_H_ 51 | -------------------------------------------------------------------------------- /src/leveldb/util/lg_coding.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #include "leveldb/lg_coding.h" 6 | 7 | #include "util/coding.h" 8 | 9 | namespace leveldb { 10 | 11 | const std::string KG_PREFIX = "//LG_ID//"; 12 | 13 | void PutFixed32LGId(std::string* dst, uint32_t lg_id) { 14 | std::string lg_str; 15 | PutLengthPrefixedSlice(&lg_str, KG_PREFIX); 16 | PutVarint32(&lg_str, lg_id); 17 | PutLengthPrefixedSlice(&lg_str, *dst); 18 | *dst = lg_str; 19 | } 20 | 21 | bool GetFixed32LGId(Slice* input, uint32_t* lg_id) { 22 | Slice lg_str(*input); 23 | Slice str; 24 | if (!GetLengthPrefixedSlice(&lg_str, &str)) { 25 | return false; 26 | } else if (str != KG_PREFIX) { 27 | return false; 28 | } else if (!GetVarint32(&lg_str, lg_id)) { 29 | return false; 30 | } 31 | if (!GetLengthPrefixedSlice(&lg_str, input)) { 32 | return false; 33 | } 34 | return true; 35 | } 36 | 37 | } // namespace leveldb 38 | -------------------------------------------------------------------------------- /src/leveldb/util/nfs_version.h: -------------------------------------------------------------------------------- 1 | #ifndef STORAGE_LEVELDB_UTIL_NFS_VERSION_H_ 2 | #define STORAGE_LEVELDB_UTIL_NFS_VERSION_H_ 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | extern const char kNfsSvnInfo[]; 9 | extern const char kNfsBuildType[]; 10 | extern const char kNfsBuildTime[]; 11 | extern const char kNfsBuilderName[]; 12 | extern const char kNfsHostName[]; 13 | extern const char kNfsCompiler[]; 14 | const char* PrintNfsVersion(); 15 | 16 | #ifdef __cplusplus 17 | } 18 | #endif 19 | 20 | #endif // STORAGE_LEVELDB_UTIL_NFS_VERSION_H_ 21 | -------------------------------------------------------------------------------- /src/leveldb/util/random.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | // 5 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 6 | // Use of this source code is governed by a BSD-style license that can be 7 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 8 | 9 | #include "util/random.h" 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #define STORAGE_DECL static __thread 17 | 18 | #if defined(__GNUC__) && __GNUC__ >= 4 19 | #define LIKELY(x) (__builtin_expect((x), 1)) 20 | #define UNLIKELY(x) (__builtin_expect((x), 0)) 21 | #else 22 | #define LIKELY(x) (x) 23 | #define UNLIKELY(x) (x) 24 | #endif 25 | 26 | namespace leveldb { 27 | 28 | Random* Random::GetTLSInstance() { 29 | STORAGE_DECL Random* tls_instance; 30 | STORAGE_DECL std::aligned_storage::type tls_instance_bytes; 31 | 32 | auto rv = tls_instance; 33 | if ((rv == nullptr)) { 34 | size_t seed = std::hash()(std::this_thread::get_id()); 35 | rv = new (&tls_instance_bytes) Random((uint32_t)seed); 36 | tls_instance = rv; 37 | } 38 | return rv; 39 | } 40 | 41 | } // namespace rocksdb 42 | -------------------------------------------------------------------------------- /src/leveldb/util/slog.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | // 5 | // Author: An Qin (qinan@baidu.com) 6 | 7 | #include "leveldb/slog.h" 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | namespace leveldb { 14 | 15 | static LogLevel s_log_level = ::leveldb::LOG_LEVEL_ERROR; 16 | 17 | LogLevel GetLogLevel() { return s_log_level; } 18 | 19 | void SetLogLevel(LogLevel level) { s_log_level = level; } 20 | 21 | void LogHandler(LogLevel level, const char* filename, int line, const char* fmt, ...) { 22 | static const char* level_names[] = {"FATAL", "ERROR", "WARNNING", "INFO", "TRACE", "DEBUG"}; 23 | char buf[1024]; 24 | va_list ap; 25 | va_start(ap, fmt); 26 | vsnprintf(buf, 1024, fmt, ap); 27 | va_end(ap); 28 | 29 | fprintf(stderr, "[LevelDB %s %s:%d] %s\n", level_names[level], filename, line, buf); 30 | fflush(stderr); 31 | 32 | if (level == ::leveldb::LOG_LEVEL_FATAL) { 33 | abort(); 34 | } 35 | } 36 | 37 | } // namespace leveldb 38 | -------------------------------------------------------------------------------- /src/leveldb/util/stop_watch.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | #pragma once 5 | #include "leveldb/env.h" 6 | 7 | namespace leveldb { 8 | // Auto-scoped. 9 | // Records the measure time into the corresponding histogram if statistics 10 | // is not nullptr. It is also saved into *elapsed if the pointer is not nullptr 11 | // and overwrite is true, it will be added to *elapsed if overwrite is false. 12 | 13 | // a nano second precision stopwatch 14 | class StopWatchMicro { 15 | public: 16 | explicit StopWatchMicro(Env* const env, bool auto_start = false) : env_(env), start_(0) { 17 | if (auto_start) { 18 | Start(); 19 | } 20 | } 21 | 22 | void Start() { start_ = env_->NowMicros(); } 23 | 24 | uint64_t ElapsedMicros(bool reset = false) { 25 | auto now = env_->NowMicros(); 26 | auto elapsed = now - start_; 27 | if (reset) { 28 | start_ = now; 29 | } 30 | return elapsed; 31 | } 32 | 33 | uint64_t ElapsedMicrosSafe(bool reset = false) { 34 | return (env_ != nullptr) ? ElapsedMicros(reset) : 0U; 35 | } 36 | 37 | private: 38 | Env* const env_; 39 | uint64_t start_; 40 | }; 41 | 42 | } // namespace leveldb 43 | -------------------------------------------------------------------------------- /src/leveldb/util/thread.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #include "util/thread.h" 6 | 7 | namespace leveldb { 8 | 9 | Thread::Thread() : started_(false), id_(0) {} 10 | 11 | Thread::~Thread() { 12 | Cancel(); 13 | Join(); 14 | } 15 | 16 | bool Thread::Start() { 17 | { 18 | MutexLock lock(&mutex_); 19 | if (!started_) { 20 | started_ = true; 21 | } else { 22 | return false; 23 | } 24 | } 25 | 26 | if (0 != pthread_create(&id_, NULL, StartRunner, this)) { 27 | started_ = false; 28 | return false; 29 | } 30 | 31 | return true; 32 | } 33 | 34 | void Thread::Join() { 35 | if (started_) { 36 | pthread_join(id_, NULL); 37 | } 38 | } 39 | 40 | void Thread::Cancel() { 41 | if (started_) { 42 | pthread_cancel(id_); 43 | } 44 | } 45 | 46 | void Thread::Stop() { 47 | MutexLock lock(&mutex_); 48 | started_ = false; 49 | } 50 | 51 | void* Thread::StartRunner(void* params) { 52 | Thread* runner = static_cast(params); 53 | runner->Run(params); 54 | runner->Stop(); 55 | return NULL; 56 | } 57 | 58 | } // namespace leveldb 59 | -------------------------------------------------------------------------------- /src/leveldb/util/thread.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef LEVELDB_UTIL_THREAD_H_ 6 | #define LEVELDB_UTIL_THREAD_H_ 7 | 8 | #include 9 | 10 | #include "util/mutexlock.h" 11 | 12 | namespace leveldb { 13 | 14 | class Thread { 15 | public: 16 | Thread(); 17 | virtual ~Thread(); 18 | 19 | bool Start(); 20 | void Join(); 21 | void Cancel(); 22 | pthread_t Id() const; 23 | bool IsRunning() const; 24 | 25 | virtual void Run(void* params) = 0; 26 | 27 | private: 28 | void Stop(); 29 | static void* StartRunner(void* params); 30 | 31 | private: 32 | bool started_; 33 | pthread_t id_; 34 | mutable port::Mutex mutex_; 35 | }; 36 | 37 | inline pthread_t Thread::Id() const { return id_; } 38 | 39 | inline bool Thread::IsRunning() const { 40 | MutexLock lock(&mutex_); 41 | return started_; 42 | } 43 | 44 | } // namespace leveldb 45 | 46 | #endif // LEVELDB_UTIL_THREAD_H_ 47 | -------------------------------------------------------------------------------- /src/load_balancer/action.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_LOAD_BALANCER_ACTION_H_ 6 | #define TERA_LOAD_BALANCER_ACTION_H_ 7 | 8 | #include 9 | #include 10 | 11 | namespace tera { 12 | namespace load_balancer { 13 | 14 | class Action { 15 | public: 16 | enum class Type { 17 | ASSIGN, 18 | MOVE, 19 | SWAP, 20 | EMPTY, 21 | }; 22 | 23 | Type GetType() const { return type_; } 24 | 25 | std::string GetGeneratorName() const { return generator_; } 26 | 27 | public: 28 | Action(Type t, const std::string& generator) : type_(t), generator_(generator) {} 29 | 30 | virtual ~Action() {} 31 | 32 | virtual Action* UndoAction() = 0; 33 | 34 | virtual std::string ToString() const = 0; 35 | 36 | private: 37 | Type type_; 38 | std::string generator_; 39 | }; 40 | 41 | } // namespace load_balancer 42 | } // namespace tera 43 | 44 | #endif // TERA_LOAD_BALANCER_ACTION_H_ 45 | -------------------------------------------------------------------------------- /src/load_balancer/actions.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #include 6 | 7 | #include "load_balancer/actions.h" 8 | 9 | namespace tera { 10 | namespace load_balancer { 11 | 12 | EmptyAction::EmptyAction() : Action(Action::Type::EMPTY, "") {} 13 | 14 | EmptyAction::~EmptyAction() {} 15 | 16 | Action* EmptyAction::UndoAction() { return new EmptyAction(); } 17 | 18 | std::string EmptyAction::ToString() const { return "EmptyAction"; } 19 | 20 | MoveAction::MoveAction(uint32_t tablet_index, uint32_t source_node_index, uint32_t dest_node_index, 21 | const std::string& generator) 22 | : Action(Action::Type::MOVE, generator), 23 | tablet_index_(tablet_index), 24 | source_node_index_(source_node_index), 25 | dest_node_index_(dest_node_index) {} 26 | 27 | MoveAction::~MoveAction() {} 28 | 29 | Action* MoveAction::UndoAction() { 30 | return new MoveAction(tablet_index_, dest_node_index_, source_node_index_, "UndoAction"); 31 | } 32 | 33 | std::string MoveAction::ToString() const { 34 | return "move " + std::to_string(tablet_index_) + " from " + std::to_string(source_node_index_) + 35 | " to " + std::to_string(dest_node_index_) + " generated by " + GetGeneratorName(); 36 | } 37 | 38 | } // namespace load_balancer 39 | } // namespace tera 40 | -------------------------------------------------------------------------------- /src/load_balancer/actions.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_LOAD_BALANCER_ACTIONS_H_ 6 | #define TERA_LOAD_BALANCER_ACTIONS_H_ 7 | 8 | #include 9 | 10 | #include "load_balancer/action.h" 11 | 12 | namespace tera { 13 | namespace load_balancer { 14 | 15 | class EmptyAction : public Action { 16 | public: 17 | EmptyAction(); 18 | virtual ~EmptyAction(); 19 | 20 | virtual Action* UndoAction() override; 21 | 22 | virtual std::string ToString() const override; 23 | }; 24 | 25 | class MoveAction : public Action { 26 | public: 27 | MoveAction(uint32_t tablet_index, uint32_t source_node_index, uint32_t dest_node_index, 28 | const std::string& generator); 29 | 30 | virtual ~MoveAction(); 31 | 32 | virtual Action* UndoAction() override; 33 | 34 | virtual std::string ToString() const override; 35 | 36 | public: 37 | uint32_t tablet_index_; 38 | uint32_t source_node_index_; 39 | uint32_t dest_node_index_; 40 | }; 41 | 42 | } // namespace load_balancer 43 | } // namespace tera 44 | 45 | #endif // TERA_LOAD_BALANCER_ACTIONS_H_ 46 | -------------------------------------------------------------------------------- /src/load_balancer/balancer.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_LOAD_BALANCER_BALANCER_H_ 6 | #define TERA_LOAD_BALANCER_BALANCER_H_ 7 | 8 | #include 9 | #include 10 | 11 | #include "load_balancer/lb_node.h" 12 | #include "load_balancer/options.h" 13 | #include "load_balancer/plan.h" 14 | #include "master/tablet_manager.h" 15 | #include "master/tabletnode_manager.h" 16 | 17 | namespace tera { 18 | namespace load_balancer { 19 | 20 | class Balancer { 21 | public: 22 | virtual ~Balancer() {} 23 | 24 | // balance the whole cluster 25 | virtual bool BalanceCluster(const std::vector>& lb_nodes, 26 | std::vector* plans) = 0; 27 | 28 | // balance for the specified table 29 | virtual bool BalanceCluster(const std::string& table_name, 30 | const std::vector>& lb_nodes, 31 | std::vector* plans) = 0; 32 | 33 | virtual std::string GetName() = 0; 34 | }; 35 | 36 | } // namespace load_balancer 37 | } // namespace tera 38 | 39 | #endif // TERA_LOAD_BALANCER_BALANCER_H_ 40 | -------------------------------------------------------------------------------- /src/load_balancer/lb_entry.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_LOAD_BALANCER_LB_ENTRY_H_ 6 | #define TERA_LOAD_BALANCER_LB_ENTRY_H_ 7 | 8 | #include 9 | 10 | #include "sofa/pbrpc/pbrpc.h" 11 | 12 | #include "tera/tera_entry.h" 13 | 14 | namespace tera { 15 | namespace load_balancer { 16 | 17 | class LBServiceImpl; 18 | class LBImpl; 19 | 20 | class LBEntry : public TeraEntry { 21 | public: 22 | LBEntry(); 23 | virtual ~LBEntry(); 24 | 25 | virtual bool StartServer(); 26 | virtual bool Run(); 27 | virtual void ShutdownServer(); 28 | 29 | private: 30 | std::unique_ptr rpc_server_; 31 | LBServiceImpl* lb_service_impl_; 32 | std::shared_ptr lb_impl_; 33 | }; 34 | 35 | } // namespace load_balancer 36 | } // namespace tera 37 | 38 | #endif // TERA_LOAD_BALANCER_LB_ENTRY_H_ 39 | -------------------------------------------------------------------------------- /src/load_balancer/lb_node.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_LOAD_BALANCER_LB_NODE_H_ 6 | #define TERA_LOAD_BALANCER_LB_NODE_H_ 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | #include "master/tablet_manager.h" 13 | #include "master/tabletnode_manager.h" 14 | 15 | namespace tera { 16 | namespace load_balancer { 17 | 18 | struct LBTablet { 19 | tera::master::TabletPtr tablet_ptr; 20 | }; 21 | 22 | struct LBTabletNode { 23 | tera::master::TabletNodePtr tablet_node_ptr; 24 | std::vector> tablets; 25 | }; 26 | 27 | } // namespace load_balancer 28 | } // namespace tera 29 | 30 | #endif // TERA_LOAD_BALANCER_LB_NODE_H_ 31 | -------------------------------------------------------------------------------- /src/load_balancer/lb_service_impl.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_LOAD_BALANCER_LB_SERVICE_IMPL_H_ 6 | #define TERA_LOAD_BALANCER_LB_SERVICE_IMPL_H_ 7 | 8 | #include 9 | 10 | #include "common/thread_pool.h" 11 | #include "proto/load_balancer_rpc.pb.h" 12 | 13 | namespace tera { 14 | namespace load_balancer { 15 | 16 | class LBImpl; 17 | 18 | class LBServiceImpl : public LoadBalancerService { 19 | public: 20 | explicit LBServiceImpl(const std::shared_ptr& lb_impl); 21 | virtual ~LBServiceImpl(); 22 | 23 | void CmdCtrl(google::protobuf::RpcController* controller, const CmdCtrlRequest* request, 24 | CmdCtrlResponse* response, google::protobuf::Closure* done); 25 | 26 | private: 27 | void DoCmdCtrl(google::protobuf::RpcController* controller, const CmdCtrlRequest* request, 28 | CmdCtrlResponse* response, google::protobuf::Closure* done); 29 | 30 | private: 31 | std::shared_ptr lb_impl_; 32 | std::unique_ptr thread_pool_; 33 | }; 34 | 35 | } // namespace load_balancer 36 | } // namespace tera 37 | 38 | #endif // TERA_LOAD_BALANCER_LB_SERVICE_IMPL_H_ 39 | -------------------------------------------------------------------------------- /src/load_balancer/test/actions_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #include "gflags/gflags.h" 6 | #include "glog/logging.h" 7 | #include "gtest/gtest.h" 8 | 9 | #include "load_balancer/actions.h" 10 | 11 | namespace tera { 12 | namespace load_balancer { 13 | 14 | class ActionsTest : public ::testing::Test {}; 15 | 16 | TEST_F(ActionsTest, MoveActionTest) { 17 | MoveAction move_action(0, 0, 1, ""); 18 | std::shared_ptr undo_action(dynamic_cast(move_action.UndoAction())); 19 | 20 | ASSERT_EQ(move_action.tablet_index_, undo_action->tablet_index_); 21 | ASSERT_EQ(move_action.source_node_index_, undo_action->dest_node_index_); 22 | ASSERT_EQ(move_action.dest_node_index_, undo_action->source_node_index_); 23 | } 24 | 25 | } // namespace load_balancer 26 | } // namespace tera 27 | 28 | /* vim: set expandtab ts=4 sw=4 sts=4 tw=100: */ 29 | -------------------------------------------------------------------------------- /src/load_balancer/test/balancer_test_main.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #include 6 | 7 | #include "gflags/gflags.h" 8 | #include "glog/logging.h" 9 | #include "gtest/gtest.h" 10 | 11 | #include "utils/utils_cmd.h" 12 | #include "master/master_env.h" 13 | 14 | int main(int argc, char** argv) { 15 | ::google::InitGoogleLogging(argv[0]); 16 | FLAGS_v = 16; 17 | FLAGS_minloglevel = 0; 18 | FLAGS_log_dir = "./log"; 19 | if (access(FLAGS_log_dir.c_str(), F_OK)) { 20 | mkdir(FLAGS_log_dir.c_str(), 0777); 21 | } 22 | std::string pragram_name("load balancer"); 23 | tera::utils::SetupLog(pragram_name); 24 | ::google::ParseCommandLineFlags(&argc, &argv, true); 25 | ::testing::InitGoogleTest(&argc, argv); 26 | 27 | using tera::master::TabletAvailability; 28 | tera::master::MasterEnv().Init( 29 | new tera::master::MasterImpl(nullptr, nullptr), nullptr, nullptr, nullptr, nullptr, nullptr, 30 | nullptr, nullptr, nullptr, 31 | std::shared_ptr(new TabletAvailability(nullptr)), nullptr); 32 | 33 | return RUN_ALL_TESTS(); 34 | } 35 | 36 | /* vim: set expandtab ts=4 sw=4 sts=4 tw=100: */ 37 | -------------------------------------------------------------------------------- /src/load_balancer/test/random_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #include "gflags/gflags.h" 6 | #include "glog/logging.h" 7 | #include "gtest/gtest.h" 8 | 9 | #include "load_balancer/random.h" 10 | 11 | namespace tera { 12 | namespace load_balancer { 13 | 14 | class RandomTest : public ::testing::Test {}; 15 | 16 | TEST_F(RandomTest, CommonTest) { 17 | int start = 0; 18 | int end = 3; 19 | size_t times = 100; 20 | 21 | for (size_t i = 0; i < times; ++i) { 22 | int rand = Random::Rand(start, end); 23 | ASSERT_TRUE(rand >= start); 24 | ASSERT_TRUE(rand < end); 25 | } 26 | } 27 | 28 | TEST_F(RandomTest, NegativeTest) { 29 | int start = -10; 30 | int end = 10; 31 | size_t times = 100; 32 | 33 | for (size_t i = 0; i < times; ++i) { 34 | int rand = Random::RandStd(start, end); 35 | ASSERT_TRUE(rand >= start); 36 | ASSERT_TRUE(rand < end); 37 | } 38 | } 39 | 40 | } // namespace load_balancer 41 | } // namespace tera 42 | 43 | /* vim: set expandtab ts=4 sw=4 sts=4 tw=100: */ 44 | -------------------------------------------------------------------------------- /src/master/table_state_machine.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | #include "proto/status_code.pb.h" 10 | #include "master/state_machine.h" 11 | #include "master/tablet_state_machine.h" 12 | 13 | namespace tera { 14 | namespace master { 15 | 16 | enum class TableEvent { 17 | kEnableTable, 18 | kDisableTable, 19 | kDeleteTable, 20 | }; 21 | 22 | std::ostream& operator<<(std::ostream& o, const TableEvent event); 23 | 24 | class TableStateMachine { 25 | public: 26 | TableStateMachine(TableStatus init_status) : curr_status_(init_status) {} 27 | ~TableStateMachine() {} 28 | 29 | bool DoStateTransition(const TableEvent event) { 30 | TableStatus post_status; 31 | if (state_transitions_.DoStateTransition(curr_status_, event, &post_status)) { 32 | curr_status_ = post_status; 33 | return true; 34 | } 35 | return false; 36 | }; 37 | 38 | TableStatus GetStatus() { return curr_status_; } 39 | void SetStatus(TableStatus status) { curr_status_ = status; } 40 | 41 | typedef StateTransitionRules TableStateTransitionRulesType; 42 | 43 | private: 44 | TableStatus curr_status_; 45 | const static TableStateTransitionRulesType state_transitions_; 46 | }; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/master/test/master_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #include "gflags/gflags.h" 6 | #include "glog/logging.h" 7 | #include "gtest/gtest.h" 8 | 9 | #include "utils/utils_cmd.h" 10 | 11 | DECLARE_string(tera_leveldb_env_type); 12 | 13 | int main(int argc, char** argv) { 14 | ::google::ParseCommandLineFlags(&argc, &argv, true); 15 | ::google::InitGoogleLogging(argv[0]); 16 | tera::utils::SetupLog("master_test"); 17 | FLAGS_tera_leveldb_env_type = "local"; 18 | ::testing::InitGoogleTest(&argc, argv); 19 | 20 | return RUN_ALL_TESTS(); 21 | } 22 | -------------------------------------------------------------------------------- /src/master/test/mock_master_zk_adapter.h: -------------------------------------------------------------------------------- 1 | #include "master/master_zk_adapter.h" 2 | 3 | namespace tera { 4 | namespace master { 5 | namespace test { 6 | 7 | class TestZkAdapter : public MasterZkAdapter { 8 | public: 9 | TestZkAdapter() : MasterZkAdapter(nullptr, std::string("")) {} 10 | virtual bool UpdateRootTabletNode(const std::string& addr) { return true; } 11 | 12 | bool MarkSafeMode() { return true; } 13 | }; 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/master/update_auth_procedure.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #include "master/update_auth_procedure.h" 6 | 7 | namespace tera { 8 | namespace master { 9 | 10 | std::ostream& operator<<(std::ostream& o, const UpdateAuthPhase& phase) { 11 | static const char* msg[] = {"UpdateAuthPhase::kUpdateMeta", "UpdateAuthPhase::kEofPhase", 12 | "UpdateAuthPhase::kUnknown"}; 13 | static uint32_t msg_size = sizeof(msg) / sizeof(const char*); 14 | using UnderType = std::underlying_type::type; 15 | uint32_t index = 16 | static_cast(phase) - static_cast(UpdateAuthPhase::kUpdateMeta); 17 | index = index < msg_size ? index : msg_size - 1; 18 | o << msg[index]; 19 | return o; 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/observer/executor/key_selector.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2017, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_OBSERVER_EXECUTOR_KEY_SELECTOR_H_ 6 | #define TERA_OBSERVER_EXECUTOR_KEY_SELECTOR_H_ 7 | 8 | #include 9 | #include 10 | 11 | #include "tera.h" 12 | 13 | namespace tera { 14 | namespace observer { 15 | 16 | class KeySelector { 17 | public: 18 | virtual ~KeySelector() {} 19 | 20 | // output: selected table name, selected start key, end_key 21 | // the range of [start_key, end_key) 22 | virtual bool SelectRange(std::string* table_name, std::string* start_key, 23 | std::string* end_key) = 0; 24 | 25 | virtual ErrorCode Observe(const std::string& table_name) = 0; 26 | }; 27 | 28 | } // namespace observer 29 | } // namespace tera 30 | 31 | #endif // TERA_OBSERVER_EXECUTOR_KEY_SELECTOR_H_ 32 | -------------------------------------------------------------------------------- /src/observer/executor/random_key_selector.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2017, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_OBSERVER_EXECUTOR_RANDOM_KEY_SELECTOR_H_ 6 | #define TERA_OBSERVER_EXECUTOR_RANDOM_KEY_SELECTOR_H_ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include "common/mutex.h" 15 | #include "observer/executor/key_selector.h" 16 | #include "tera.h" 17 | 18 | namespace tera { 19 | namespace observer { 20 | 21 | class RandomKeySelector : public KeySelector { 22 | public: 23 | RandomKeySelector(); 24 | virtual ~RandomKeySelector(); 25 | 26 | virtual bool SelectRange(std::string* table_name, std::string* start_key, std::string* end_key); 27 | 28 | virtual ErrorCode Observe(const std::string& table_name); 29 | 30 | private: 31 | void Update(); 32 | 33 | private: 34 | tera::Client* client_; 35 | mutable Mutex table_mutex_; 36 | std::vector observe_tables_; 37 | std::shared_ptr>> tables_; 38 | std::thread update_thread_; 39 | 40 | mutable Mutex quit_mutex_; 41 | bool quit_; 42 | common::CondVar cond_; 43 | }; 44 | 45 | } // namespace observer 46 | } // namespace tera 47 | 48 | #endif // TERA_OBSERVER_EXECUTOR_RANDOM_KEY_SELECTOR_H_ 49 | -------------------------------------------------------------------------------- /src/observer/executor/tablet_bucket_key_selector.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2017, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include "common/mutex.h" 13 | #include "observer/executor/key_selector.h" 14 | #include "tera.h" 15 | 16 | namespace tera { 17 | namespace observer { 18 | 19 | class TabletBucketKeySelector : public KeySelector { 20 | public: 21 | TabletBucketKeySelector(int32_t bucket_id, int32_t bucket_cnt); 22 | virtual ~TabletBucketKeySelector(); 23 | 24 | virtual bool SelectRange(std::string* table_name, std::string* start_key, std::string* end_key); 25 | 26 | virtual ErrorCode Observe(const std::string& table_name); 27 | 28 | private: 29 | tera::Client* client_; 30 | std::vector observe_tables_; 31 | std::shared_ptr>> tables_; 32 | int32_t bucket_id_; 33 | int32_t bucket_cnt_; 34 | }; 35 | 36 | } // namespace observer 37 | } // namespace tera 38 | -------------------------------------------------------------------------------- /src/observer/rowlocknode/remote_rowlocknode.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2017, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #include "observer/rowlocknode/remote_rowlocknode.h" 6 | 7 | #include "gflags/gflags.h" 8 | 9 | DECLARE_int32(rowlock_thread_max_num); 10 | 11 | namespace tera { 12 | namespace observer { 13 | 14 | RemoteRowlockNode::RemoteRowlockNode(RowlockNodeImpl* rowlocknode_impl) 15 | : rowlocknode_impl_(rowlocknode_impl) {} 16 | 17 | RemoteRowlockNode::~RemoteRowlockNode() {} 18 | 19 | void RemoteRowlockNode::Lock(google::protobuf::RpcController* controller, 20 | const RowlockRequest* request, RowlockResponse* response, 21 | google::protobuf::Closure* done) { 22 | rowlocknode_impl_->TryLock(request, response, done); 23 | } 24 | 25 | void RemoteRowlockNode::UnLock(google::protobuf::RpcController* controller, 26 | const RowlockRequest* request, RowlockResponse* response, 27 | google::protobuf::Closure* done) { 28 | rowlocknode_impl_->UnLock(request, response, done); 29 | } 30 | 31 | } // namespace observer 32 | } // namespace tera 33 | -------------------------------------------------------------------------------- /src/observer/rowlocknode/remote_rowlocknode.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2017, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_OBSERVER_ROWLOCKNODE_REMOTE_ROWLOCKNODE_H_ 6 | #define TERA_OBSERVER_ROWLOCKNODE_REMOTE_ROWLOCKNODE_H_ 7 | 8 | #include "common/base/scoped_ptr.h" 9 | #include "common/thread_pool.h" 10 | #include "observer/rowlocknode/rowlocknode_impl.h" 11 | 12 | namespace tera { 13 | namespace observer { 14 | 15 | class RemoteRowlockNode : public RowlockService { 16 | public: 17 | explicit RemoteRowlockNode(RowlockNodeImpl* rowlocknode_impl); 18 | ~RemoteRowlockNode(); 19 | 20 | void Lock(google::protobuf::RpcController* controller, const RowlockRequest* request, 21 | RowlockResponse* response, google::protobuf::Closure* done); 22 | 23 | void UnLock(google::protobuf::RpcController* controller, const RowlockRequest* request, 24 | RowlockResponse* response, google::protobuf::Closure* done); 25 | 26 | private: 27 | RowlockNodeImpl* rowlocknode_impl_; 28 | }; 29 | 30 | } // namespace observer 31 | } // namespace tera 32 | #endif // TERA_OBSERVER_ROWLOCKNODE_REMOTE_ROWLOCKNODE_H_ 33 | -------------------------------------------------------------------------------- /src/observer/rowlocknode/rowlocknode_entry.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2017, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_OBSERVER_ROWLOCKNODE_ROWLOCKNODE_ENTRY_H_ 6 | #define TERA_OBSERVER_ROWLOCKNODE_ROWLOCKNODE_ENTRY_H_ 7 | 8 | #include 9 | 10 | #include "common/base/scoped_ptr.h" 11 | #include "observer/rowlocknode/remote_rowlocknode.h" 12 | #include "observer/rowlocknode/rowlocknode_impl.h" 13 | #include "tera/tera_entry.h" 14 | 15 | namespace tera { 16 | namespace observer { 17 | 18 | class RowlockNodeEntry : public tera::TeraEntry { 19 | public: 20 | RowlockNodeEntry(); 21 | virtual ~RowlockNodeEntry(); 22 | 23 | virtual bool StartServer(); 24 | virtual bool Run(); 25 | virtual void ShutdownServer(); 26 | void SetProcessorAffinity(); 27 | 28 | private: 29 | common::Mutex mutex_; 30 | 31 | scoped_ptr rowlocknode_impl_; 32 | RemoteRowlockNode* remote_rowlocknode_; 33 | scoped_ptr rpc_server_; 34 | }; 35 | 36 | } // namespace observer 37 | } // namespace tera 38 | 39 | #endif // TERA_OBSERVER_ROWLOCKNODE_ROWLOCKNODE_ENTRY_H_ 40 | -------------------------------------------------------------------------------- /src/observer/rowlocknode/rowlocknode_zk_adapter_base.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2017, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | // 5 | #ifndef TERA_OBSERVER_ROWLOCKNODE_ROWLOCKNODE_ZK_ADAPTER_BASE_H_ 6 | #define TERA_OBSERVER_ROWLOCKNODE_ROWLOCKNODE_ZK_ADAPTER_BASE_H_ 7 | 8 | #include "zk/zk_adapter.h" 9 | 10 | namespace tera { 11 | namespace observer { 12 | 13 | class RowlockNodeZkAdapterBase : public tera::zk::ZooKeeperAdapter { 14 | public: 15 | virtual ~RowlockNodeZkAdapterBase() {} 16 | virtual void Init() = 0; 17 | }; 18 | 19 | } // namespace observer 20 | } // namespace tera 21 | #endif // TERA_OBSERVER_ROWLOCKNODE_ROWLOCKNODE_ZK_ADAPTER_BASE_H_ 22 | -------------------------------------------------------------------------------- /src/observer/rowlockproxy/remote_rowlock_proxy.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2017, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #include "observer/rowlockproxy/remote_rowlock_proxy.h" 6 | 7 | #include "gflags/gflags.h" 8 | 9 | DECLARE_int32(rowlock_thread_max_num); 10 | 11 | namespace tera { 12 | namespace observer { 13 | 14 | RemoteRowlockProxy::RemoteRowlockProxy(RowlockProxyImpl* rowlock_proxy_impl) 15 | : rowlock_proxy_impl_(rowlock_proxy_impl) {} 16 | 17 | RemoteRowlockProxy::~RemoteRowlockProxy() {} 18 | 19 | void RemoteRowlockProxy::Lock(google::protobuf::RpcController* controller, 20 | const RowlockRequest* request, RowlockResponse* response, 21 | google::protobuf::Closure* done) { 22 | rowlock_proxy_impl_->TryLock(request, response, done); 23 | } 24 | 25 | void RemoteRowlockProxy::UnLock(google::protobuf::RpcController* controller, 26 | const RowlockRequest* request, RowlockResponse* response, 27 | google::protobuf::Closure* done) { 28 | rowlock_proxy_impl_->UnLock(request, response, done); 29 | } 30 | 31 | } // namespace observer 32 | } // namespace tera 33 | -------------------------------------------------------------------------------- /src/observer/rowlockproxy/remote_rowlock_proxy.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2017, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_OBSERVER_ROWLOCKPROXY_REMOTE_ROWLOCK_PROXY_H_ 6 | #define TERA_OBSERVER_ROWLOCKPROXY_REMOTE_ROWLOCK_PROXY_H_ 7 | 8 | #include 9 | 10 | #include "common/base/scoped_ptr.h" 11 | #include "common/thread_pool.h" 12 | #include "observer/rowlockproxy/rowlock_proxy_impl.h" 13 | 14 | namespace tera { 15 | namespace observer { 16 | 17 | class RemoteRowlockProxy : public RowlockService { 18 | public: 19 | explicit RemoteRowlockProxy(RowlockProxyImpl* rowlock_proxy_impl); 20 | ~RemoteRowlockProxy(); 21 | 22 | void Lock(google::protobuf::RpcController* controller, const RowlockRequest* request, 23 | RowlockResponse* response, google::protobuf::Closure* done); 24 | 25 | void UnLock(google::protobuf::RpcController* controller, const RowlockRequest* request, 26 | RowlockResponse* response, google::protobuf::Closure* done); 27 | 28 | private: 29 | RowlockProxyImpl* rowlock_proxy_impl_; 30 | }; 31 | 32 | } // namespace observer 33 | } // namespace tera 34 | #endif // TERA_OBSERVER_ROWLOCKPROXY_REMOTE_ROWLOCK_PROXY_H_ 35 | -------------------------------------------------------------------------------- /src/observer/rowlockproxy/rowlock_proxy_entry.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2017, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_OBSERVER_ROWLOCKPROXY_ROWLOCK_PROXY_ENTRY_H_ 6 | #define TERA_OBSERVER_ROWLOCKPROXY_ROWLOCK_PROXY_ENTRY_H_ 7 | 8 | #include 9 | 10 | #include 11 | 12 | #include "observer/rowlockproxy/remote_rowlock_proxy.h" 13 | #include "observer/rowlockproxy/rowlock_proxy_impl.h" 14 | #include "tera/tera_entry.h" 15 | 16 | namespace tera { 17 | namespace observer { 18 | 19 | class RowlockProxyEntry : public tera::TeraEntry { 20 | public: 21 | RowlockProxyEntry(); 22 | virtual ~RowlockProxyEntry(); 23 | 24 | virtual bool StartServer(); 25 | virtual bool Run(); 26 | virtual void ShutdownServer(); 27 | 28 | private: 29 | std::unique_ptr rowlock_proxy_impl_; 30 | RemoteRowlockProxy* remote_rowlock_proxy_; 31 | std::unique_ptr rpc_server_; 32 | }; 33 | 34 | } // namespace observer 35 | } // namespace tera 36 | 37 | #endif // TERA_OBSERVER_ROWLOCKPROXY_ROWLOCK_PROXY_ENTRY_H_ 38 | -------------------------------------------------------------------------------- /src/proto/lb_client.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #include 6 | 7 | #include "gflags/gflags.h" 8 | 9 | #include "proto/lb_client.h" 10 | 11 | DECLARE_int32(tera_master_connect_retry_times); 12 | DECLARE_int32(tera_master_connect_retry_period); 13 | DECLARE_int32(tera_master_connect_timeout_period); 14 | 15 | namespace tera { 16 | namespace load_balancer { 17 | 18 | LBClient::LBClient(const std::string& server_addr, int32_t rpc_timeout) 19 | : RpcClient(server_addr), rpc_timeout_(rpc_timeout) {} 20 | 21 | LBClient::~LBClient() {} 22 | 23 | bool LBClient::CmdCtrl(const CmdCtrlRequest* request, CmdCtrlResponse* response) { 24 | return SendMessageWithRetry( 25 | &LoadBalancerService::Stub::CmdCtrl, request, response, 26 | (std::function)NULL, "CmdCtrl", 27 | rpc_timeout_); 28 | } 29 | 30 | } // namespace load_balancer 31 | } // namespace tera 32 | -------------------------------------------------------------------------------- /src/proto/lb_client.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_LOAD_BALANCER_LB_CLIENT_H_ 6 | #define TERA_LOAD_BALANCER_LB_CLIENT_H_ 7 | 8 | #include 9 | 10 | #include "proto/load_balancer_rpc.pb.h" 11 | #include "proto/rpc_client.h" 12 | 13 | DECLARE_int32(tera_rpc_timeout_period); 14 | 15 | namespace tera { 16 | namespace load_balancer { 17 | 18 | class LBClient : public RpcClient { 19 | public: 20 | LBClient(const std::string& server_addr = "", 21 | int32_t rpc_timeout = FLAGS_tera_rpc_timeout_period); 22 | virtual ~LBClient(); 23 | 24 | virtual bool CmdCtrl(const CmdCtrlRequest* request, CmdCtrlResponse* response); 25 | 26 | private: 27 | int32_t rpc_timeout_; 28 | }; 29 | 30 | } // namespace load_balancer 31 | } // namespace tera 32 | 33 | #endif // TERA_LOAD_BALANCER_LB_CLIENT_H_ 34 | -------------------------------------------------------------------------------- /src/proto/load_balancer_rpc.proto: -------------------------------------------------------------------------------- 1 | import "sofa/pbrpc/rpc_option.proto"; 2 | import "master_rpc.proto"; 3 | 4 | package tera; 5 | 6 | service LoadBalancerService { 7 | rpc CmdCtrl(CmdCtrlRequest) returns(CmdCtrlResponse); 8 | } 9 | 10 | option cc_generic_services = true; 11 | 12 | -------------------------------------------------------------------------------- /src/proto/quota.proto: -------------------------------------------------------------------------------- 1 | package tera; 2 | 3 | import "table_meta.proto"; 4 | 5 | enum QuotaOperationType { 6 | kQuotaWriteReqs = 1; 7 | kQuotaWriteBytes = 2; 8 | kQuotaReadReqs = 3; 9 | kQuotaReadBytes = 4; 10 | kQuotaScanReqs = 5; 11 | kQuotaScanBytes = 6; 12 | } 13 | 14 | message QuotaInfo { 15 | required QuotaOperationType type = 1; 16 | required int64 limit = 2; 17 | // default 1s 18 | required int64 period = 3 [default = 1]; 19 | } 20 | 21 | message TableQuota { 22 | enum TableQuotaType { 23 | kSetQuota = 1; 24 | kDelQuota = 2; // when type == kDelQuota, doesn't need quota_infos. 25 | } 26 | required string table_name = 1; 27 | repeated QuotaInfo quota_infos = 2; 28 | required TableQuotaType type = 3 [default = kSetQuota]; 29 | } 30 | 31 | message TsQuota { 32 | required string ts_addr = 1; 33 | repeated TableQuota table_quotas = 2; 34 | } 35 | -------------------------------------------------------------------------------- /src/proto/rowlocknode_rpc.proto: -------------------------------------------------------------------------------- 1 | import "sofa/pbrpc/rpc_option.proto"; 2 | import "status_code.proto"; 3 | 4 | package tera; 5 | 6 | message RowlockRequest { 7 | required string table_name = 1; 8 | required string row = 2; 9 | } 10 | 11 | message RowlockResponse { 12 | required StatusCode lock_status = 1; 13 | } 14 | 15 | service RowlockService { 16 | rpc Lock(RowlockRequest) returns(RowlockResponse); 17 | rpc UnLock(RowlockRequest) returns(RowlockResponse); 18 | } 19 | option cc_generic_services = true; 20 | -------------------------------------------------------------------------------- /src/proto/rpc_client.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #include "proto/rpc_client.h" 6 | 7 | namespace tera { 8 | 9 | sofa::pbrpc::RpcChannelOptions RpcClientBase::channel_options_; 10 | std::map RpcClientBase::rpc_channel_list_; 11 | sofa::pbrpc::RpcClientOptions RpcClientBase::rpc_client_options_; 12 | sofa::pbrpc::RpcClient RpcClientBase::rpc_client_; 13 | Mutex RpcClientBase::mutex_; 14 | 15 | } // namespace tera 16 | -------------------------------------------------------------------------------- /src/proto/stat_table.proto: -------------------------------------------------------------------------------- 1 | import "status_code.proto"; 2 | import "tabletnode_rpc.proto"; 3 | 4 | package tera; 5 | 6 | message TabletLoadContext { 7 | required LoadTabletRequest load_request = 1; 8 | required bytes tabletnode_session_id = 2; 9 | } 10 | 11 | // tablet loading and compaction corruption message 12 | message TabletCorruptMessage { 13 | required bytes tablet = 1; 14 | required bytes tabletnode = 2; 15 | required uint32 corrupt_phase = 3; 16 | optional uint32 corrupt_type = 4; 17 | optional bytes locality_group = 5; 18 | optional bytes detail_message = 6; 19 | optional bytes load_context = 7; 20 | } 21 | -------------------------------------------------------------------------------- /src/proto/test_helper.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_PROTO_TEST_HELPER_H_ 6 | #define TERA_PROTO_TEST_HELPER_H_ 7 | 8 | #include "proto/table_schema.pb.h" 9 | 10 | namespace tera { 11 | 12 | TableSchema DefaultTableSchema(); 13 | 14 | ColumnFamilySchema DefaultCFSchema(const std::string& lg_name, uint32_t id); 15 | 16 | LocalityGroupSchema DefaultLGSchema(uint32_t id); 17 | 18 | } // namespace tera 19 | 20 | #endif // TERA_PROTO_TEST_HELPER_H_ 21 | -------------------------------------------------------------------------------- /src/proto/timeoracle_rpc.proto: -------------------------------------------------------------------------------- 1 | import "sofa/pbrpc/rpc_option.proto"; 2 | import "status_code.proto"; 3 | 4 | package tera; 5 | 6 | message GetTimestampRequest { 7 | optional uint64 count = 1; 8 | } 9 | 10 | message GetTimestampResponse { 11 | optional StatusCode status = 1; 12 | optional int64 start_timestamp = 2; 13 | optional uint64 count = 3; 14 | } 15 | 16 | service TimeoracleServer { 17 | rpc GetTimestamp(GetTimestampRequest) returns(GetTimestampResponse); 18 | } 19 | 20 | option cc_generic_services = true; 21 | -------------------------------------------------------------------------------- /src/quota/helpers/master_quota_helper.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #pragma once 6 | 7 | #include 8 | #include "proto/quota.pb.h" 9 | 10 | namespace tera { 11 | 12 | namespace master { 13 | struct MetaWriteRecord; 14 | } 15 | 16 | namespace quota { 17 | 18 | class MasterQuotaHelper { 19 | public: 20 | // New**() func will new a instance in heap, it's memory should delete by user. 21 | static master::MetaWriteRecord* NewMetaRecordFromQuota(const TableQuota& table_quota); 22 | static TableQuota* NewTableQuotaFromMetaValue(const std::string& value); 23 | 24 | static void PackDeleteQuotaRecords(const std::string& table_name, 25 | std::vector& records); 26 | 27 | static std::string GetTableNameFromMetaKey(const std::string& key); 28 | 29 | // Merge table_quota to target_table_quota, 30 | // It'll be incremental merge, that means quota update incremental 31 | static bool MergeTableQuota(const TableQuota& table_quota, TableQuota* target_table_quota); 32 | 33 | // default value for quota info, period = 1, limit = -1 34 | static void SetDefaultQuotaInfo(QuotaInfo* quota_info, QuotaOperationType type); 35 | }; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/quota/helpers/quota_utils.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #pragma once 6 | 7 | #include 8 | #include "proto/quota.pb.h" 9 | 10 | namespace tera { 11 | namespace quota { 12 | 13 | class QuotaUtils { 14 | public: 15 | static std::string DebugPrintTableQuota(const TableQuota& table_quota); 16 | static std::string GetQuotaOperation(QuotaOperationType type); 17 | }; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/quota/limiter/general_quota_limiter.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | #include "quota/limiter/quota_limiter.h" 10 | #include "quota/limiter/rate_limiter.h" 11 | #include "common/rwmutex.h" 12 | 13 | namespace tera { 14 | namespace quota { 15 | 16 | using RateLimiterPtr = std::unique_ptr; 17 | 18 | // Cause of copy shared_ptr for read, 19 | // and new a Quotalimiter for write to swap, 20 | // so CheckAndConsume will happened after Reset and 21 | // won't have any conflict, DOESN'T NEED ANY synchronization primitive. 22 | class GeneralQuotaLimiter : public QuotaLimiter { 23 | public: 24 | explicit GeneralQuotaLimiter(const std::string& table_name); 25 | virtual ~GeneralQuotaLimiter() {} 26 | 27 | void Reset(const TableQuota& table_quota) override; 28 | 29 | // if quota limited, return false 30 | // otherwise, consume the quota and return true 31 | bool CheckAndConsume(const Throttle& throttle) override; 32 | 33 | private: 34 | std::string table_name_; 35 | std::map op_rate_limiters_; 36 | RWMutex rw_mutex_; 37 | }; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/quota/limiter/general_rate_limiter.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #pragma once 6 | 7 | #include 8 | #include "quota/limiter/rate_limiter.h" 9 | #include "common/counter.h" 10 | #include "common/metric/metric_counter.h" 11 | #include "proto/quota.pb.h" 12 | 13 | namespace tera { 14 | namespace quota { 15 | 16 | // thread safe, only int64_t type value change. Doesn't need lock 17 | class GeneralRateLimiter : public RateLimiter { 18 | public: 19 | explicit GeneralRateLimiter(const std::string& table_name, QuotaOperationType type); 20 | virtual ~GeneralRateLimiter() {} 21 | 22 | void Reset(int64_t limit, int64_t period_sec) override; 23 | 24 | // If reach the period of quota, will reset the avail_ to limit_ 25 | // then check user request amount greater then 26 | // avail_(means out of quota, return false) or not(retrun true). 27 | bool RefillAndCheck(int64_t amount) override; 28 | 29 | void Consume(int64_t amount) override; 30 | 31 | private: 32 | void RefillAvail(); 33 | 34 | private: 35 | std::string quota_type_; 36 | Counter limit_; 37 | Counter avail_; 38 | tera::MetricCounter limit_per_sec_; 39 | int64_t period_sec_; 40 | int64_t next_refill_ms_; 41 | std::string table_name_; 42 | }; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/quota/limiter/limiter_factory.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #pragma once 6 | 7 | #include 8 | #include "quota/limiter/general_quota_limiter.h" 9 | 10 | namespace tera { 11 | namespace quota { 12 | 13 | static const std::string general_quota_limiter_type = "general_quota_limiter"; 14 | 15 | class LimiterFactory { 16 | public: 17 | LimiterFactory() {} 18 | ~LimiterFactory() {} 19 | static QuotaLimiter* CreateQuotaLimiter(const std::string& limiter_type, 20 | const std::string& table_name) { 21 | if (general_quota_limiter_type == limiter_type) { 22 | return new GeneralQuotaLimiter(table_name); 23 | } else { 24 | LOG(ERROR) << "Not surport limit_type = " << limiter_type; 25 | return nullptr; 26 | } 27 | } 28 | }; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/quota/limiter/quota_limiter.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #pragma once 6 | 7 | #include "proto/quota.pb.h" 8 | 9 | namespace tera { 10 | namespace quota { 11 | 12 | struct Throttle { 13 | Throttle() 14 | : write_reqs(0), write_bytes(0), read_reqs(0), read_bytes(0), scan_reqs(0), scan_bytes(0) {} 15 | int64_t write_reqs; 16 | int64_t write_bytes; 17 | int64_t read_reqs; 18 | int64_t read_bytes; 19 | int64_t scan_reqs; 20 | int64_t scan_bytes; 21 | }; 22 | 23 | class QuotaLimiter { 24 | public: 25 | virtual void Reset(const TableQuota& table_quota) = 0; 26 | 27 | // if quota limited, return false 28 | // otherwise, consume the quota and return true 29 | virtual bool CheckAndConsume(const Throttle& throttle) = 0; 30 | }; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/quota/limiter/rate_limiter.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #pragma once 6 | 7 | namespace tera { 8 | namespace quota { 9 | 10 | class RateLimiter { 11 | public: 12 | // Reset the limit and period_sec 13 | virtual void Reset(int64_t limit, int64_t period_sec) = 0; 14 | 15 | // If reach the period of quota, will reset the avail_ to limit_ 16 | // then check user request amount greater then 17 | // avail_(means out of quota, return false) or not(retrun true). 18 | virtual bool RefillAndCheck(int64_t amount) = 0; 19 | 20 | // if io pass quota limiter, consume io amount 21 | virtual void Consume(int64_t amount) = 0; 22 | }; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/quota/quota_limiter_container.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | #include "common/rwmutex.h" 10 | #include "proto/quota.pb.h" 11 | #include "quota/limiter/quota_limiter.h" 12 | 13 | namespace tera { 14 | namespace quota { 15 | 16 | using QuotaLimiterPtr = std::shared_ptr; 17 | 18 | class QuotaLimiterContainer { 19 | public: 20 | explicit QuotaLimiterContainer() {} 21 | virtual ~QuotaLimiterContainer() {} 22 | 23 | bool GetTableLimiter(const std::string& table_name, QuotaLimiterPtr* limiter) const; 24 | 25 | bool ResetQuota(const TableQuota& table_quota); 26 | 27 | private: 28 | std::unordered_map table_quotas_; 29 | mutable RWMutex rw_mutex_; 30 | }; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/sample/Makefile: -------------------------------------------------------------------------------- 1 | include ../../depends.mk 2 | 3 | OPT ?= -g2 -Wall -Werror # (B) Debug mode, w/ full line-level debugging symbols 4 | 5 | CC = cc 6 | CXX = g++ 7 | 8 | SHARED_CFLAGS = -fPIC 9 | SHARED_LDFLAGS = -shared -Wl,-soname -Wl, 10 | 11 | INCPATH += -I../../include $(DEPS_INCPATH) 12 | CFLAGS += $(OPT) $(SHARED_CFLAGS) $(INCPATH) 13 | CXXFLAGS += -std=gnu++11 $(OPT) $(SHARED_CFLAGS) $(INCPATH) 14 | LDFLAGS += ../../build/lib/libtera.a $(DEPS_LDPATH) $(DEPS_LDFLAGS) -lpthread -lz 15 | 16 | SAMPLE_SRC := ./tera_sample.cc tera_row_txn_sample.cc atomic_sample.cc global_txn_async_sample.cc 17 | SAMPLE_OBJ := $(SAMPLE_SRC:.cc=.o) 18 | 19 | .PHONY: clean 20 | 21 | all: sample_demo tera_row_txn_sample atomic_sample global_txn_async_sample global_txn_sync_sample 22 | 23 | sample_demo: ./tera_sample.o 24 | $(CXX) -o $@ $^ $(LDFLAGS) 25 | 26 | tera_row_txn_sample: tera_row_txn_sample.o 27 | $(CXX) -o $@ $^ $(LDFLAGS) 28 | 29 | global_txn_async_sample: global_txn_async_sample.o 30 | $(CXX) -o $@ $^ $(LDFLAGS) 31 | 32 | global_txn_sync_sample: global_txn_sync_sample.o 33 | $(CXX) -o $@ $^ $(LDFLAGS) 34 | 35 | atomic_sample: atomic_sample.o 36 | $(CXX) -o $@ $^ $(LDFLAGS) 37 | 38 | $(SAMPLE_OBJ): %.o: %.cc 39 | $(CXX) $(CXXFLAGS) -c $< -o $@ 40 | 41 | clean: 42 | rm -f *.o 43 | rm -f ./sample_demo 44 | rm -f ./tera_row_txn_sample 45 | rm -f ./global_txn_async_sample 46 | rm -f ./global_txn_sync_sample 47 | rm -f ./atomic_sample 48 | 49 | -------------------------------------------------------------------------------- /src/sdk/cookie.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_SDK_COOKIE_H_ 6 | #define TERA_SDK_COOKIE_H_ 7 | 8 | #include 9 | 10 | #include "proto/table_meta.pb.h" 11 | 12 | namespace tera { 13 | namespace sdk { 14 | 15 | void DumpCookie(const std::string& cookie_file, const std::string& cookie_lock_file, 16 | const SdkCookie& cookie); 17 | bool RestoreCookie(const std::string& cookie_file, bool delete_broken_cookie_file, 18 | SdkCookie* cookie); 19 | 20 | // helper 21 | bool DumpCookieFile(const std::string& cookie_file); 22 | bool FindKeyInCookieFile(const std::string& cookie_file, const std::string& key); 23 | 24 | } // namespace sdk 25 | } // namespace tera 26 | 27 | #endif // TERA_SDK_COOKIE_H_ 28 | -------------------------------------------------------------------------------- /src/sdk/filter_list_base.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | // The Designs of Filter and related codes are inspired by hbase which is licensed under 6 | // Apache 2.0 License (found in the LICENSE.Apache file in the root directory). Please refer to 7 | // https://hbase.apache.org/2.0/apidocs/org/apache/hadoop/hbase/filter/Filter.html 8 | // to see more detailed design of hbase filter. 9 | 10 | #include 11 | #include 12 | #include "tera.h" 13 | #include "proto/filter.pb.h" 14 | #include "sdk/filter_list_base.h" 15 | #include "gflags/gflags.h" 16 | #include "glog/logging.h" 17 | 18 | namespace tera { 19 | namespace filter { 20 | 21 | FilterListBase::~FilterListBase() {} 22 | 23 | bool FilterListBase::IsEmpty() { return filters_.empty(); } 24 | 25 | const std::vector& FilterListBase::GetFilters() { return filters_; } 26 | 27 | } // namesapce filter 28 | } // namesapce tera 29 | -------------------------------------------------------------------------------- /src/sdk/filter_list_base.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | // The Designs of Filter and related codes are inspired by hbase which is licensed under 6 | // Apache 2.0 License (found in the LICENSE.Apache file in the root directory). Please refer to 7 | // https://hbase.apache.org/2.0/apidocs/org/apache/hadoop/hbase/filter/Filter.html 8 | // to see more detailed design of hbase filter. 9 | 10 | #pragma once 11 | 12 | #include 13 | #include 14 | #include 15 | #include "tera/filter.h" 16 | 17 | namespace tera { 18 | namespace filter { 19 | 20 | class FilterListBase : public FilterBase { 21 | public: 22 | virtual ~FilterListBase(); 23 | virtual void AddFilter(const FilterPtr& filter) = 0; 24 | const std::vector& GetFilters(); 25 | 26 | protected: 27 | bool IsEmpty(); 28 | 29 | protected: 30 | std::vector filters_; 31 | }; 32 | 33 | } // namesapce filter 34 | } // namesapce tera 35 | -------------------------------------------------------------------------------- /src/sdk/filter_list_with_and.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | // The Designs of Filter and related codes are inspired by hbase which is licensed under 6 | // Apache 2.0 License (found in the LICENSE.Apache file in the root directory). Please refer to 7 | // https://hbase.apache.org/2.0/apidocs/org/apache/hadoop/hbase/filter/Filter.html 8 | // to see more detailed design of hbase filter. 9 | 10 | #pragma once 11 | 12 | #include 13 | #include 14 | #include "sdk/filter_list_base.h" 15 | 16 | namespace tera { 17 | namespace filter { 18 | 19 | class FilterListWithAND : public FilterListBase { 20 | public: 21 | FilterListWithAND(); 22 | virtual ~FilterListWithAND(); 23 | void Reset(); 24 | virtual ReturnCode FilterCell(const std::string& column_family, 25 | const std::string& column_qualifier, const std::string& value); 26 | virtual bool FilterRow(); 27 | virtual void AddFilter(const FilterPtr& filter); 28 | 29 | private: 30 | ReturnCode MergeReturnCode(ReturnCode rc, ReturnCode localRC); 31 | }; 32 | 33 | } // namesapce filter 34 | } // namesapce tera 35 | -------------------------------------------------------------------------------- /src/sdk/filter_list_with_or.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | // The Designs of Filter and related codes are inspired by hbase which is licensed under 6 | // Apache 2.0 License (found in the LICENSE.Apache file in the root directory). Please refer to 7 | // https://hbase.apache.org/2.0/apidocs/org/apache/hadoop/hbase/filter/Filter.html 8 | // to see more detailed design of hbase filter. 9 | 10 | #pragma once 11 | 12 | #include 13 | #include 14 | #include "sdk/filter_list_base.h" 15 | 16 | namespace tera { 17 | namespace filter { 18 | 19 | class FilterListWithOR : public FilterListBase { 20 | public: 21 | FilterListWithOR(); 22 | virtual ~FilterListWithOR(); 23 | void Reset(); 24 | virtual ReturnCode FilterCell(const std::string& column_family, 25 | const std::string& column_qualifier, const std::string& value); 26 | virtual bool FilterRow(); 27 | virtual void AddFilter(const FilterPtr& filter); 28 | 29 | private: 30 | ReturnCode MergeReturnCode(ReturnCode rc, ReturnCode localRC); 31 | bool NeedFilter(ReturnCode prevRC); 32 | 33 | private: 34 | std::vector prev_filter_rc_list_; 35 | }; 36 | 37 | } // namesapce filter 38 | } // namesapce tera 39 | -------------------------------------------------------------------------------- /src/sdk/filter_utils.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_SDK_FILTER_UTILS_H_ 6 | #define TERA_SDK_FILTER_UTILS_H_ 7 | 8 | #include "proto/tabletnode_rpc.pb.h" 9 | #include "proto/filter.pb.h" 10 | #include "tera.h" 11 | 12 | using std::string; 13 | 14 | namespace tera { 15 | 16 | bool CheckFilterString(const string& filter_str); 17 | string RemoveInvisibleChar(const string& schema); 18 | 19 | bool TransBinCompOp(BinCompOp bin_comp_op, filter::CompareOperator* op); 20 | 21 | namespace filter { 22 | CompareType TransCompareOperator(CompareOperator op); 23 | CompareOperator TransCompareType(CompareType op); 24 | 25 | FilterDesc::FilterType TransFilterType(FilterType type); 26 | ComparatorDesc::ComparatorType TransComparatorType(ComparatorType type); 27 | 28 | FilterValueType TransIntegerValueType(IntegerValueType type); 29 | IntegerValueType TransFilterValueType(FilterValueType type); 30 | 31 | FilterListDesc::Operator TransFilterListOp(FilterList::Operator op); 32 | FilterList::Operator TransFilterListDescOp(FilterListDesc::Operator op); 33 | 34 | } // namespace filter 35 | } // namespace tera 36 | #endif // TERA_SDK_FILTER_UTILS_H_ 37 | -------------------------------------------------------------------------------- /src/sdk/java/native-src/gen_jni_headers.sh: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | 3 | classpath="../tera/target/classes" 4 | 5 | javah -classpath $classpath -o jni_tera_base.h com.baidu.tera.client.TeraBase 6 | javah -classpath $classpath -o jni_tera_client.h com.baidu.tera.client.TeraClientImpl 7 | javah -classpath $classpath -o jni_tera_table.h com.baidu.tera.client.TeraTableImpl 8 | javah -classpath $classpath -o jni_tera_mutation.h com.baidu.tera.client.TeraMutationImpl 9 | javah -classpath $classpath -o jni_tera_reader.h com.baidu.tera.client.TeraReaderImpl 10 | javah -classpath $classpath -o jni_tera_result.h com.baidu.tera.client.TeraResultImpl 11 | javah -classpath $classpath -o jni_tera_scan.h com.baidu.tera.client.TeraScanImpl 12 | javah -classpath $classpath -o jni_tera_result_stream.h com.baidu.tera.client.ScanResultStreamImpl 13 | -------------------------------------------------------------------------------- /src/sdk/java/native-src/jni_md.h: -------------------------------------------------------------------------------- 1 | /* 2 | * %W% %E% 3 | * 4 | * Copyright (c) 2006, Oracle and/or its affiliates. All rights reserved. 5 | * ORACLE PROPRIETARY/CONFIDENTIAL. Use is subject to license terms. 6 | */ 7 | 8 | #ifndef _JAVASOFT_JNI_MD_H_ 9 | #define _JAVASOFT_JNI_MD_H_ 10 | 11 | #define JNIEXPORT __attribute__((visibility("default"))) 12 | #define JNIIMPORT 13 | #define JNICALL 14 | 15 | #pragma GCC visibility push(default) 16 | 17 | typedef int jint; 18 | #ifdef _LP64 /* 64-bit Solaris */ 19 | typedef long jlong; 20 | #else 21 | typedef long long jlong; 22 | #endif 23 | 24 | typedef signed char jbyte; 25 | 26 | #pragma GCC visibility pop 27 | 28 | #endif /* !_JAVASOFT_JNI_MD_H_ */ 29 | -------------------------------------------------------------------------------- /src/sdk/java/native-src/jni_tera_base.h: -------------------------------------------------------------------------------- 1 | /* DO NOT EDIT THIS FILE - it is machine generated */ 2 | #include 3 | /* Header for class com_baidu_tera_client_TeraBase */ 4 | 5 | #ifndef _Included_com_baidu_tera_client_TeraBase 6 | #define _Included_com_baidu_tera_client_TeraBase 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | /* 11 | * Class: com_baidu_tera_client_TeraBase 12 | * Method: nativeInitGlog 13 | * Signature: (Ljava/lang/String;)V 14 | */ 15 | JNIEXPORT void JNICALL 16 | Java_com_baidu_tera_client_TeraBase_nativeInitGlog(JNIEnv *, jobject, jstring); 17 | 18 | /* 19 | * Class: com_baidu_tera_client_TeraBase 20 | * Method: nativeGlog 21 | * Signature: (Ljava/lang/String;Ljava/lang/String;)V 22 | */ 23 | JNIEXPORT void JNICALL 24 | Java_com_baidu_tera_client_TeraBase_nativeGlog(JNIEnv *, jobject, jstring, jstring); 25 | 26 | /* 27 | * Class: com_baidu_tera_client_TeraBase 28 | * Method: nativeVlog 29 | * Signature: (JLjava/lang/String;)V 30 | */ 31 | JNIEXPORT void JNICALL 32 | Java_com_baidu_tera_client_TeraBase_nativeVlog(JNIEnv *, jobject, jlong, jstring); 33 | 34 | #ifdef __cplusplus 35 | } 36 | #endif 37 | #endif 38 | -------------------------------------------------------------------------------- /src/sdk/java/native-src/jni_tera_common.h: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2014, Baidu Inc. 2 | // Author: Xu Peilin (xupeilin@baidu.com) 3 | // 4 | // Description: 5 | // 6 | 7 | #ifndef _JAVATERA_NATIVE_SRC_JNI_TERA_COMMON_H_ 8 | #define _JAVATERA_NATIVE_SRC_JNI_TERA_COMMON_H_ 9 | 10 | #include 11 | 12 | #include 13 | #include 14 | 15 | #include "tera.h" 16 | 17 | void SendErrorJ(JNIEnv* env, jobject jobj, std::string msg); 18 | 19 | void InitFlags(std::string confpath); 20 | 21 | void InitGlog(std::string prefix); 22 | 23 | std::string ConvertDescToString(tera::TableDescriptor* desc); 24 | 25 | void JByteArrayToString(JNIEnv* env, jbyteArray& jbarray, std::string* str); 26 | 27 | void StringToJByteArray(JNIEnv* env, const std::string& str, jbyteArray* jbarray); 28 | 29 | #endif // _JAVATERA_NATIVE_SRC_JNI_TERA_COMMON_H_ 30 | -------------------------------------------------------------------------------- /src/sdk/java/native-src/jni_tera_reader.h: -------------------------------------------------------------------------------- 1 | /* DO NOT EDIT THIS FILE - it is machine generated */ 2 | #include 3 | /* Header for class com_baidu_tera_client_TeraReaderImpl */ 4 | 5 | #ifndef _Included_com_baidu_tera_client_TeraReaderImpl 6 | #define _Included_com_baidu_tera_client_TeraReaderImpl 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | /* 11 | * Class: com_baidu_tera_client_TeraReaderImpl 12 | * Method: nativeAddColumn 13 | * Signature: (J[B[B)Z 14 | */ 15 | JNIEXPORT jboolean JNICALL 16 | Java_com_baidu_tera_client_TeraReaderImpl_nativeAddColumn(JNIEnv *, jobject, jlong, jbyteArray, 17 | jbyteArray); 18 | 19 | /* 20 | * Class: com_baidu_tera_client_TeraReaderImpl 21 | * Method: nativeAddFamily 22 | * Signature: (J[B)Z 23 | */ 24 | JNIEXPORT jboolean JNICALL 25 | Java_com_baidu_tera_client_TeraReaderImpl_nativeAddFamily(JNIEnv *, jobject, jlong, jbyteArray); 26 | 27 | /* 28 | * Class: com_baidu_tera_client_TeraReaderImpl 29 | * Method: nativeDeleteReader 30 | * Signature: (J)Z 31 | */ 32 | JNIEXPORT jboolean JNICALL 33 | Java_com_baidu_tera_client_TeraReaderImpl_nativeDeleteReader(JNIEnv *, jobject, jlong); 34 | 35 | #ifdef __cplusplus 36 | } 37 | #endif 38 | #endif 39 | -------------------------------------------------------------------------------- /src/sdk/java/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | com.baidu.tera 6 | javatera 7 | 1.0-SNAPSHOT 8 | pom 9 | 10 | tera 11 | 12 | 13 | 14 | UTF-8 15 | 16 | 17 | 18 | 19 | junit 20 | junit-dep 21 | 4.8.2 22 | test 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /src/sdk/java/tera/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | 6 | com.baidu.tera 7 | javatera 8 | 1.0-SNAPSHOT 9 | 10 | tera 11 | 12 | 13 | UTF-8 14 | 15 | 16 | -------------------------------------------------------------------------------- /src/sdk/java/tera/src/main/java/com/baidu/tera/client/TeraCell.java: -------------------------------------------------------------------------------- 1 | package com.baidu.tera.client; 2 | 3 | public class TeraCell { 4 | private int status = -1; 5 | private String rowKey; 6 | private String family; 7 | private String qualifier; 8 | private String value; 9 | private long timeStamp; 10 | 11 | public TeraCell(String rowKey, String family, String qualifier, String value, long timeStamp) { 12 | this.rowKey = rowKey; 13 | this.family = family; 14 | this.qualifier = qualifier; 15 | this.value = value; 16 | this.timeStamp = timeStamp; 17 | status = 0; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/sdk/java/tera/src/main/java/com/baidu/tera/client/TeraReaderImpl.java: -------------------------------------------------------------------------------- 1 | package com.baidu.tera.client; 2 | 3 | public class TeraReaderImpl { 4 | private native boolean nativeAddColumn(long teraMutationPointer, byte[] family, byte[] column); 5 | private native boolean nativeAddFamily(long teraMutationPointer, byte[] family); 6 | private native boolean nativeDeleteReader(long nativeTeraMutationPointer); 7 | private long nativeReaderPointer; 8 | 9 | private byte[] rowKey; 10 | 11 | public TeraReaderImpl(long nativeTeraReaderPointer, byte[] rowKey) { 12 | this.nativeReaderPointer = nativeTeraReaderPointer; 13 | this.rowKey = rowKey; 14 | } 15 | 16 | public boolean add(byte[] family, byte[] column) { 17 | if (column == null) { 18 | return nativeAddFamily(nativeReaderPointer, family); 19 | } else { 20 | return nativeAddColumn(nativeReaderPointer, family, column); 21 | } 22 | } 23 | 24 | public long getNativeReaderPointer() { 25 | return nativeReaderPointer; 26 | } 27 | 28 | public void finalize() throws Throwable { 29 | nativeDeleteReader(nativeReaderPointer); 30 | nativeReaderPointer = 0; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/sdk/java/tera/src/test/java/com/baidu/tera/client/TeraBaseTest.java: -------------------------------------------------------------------------------- 1 | package com.baidu.tera.client; 2 | 3 | import org.junit.Test; 4 | import static org.junit.Assert.*; 5 | 6 | /** 7 | * Unit test for TeraBase. 8 | */ 9 | public class TeraBaseTest 10 | { 11 | public TeraBaseTest() 12 | { 13 | 14 | } 15 | @Test 16 | public void testBasicFeature() 17 | { 18 | System.out.println("Now running " + Thread.currentThread().getStackTrace()[1].getMethodName() + " ..."); 19 | 20 | TeraBase teraBase = new TeraBase(); 21 | // teraBase.InitGlog("tera-base-test"); 22 | teraBase.LOG("INFO", "JAVA: LOG test"); 23 | teraBase.LOG_INFO("JAVA: LOG_INFO test"); 24 | teraBase.LOG_WARNING("JAVA: LOG_WARNING test"); 25 | // teraBase.LOG_ERROR("JAVA: LOG_ERROR test"); 26 | teraBase.VLOG(0, "JAVA: VLOG test level 0"); 27 | // System.out.println(teraBase.getNativeMessage()); 28 | assertTrue(true); 29 | } 30 | } -------------------------------------------------------------------------------- /src/sdk/python/checker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # 通过Python调用libtera_c.so导出的函数时，需要明确指定每个函数的参数类型， 4 | # 否则可能出现参数类型不兼容的问题，导致程序错误或core在libtera_c.so里， 5 | # Python无法自动判断调用C函数时该传递什么类型参数，所以需要程序员自己指定或说明。 6 | # 如果程序员忘记指定参数类型，Python也不会报错，所以只能通过一个外部脚本去检测， 7 | # 保证每个导出的C函数都在Python代码里明确指定了参数类型。 8 | 9 | # 例如： 10 | # tera_c.h导出 11 | # tera_client_t* tera_client_open(const char* conf_path, const char* log_prefix, char** err); 12 | # 13 | # Python代码里就应该对应 14 | # lib.tera_client_open.argtypes = [c_char_p, c_char_p, POINTER(c_char_p)] 15 | # lib.tera_client_open.restype = c_void_p 16 | 17 | grep "tera_.*(" include/tera_c.h | awk '{print $2}' | awk -F'(' '{print $1}' | while read afunc 18 | do 19 | if grep -q "lib.$afunc.argtypes " src/sdk/python/TeraSdk.py && grep -q "lib.$afunc.restype " src/sdk/python/TeraSdk.py 20 | then 21 | continue 22 | fi 23 | echo "maybe forget to specify the required argument/return type(s) for $afunc" 24 | exit 1 25 | done 26 | -------------------------------------------------------------------------------- /src/sdk/sdk_perf.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_SDK_SDK_PERF_H_ 6 | #define TERA_SDK_SDK_PERF_H_ 7 | 8 | #include 9 | 10 | #include "gflags/gflags.h" 11 | #include "glog/logging.h" 12 | 13 | #include "common/metric/metric_counter.h" 14 | #include "common/metric/collector_report.h" 15 | #include "common/this_thread.h" 16 | #include "tera.h" 17 | 18 | DECLARE_int32(tera_sdk_perf_collect_interval); 19 | 20 | namespace tera { 21 | namespace sdk { 22 | 23 | class PerfCollecter { 24 | public: 25 | PerfCollecter() : stopped_(false) {} 26 | ~PerfCollecter() {} 27 | 28 | void Run() { thread_ = std::thread{&PerfCollecter::ScheduleCollect, this}; } 29 | 30 | void Stop() { 31 | stopped_ = true; 32 | thread_.join(); 33 | } 34 | 35 | private: 36 | void ScheduleCollect() { 37 | while (!stopped_) { 38 | CollectorReportPublisher::GetInstance().Refresh(); 39 | DumpLog(); 40 | ThisThread::Sleep(FLAGS_tera_sdk_perf_collect_interval); 41 | } 42 | } 43 | 44 | void DumpLog(); 45 | 46 | private: 47 | std::thread thread_; 48 | bool stopped_; 49 | }; 50 | 51 | } // namespace sdk 52 | } // namespace tera 53 | 54 | #endif // TERA_SDK_SDK_PERF_H_ 55 | -------------------------------------------------------------------------------- /src/sdk/tera_easy.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | // 5 | // Author: yanshiguang02@baidu.com 6 | 7 | #ifndef TERA_TERA_EASY_H_ 8 | #define TERA_TERA_EASY_H_ 9 | 10 | #include 11 | 12 | #include 13 | #include 14 | 15 | #pragma GCC visibility push(default) 16 | 17 | namespace teraeasy { 18 | 19 | typedef std::string Key; 20 | typedef std::string ColumnKey; 21 | typedef std::string Value; 22 | typedef int64_t Timestamp; 23 | typedef std::map Column; 24 | typedef std::map Record; 25 | typedef std::pair KVPair; 26 | 27 | class Table { 28 | public: 29 | Table() {} 30 | 31 | virtual ~Table() {} 32 | 33 | virtual bool Read(const Key& row_key, Record* record) = 0; 34 | 35 | virtual bool Write(const Key& row_key, const Record& record) = 0; 36 | 37 | virtual void Flush() = 0; 38 | 39 | virtual bool Delete(const Key& row_key) = 0; 40 | 41 | virtual bool SetScanner(const Key& start, const Key& end) = 0; 42 | 43 | virtual bool NextPair(KVPair* kv_pair) = 0; 44 | 45 | private: 46 | Table(const Table&); 47 | void operator=(const Table&); 48 | }; 49 | 50 | Table* OpenTable(const std::string& table_name, const std::string& conf_path = ""); 51 | } 52 | 53 | #pragma GCC visibility pop 54 | 55 | #endif // TERA_TERA_EASY_H_ 56 | -------------------------------------------------------------------------------- /src/sdk/test/filter_utils_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #include "filter_utils.h" 6 | 7 | #include "gtest/gtest.h" 8 | 9 | namespace tera { 10 | 11 | void PrintBytes(const std::string& c, int n) { 12 | fprintf(stderr, "-------------------"); 13 | for (int i = 0; i < n; ++i) { 14 | fprintf(stderr, "%2x ", (unsigned char)c[i]); 15 | } 16 | fprintf(stderr, "\n"); 17 | } 18 | 19 | TEST(FilterUtils, RemoveInvisibleChar) { 20 | string schema = ""; 21 | schema = RemoveInvisibleChar(schema); 22 | EXPECT_TRUE(schema == ""); 23 | 24 | schema = " "; 25 | schema = RemoveInvisibleChar(schema); 26 | EXPECT_TRUE(schema == ""); 27 | 28 | schema = "a "; 29 | schema = RemoveInvisibleChar(schema); 30 | EXPECT_TRUE(schema == "a"); 31 | 32 | schema = "a\n \t "; 33 | schema = RemoveInvisibleChar(schema); 34 | EXPECT_TRUE(schema == "a"); 35 | } 36 | } // namespace tera 37 | -------------------------------------------------------------------------------- /src/sdk/test/global_txn_testutils.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_SDK_TEST_GLOBAL_TXN_TESTUTILS_H_ 6 | #define TERA_SDK_TEST_GLOBAL_TXN_TESTUTILS_H_ 7 | 8 | #include 9 | 10 | namespace tera { 11 | 12 | class GlobalTxnTestHelper { 13 | public: 14 | GlobalTxnTestHelper(const std::string& conffile); 15 | ~GlobalTxnTestHelper() {} 16 | int64_t GetStartTs(); 17 | int64_t GetPrewriteStartTs(); 18 | int64_t GetCommitTs(); 19 | void Wait(int64_t start_ts); 20 | void GetWait(int64_t start_ts); 21 | void LoadTxnConf(); 22 | 23 | private: 24 | void ExitNow(int64_t start_ts, int position); 25 | void PrintLog(int64_t start_ts, const std::string& log_str, int64_t next_wait_time = -1); 26 | int pos_; 27 | size_t get_pos_; 28 | std::string conf_file_; 29 | int64_t start_ts_; 30 | int64_t prewrite_start_ts_; 31 | int64_t commit_ts_; 32 | int64_t ts_[8]; 33 | std::vector get_ts_list_; 34 | int64_t helper_create_time_; 35 | }; 36 | 37 | } // namespace tera 38 | 39 | #endif // TERA_SDK_TEST_GLOBAL_TXN_TESTUTILS_H_ 40 | -------------------------------------------------------------------------------- /src/sdk/test/sdk_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #include "gflags/gflags.h" 6 | #include "glog/logging.h" 7 | #include "gtest/gtest.h" 8 | DECLARE_bool(tera_sdk_tso_client_enabled); 9 | DECLARE_bool(tera_sdk_client_for_gtxn); 10 | 11 | int main(int argc, char* argv[]) { 12 | FLAGS_tera_sdk_client_for_gtxn = true; 13 | FLAGS_tera_sdk_tso_client_enabled = false; 14 | ::testing::InitGoogleTest(&argc, argv); 15 | return RUN_ALL_TESTS(); 16 | } 17 | -------------------------------------------------------------------------------- /src/tabletnode/tabletnode_entry.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | // 5 | // Author: An Qin (qinan@baidu.com) 6 | 7 | #ifndef TERA_TABLETNODE_TABLETNODE_ENTRY_H_ 8 | #define TERA_TABLETNODE_TABLETNODE_ENTRY_H_ 9 | 10 | #include 11 | 12 | #include "common/base/scoped_ptr.h" 13 | #include "common/metric/metric_http_server.h" 14 | #include "tera/tera_entry.h" 15 | 16 | namespace tera { 17 | 18 | namespace master { 19 | class MasterClient; 20 | } 21 | 22 | namespace tabletnode { 23 | 24 | class TabletNodeImpl; 25 | class RemoteTabletNode; 26 | 27 | class TabletNodeEntry : public TeraEntry { 28 | public: 29 | TabletNodeEntry(); 30 | ~TabletNodeEntry(); 31 | 32 | bool StartServer(); 33 | bool Run(); 34 | void ShutdownServer(); 35 | 36 | void SetProcessorAffinity(); 37 | 38 | private: 39 | scoped_ptr tabletnode_impl_; 40 | RemoteTabletNode* remote_tabletnode_; 41 | scoped_ptr rpc_server_; 42 | scoped_ptr metric_http_server_; 43 | }; 44 | 45 | } // namespace tabletnode 46 | } // namespace tera 47 | 48 | #endif // TERA_TABLETNODE_TABLETNODE_ENTRY_H_ 49 | -------------------------------------------------------------------------------- /src/tabletnode/test/mock_zk_adapter.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_TABLETNODE_MOCK_ZK_ADAPTER_H_ 6 | #define TERA_TABLETNODE_MOCK_ZK_ADAPTER_H_ 7 | 8 | #include "zk/dump_zk_adapter.h" 9 | 10 | namespace tera { 11 | namespace tabletnode { 12 | 13 | class MockDummyNodeZkAdapter : public zk::DummyNodeZkAdapter { 14 | public: 15 | MOCK_METHOD3(OnChildrenChanged, 16 | void(const std::string& path, const std::vector& name_list, 17 | const std::vector& data_list)); 18 | MOCK_METHOD2(OnNodeValueChanged, void(const std::string& path, const std::string& value)); 19 | MOCK_METHOD1(OnNodeCreated, void(const std::string& path)); 20 | MOCK_METHOD1(OnNodeDeleted, void(const std::string& path)); 21 | MOCK_METHOD3(OnWatchFailed, void(const std::string& path, int watch_type, int err)); 22 | MOCK_METHOD1(OnSessionTimeout, void()); 23 | }; 24 | 25 | } // namespace tabletnode 26 | } // namespace tera 27 | 28 | #endif // TERA_TABLETNODE_MOCK_ZK_ADAPTER_H_ 29 | -------------------------------------------------------------------------------- /src/tabletnode/test/tabletnode_sysinfo_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #define private public 6 | 7 | #include "tabletnode_sysinfo.h" 8 | #include "common/timer.h" 9 | #include "gtest/gtest.h" 10 | 11 | namespace tera { 12 | namespace tabletnode { 13 | 14 | class TabletNodeSysInfoTest : public ::testing::Test, public TabletNodeSysInfo { 15 | public: 16 | TabletNodeSysInfoTest() {} 17 | ~TabletNodeSysInfoTest() {} 18 | }; 19 | 20 | TEST_F(TabletNodeSysInfoTest, CollectHardwareInfo) { 21 | int64_t ts = get_micros(); 22 | CollectHardwareInfo(); 23 | ts = get_micros() - ts; 24 | LOG(ERROR) << "cost: " << ts << " ms."; 25 | } 26 | 27 | TEST_F(TabletNodeSysInfoTest, ToString) { 28 | SetCurrentTime(); 29 | AddExtraInfo("read", 100); 30 | } 31 | } // namespace tabletnode 32 | } // namespace tera 33 | -------------------------------------------------------------------------------- /src/tera_main_wrapper.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include 12 | 13 | #include "version.h" 14 | 15 | DECLARE_string(tera_role); 16 | 17 | int main(int argc, char** argv) { 18 | if (argc > 1 && strcmp(argv[1], "version") == 0) { 19 | PrintSystemVersion(); 20 | return 0; 21 | } 22 | 23 | google::AllowCommandLineReparsing(); 24 | google::ParseCommandLineFlags(&argc, &argv, false); 25 | 26 | const char* program = NULL; 27 | if (FLAGS_tera_role == "master") { 28 | program = "./tera_master"; 29 | } else if (FLAGS_tera_role == "tabletnode") { 30 | program = "./tabletserver"; 31 | } else { 32 | std::cerr << "FLAGS_tera_role should be one of (master | tabletnode)" << std::endl; 33 | return -1; 34 | } 35 | 36 | std::vector myargv; 37 | myargv.resize(argc + 1); 38 | myargv[0] = (char*)"tera_main"; 39 | for (int i = 1; i < argc; i++) { 40 | myargv[i] = argv[i]; 41 | } 42 | myargv[argc] = NULL; 43 | if (-1 == execv(program, &myargv[0])) { 44 | std::cerr << "execv " << program << " error: " << errno << std::endl; 45 | return -1; 46 | } 47 | return 0; 48 | } 49 | -------------------------------------------------------------------------------- /src/timeoracle/timeoracle.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #include "timeoracle/timeoracle.h" 6 | 7 | namespace tera { 8 | namespace timeoracle { 9 | 10 | std::atomic Timeoracle::s_last_timestamp_ms; 11 | 12 | } // namespace timeoracle 13 | } // namespace tera 14 | -------------------------------------------------------------------------------- /src/timeoracle/timeoracle_entry.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_TIMEORACLE_TIMEORACLE_ENTRY_H_ 6 | #define TERA_TIMEORACLE_TIMEORACLE_ENTRY_H_ 7 | 8 | #include 9 | 10 | #include "tera/tera_entry.h" 11 | #include 12 | #include 13 | #include 14 | 15 | namespace tera { 16 | namespace timeoracle { 17 | 18 | class RemoteTimeoracle; 19 | class TimeoracleZkAdapterBase; 20 | 21 | class TimeoracleEntry : public TeraEntry { 22 | public: 23 | TimeoracleEntry(); 24 | ~TimeoracleEntry(); 25 | 26 | virtual bool Start() override; 27 | virtual bool Run() override; 28 | virtual void ShutdownServer() override; 29 | 30 | private: 31 | bool InitZKAdaptor(); 32 | bool StartServer(); 33 | void LeaseThread(); 34 | 35 | private: 36 | std::string local_addr_; 37 | RemoteTimeoracle* remote_timeoracle_; 38 | std::unique_ptr sofa_pbrpc_server_; 39 | int64_t startup_timestamp_; 40 | std::unique_ptr zk_adapter_; 41 | std::thread lease_thread_; 42 | std::atomic need_quit_; 43 | }; 44 | 45 | } // namespace timeoracle 46 | } // namespace tera 47 | 48 | #endif // TERA_TIMEORACLE_TIMEORACLE_ENTRY_H_ 49 | -------------------------------------------------------------------------------- /src/utils/cli_mt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidu/tera/dbcd28af792d879d961bf9fc7eb60de81b437646/src/utils/cli_mt -------------------------------------------------------------------------------- /src/utils/config_utils.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #include "utils/config_utils.h" 6 | 7 | #include "common/file/file_path.h" 8 | #include "gflags/gflags.h" 9 | 10 | namespace tera { 11 | namespace utils { 12 | 13 | bool LoadFlagFile(const std::string& file) { 14 | if (!IsExist(file)) { 15 | return false; 16 | } 17 | std::string flag = "--flagfile=" + file; 18 | int argc = 2; 19 | char** argv = new char* [3]; 20 | argv[0] = const_cast("dummy"); 21 | argv[1] = const_cast(flag.c_str()); 22 | argv[2] = NULL; 23 | ::google::ParseCommandLineFlags(&argc, &argv, false); 24 | argv[1] = NULL; 25 | delete[] argv; 26 | return true; 27 | } 28 | 29 | } // namespace utils 30 | } // namespace tera 31 | -------------------------------------------------------------------------------- /src/utils/config_utils.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_UTILS_CONFIG_UTILS_H_ 6 | #define TERA_UTILS_CONFIG_UTILS_H_ 7 | 8 | #include 9 | 10 | namespace tera { 11 | namespace utils { 12 | 13 | // `file' should be path/to/file, like "../conf/tera.flag" 14 | bool LoadFlagFile(const std::string& file); 15 | 16 | } // namespace utils 17 | } // namespace tera 18 | 19 | #endif // TERA_UTILS_CONFIG_UTILS_H_ 20 | -------------------------------------------------------------------------------- /src/utils/crypt.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_UTILS_CRYPT_H 6 | #define TERA_UTILS_CRYPT_H 7 | 8 | #include 9 | 10 | #include 11 | #include 12 | 13 | #define HASH_STRING_LEN 8 14 | 15 | namespace tera { 16 | 17 | // return 0: all is ok, result(hash number) stored at the location given by 18 | // @result; 19 | // otherwise: invalid arguments. 20 | int32_t GetHashNumber(const std::string& str, uint32_t seed, uint32_t* result); 21 | 22 | int32_t GetHashString(const std::string& str, uint32_t seed, std::string* result); 23 | 24 | } // namespace tera 25 | 26 | #endif // TERA_UTILS_CRYPT_H 27 | -------------------------------------------------------------------------------- /src/utils/fragment.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_FRAGMENT_UTIL_H_ 6 | #define TERA_FRAGMENT_UTIL_H_ 7 | 8 | #include 9 | #include 10 | 11 | namespace tera { 12 | 13 | class RangeFragment { 14 | public: 15 | // caller should use Lock to avoid data races 16 | // On success, return true. Otherwise, return false due to invalid argumetns 17 | bool AddToRange(const std::string& start, const std::string& end); 18 | 19 | bool IsCompleteRange() const; 20 | 21 | bool IsCoverRange(const std::string& start, const std::string& end) const; 22 | 23 | std::string DebugString() const; 24 | 25 | private: 26 | std::list > range_; 27 | }; 28 | 29 | } // namespace tera 30 | 31 | #endif // TERA_FRAGMENT_UTIL_H_ 32 | -------------------------------------------------------------------------------- /src/utils/libzookeeper_mt.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baidu/tera/dbcd28af792d879d961bf9fc7eb60de81b437646/src/utils/libzookeeper_mt.so -------------------------------------------------------------------------------- /src/utils/network_utils.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #include "utils/network_utils.h" 6 | 7 | namespace tera { 8 | namespace utils { 9 | 10 | std::string GetRemoteAddress(google::protobuf::RpcController* controller) { 11 | assert(controller != NULL); 12 | sofa::pbrpc::RpcController* cntl = static_cast(controller); 13 | return cntl->RemoteAddress(); 14 | } 15 | 16 | } // namespace utils 17 | } // namepsace tera 18 | -------------------------------------------------------------------------------- /src/utils/network_utils.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_UTILS_NETWORK_UTILS_H_ 6 | #define TERA_UTILS_NETWORK_UTILS_H_ 7 | 8 | #include 9 | 10 | #include "sofa/pbrpc/pbrpc.h" 11 | 12 | namespace tera { 13 | namespace utils { 14 | 15 | std::string GetRemoteAddress(google::protobuf::RpcController* controller); 16 | 17 | } // namespace utils 18 | } // namespace tera 19 | 20 | #endif // TERA_UTILS_NETWORK_UTILS_H_ 21 | -------------------------------------------------------------------------------- /src/utils/schema_utils.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #include "utils/schema_utils.h" 6 | 7 | #include 8 | 9 | #include 10 | 11 | namespace tera { 12 | 13 | bool IsSchemaCfDiff(const TableSchema& a, const TableSchema& b) { 14 | std::stringstream s0; 15 | std::stringstream s1; 16 | for (int i = 0; i < a.column_families_size(); ++i) { 17 | s0 << a.column_families(i).ShortDebugString(); 18 | } 19 | LOG(INFO) << "[utils] " << s0.str(); 20 | for (int i = 0; i < b.column_families_size(); ++i) { 21 | s1 << b.column_families(i).ShortDebugString(); 22 | } 23 | LOG(INFO) << "[utils] " << s1.str(); 24 | return (s0.str().compare(s1.str()) != 0); 25 | } 26 | 27 | bool IsSchemaLgDiff(const TableSchema& a, const TableSchema& b) { 28 | std::stringstream s0; 29 | std::stringstream s1; 30 | for (int i = 0; i < a.locality_groups_size(); ++i) { 31 | s0 << a.locality_groups(i).ShortDebugString(); 32 | } 33 | LOG(INFO) << "[utils] " << s0.str(); 34 | for (int i = 0; i < b.locality_groups_size(); ++i) { 35 | s1 << b.locality_groups(i).ShortDebugString(); 36 | } 37 | LOG(INFO) << "[utils] " << s1.str(); 38 | return (s0.str().compare(s1.str()) != 0); 39 | } 40 | 41 | } // namespace tera 42 | -------------------------------------------------------------------------------- /src/utils/schema_utils.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_SCHEMA_UTILS_H_ 6 | #define TERA_SCHEMA_UTILS_H_ 7 | 8 | #include "proto/table_schema.pb.h" 9 | 10 | namespace tera { 11 | 12 | bool IsSchemaCfDiff(const TableSchema& a, const TableSchema& b); 13 | 14 | bool IsSchemaLgDiff(const TableSchema& a, const TableSchema& b); 15 | 16 | } // namespace tera 17 | 18 | #endif // TERA_SCHEMA_UTILS_H_ 19 | -------------------------------------------------------------------------------- /src/utils/string_util.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_UTIL_STRING_UTIL_H_ 6 | #define TERA_UTIL_STRING_UTIL_H_ 7 | 8 | #include 9 | 10 | namespace tera { 11 | 12 | extern const size_t kNameLenMin; 13 | extern const size_t kNameLenMax; 14 | 15 | // binary string to debug string 16 | std::string DebugString(const std::string& src); 17 | // debug string to binary string 18 | bool ParseDebugString(const std::string& src, std::string* dst); 19 | 20 | bool IsValidName(const std::string& str); 21 | bool IsValidTableName(const std::string& str); 22 | bool IsValidGroupName(const std::string& name); 23 | bool IsValidUserName(const std::string& name); 24 | 25 | bool IsValidColumnFamilyName(const std::string& str); 26 | std::string RoundNumberToNDecimalPlaces(double n, int d); 27 | int EditDistance(const std::string& a, const std::string& b); 28 | } // namespace tera 29 | 30 | #endif // TERA_UTIL_STRING_UTIL_H_ 31 | -------------------------------------------------------------------------------- /src/utils/utils_cmd.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_UTILS_UTILS_CMD_H_ 6 | #define TERA_UTILS_UTILS_CMD_H_ 7 | 8 | #include 9 | #include 10 | 11 | #include "leveldb/env.h" 12 | 13 | namespace tera { 14 | namespace utils { 15 | 16 | std::string GetBinaryLocationDir(); 17 | 18 | std::string GetCurrentLocationDir(); 19 | 20 | std::string GetValueFromEnv(const std::string& env_name); 21 | 22 | std::string ConvertByteToString(const uint64_t size); 23 | 24 | std::string GetLocalHostAddr(); 25 | 26 | std::string GetLocalHostName(); 27 | 28 | bool ExecuteShellCmd(const std::string& cmd, std::string* ret_str = NULL); 29 | 30 | void SetupLog(const std::string& program_name); 31 | 32 | } // namespace utils 33 | } // namespace tera 34 | 35 | #endif // TERA_UTILS_UTILS_CMD_H_ 36 | -------------------------------------------------------------------------------- /src/version.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_VERSION_H_ 6 | #define TERA_VERSION_H_ 7 | 8 | extern const char kGitInfo[]; 9 | extern const char kBuildType[]; 10 | extern const char kBuildTime[]; 11 | extern const char kBuilderName[]; 12 | extern const char kHostName[]; 13 | extern const char kCompiler[]; 14 | void PrintSystemVersion(); 15 | std::string SystemVersionInfo(); 16 | 17 | #endif // TERA_VERSION_H_ 18 | -------------------------------------------------------------------------------- /src/zk/dummy_zk_adapter.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. 4 | 5 | #ifndef TERA_ZK_DUMMY_ZK_ADAPTER_H_ 6 | #define TERA_ZK_DUMMY_ZK_ADAPTER_H_ 7 | 8 | #include "zk/zk_adapter.h" 9 | 10 | namespace tera { 11 | namespace zk { 12 | 13 | class DummyNodeZkAdapter : public ZooKeeperAdapter { 14 | public: 15 | virtual ~DummyNodeZkAdapter() {} 16 | 17 | protected: 18 | virtual void OnChildrenChanged(const std::string& path, const std::vector& name_list, 19 | const std::vector& data_list) {} 20 | virtual void OnNodeValueChanged(const std::string& path, const std::string& value) {} 21 | virtual void OnNodeCreated(const std::string& path) {} 22 | virtual void OnNodeDeleted(const std::string& path) {} 23 | virtual void OnWatchFailed(const std::string& path, int watch_type, int err) {} 24 | virtual void OnSessionTimeout() {} 25 | }; 26 | 27 | } // namespace zk 28 | } // namespace tera 29 | 30 | #endif // TERA_ZK_DUMMY_ZK_ADAPTER_H_ 31 | --------------------------------------------------------------------------------