├── .gitignore ├── CMakeLists.txt ├── COPYING ├── LinUCBSchedule ├── EpsilonGreedy.py ├── LayerUCB.py ├── LinUCB.py ├── Logs │ └── .gitkeep ├── ScheduleFrame.py ├── analyse │ ├── activate_fun.py │ ├── analyse_config.py │ └── analyse_data.py ├── config.py ├── figures │ └── .gitkeep ├── main.py ├── server.py ├── server_util.py └── util.py ├── README.md ├── README.zh.md ├── build.sh ├── doc ├── Deployment_doc.pdf ├── cache_framework_design.md ├── conf_spec │ └── newcache.conf_spec ├── frequently_asked_questions.md ├── image │ ├── HybridCache_architecture.PNG │ ├── JYCache_architecture.PNG │ ├── JYCache_architecture_en.png │ ├── LocalWriteCache_throttle.png │ ├── OLUCB_CS.png │ ├── OLUCB_frame.png │ ├── page_structure.jpg │ ├── result_LinUCB.png │ ├── result_throttle.png │ ├── serial_read.png │ ├── system_positioning.png │ └── system_purpose.png ├── 技术报告.md └── 项目PR说明.md ├── global_cache ├── CMakeLists.txt ├── Common.cpp ├── Common.h ├── ErasureCodingWriteCacheClient.cpp ├── ErasureCodingWriteCacheClient.h ├── EtcdClient.h ├── FileSystemDataAdaptor.h ├── GarbageCollectorMain.cpp ├── GlobalCacheClient.cpp ├── GlobalCacheClient.h ├── GlobalCacheServer.cpp ├── GlobalCacheServer.h ├── GlobalCacheServerMain.cpp ├── GlobalDataAdaptor.cpp ├── GlobalDataAdaptor.h ├── Placement.h ├── ReadCache.cpp ├── ReadCache.h ├── ReadCacheClient.cpp ├── ReadCacheClient.h ├── ReplicationWriteCacheClient.cpp ├── ReplicationWriteCacheClient.h ├── S3DataAdaptor.cpp ├── S3DataAdaptor.h ├── WriteCache.cpp ├── WriteCache.h ├── WriteCacheClient.h └── gcache.proto ├── hadoop_posix ├── .bazelrc ├── .gitignore ├── BUILD ├── README.md ├── WORKSPACE ├── copts.bzl ├── hadoop_sdk │ ├── .clang-format │ ├── README.md │ ├── java │ │ ├── native │ │ │ ├── BUILD │ │ │ ├── io_aisoft9_jycache_fs_libfs_JYCacheFSMount.cpp │ │ │ └── io_aisoft9_jycache_fs_libfs_JYCacheFSMount.h │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ └── java │ │ │ │ └── io │ │ │ │ └── aisoft9 │ │ │ │ └── jycache │ │ │ │ └── fs │ │ │ │ ├── flink │ │ │ │ ├── JYCacheFileSystemFactory.java │ │ │ │ └── JYCacheFileSystemTableFactory.java │ │ │ │ ├── hadoop │ │ │ │ ├── JYCacheFS.java │ │ │ │ ├── JYCacheFSInputStream.java │ │ │ │ ├── JYCacheFSOutputStream.java │ │ │ │ ├── JYCacheFSProto.java │ │ │ │ ├── JYCacheFSTalker.java │ │ │ │ ├── JYCacheFileSystem.java │ │ │ │ └── permission │ │ │ │ │ ├── Group.java │ │ │ │ │ ├── Helper.java │ │ │ │ │ ├── Permission.java │ │ │ │ │ └── User.java │ │ │ │ └── libfs │ │ │ │ ├── JYCacheFSMount.java │ │ │ │ ├── JYCacheFSNativeLoader.java │ │ │ │ ├── JYCacheFSStat.java │ │ │ │ └── JYCacheFSStatVFS.java │ │ │ └── test │ │ │ └── java │ │ │ └── io │ │ │ └── aisoft9 │ │ │ └── jycache │ │ │ └── fs │ │ │ └── others │ │ │ └── TestOthers.java │ └── libjycachefs │ │ ├── BUILD │ │ ├── README.md │ │ ├── client.BUILD │ │ ├── examples │ │ ├── Makefile │ │ ├── append.c │ │ ├── chmod.c │ │ ├── common.h │ │ ├── fstat.c │ │ ├── ls.c │ │ ├── mkdir.c │ │ ├── read.c │ │ ├── rename.c │ │ ├── rmdir.c │ │ ├── stat.c │ │ ├── statfs.c │ │ ├── touch.c │ │ ├── unlink.c │ │ └── write.c │ │ ├── libjycachefs.cpp │ │ └── libjycachefs.h └── util │ └── sdk.sh ├── install.sh ├── intercept ├── CMakeLists.txt ├── client.cpp ├── common │ ├── CMakeLists.txt │ ├── common.cpp │ └── common.h ├── discovery │ ├── CMakeLists.txt │ ├── discovery.h │ ├── iceoryx_discovery.cpp │ └── iceoryx_discovery.h ├── filesystem │ ├── CMakeLists.txt │ ├── abstract_filesystem.h │ ├── curve_filesystem.cpp │ ├── curve_filesystem.h │ ├── dummy_filesystem.cpp │ ├── dummy_filesystem.h │ ├── libcurvefs_external.cpp │ ├── libcurvefs_external.h │ ├── s3fs_filesystem.cpp │ ├── s3fs_filesystem.h │ └── s3fs_lib.h ├── internal │ ├── CMakeLists.txt │ ├── metainfo.h │ ├── posix_op_req_res.cpp │ └── posix_op_req_res.h ├── middleware │ ├── CMakeLists.txt │ ├── iceoryx_wrapper.cpp │ ├── iceoryx_wrapper.h │ ├── req_res_middleware_wrapper.cpp │ └── req_res_middleware_wrapper.h ├── posix │ ├── CMakeLists.txt │ ├── libsyscall_intercept_hook_point.h │ ├── posix_helper.h │ ├── posix_op.cpp │ ├── posix_op.h │ └── syscall_client.h ├── registry │ ├── CMakeLists.txt │ ├── client_server_registry.cpp │ └── client_server_registry.h └── server.cpp ├── local_cache ├── CMakeLists.txt ├── accessor.h ├── common.cpp ├── common.h ├── config.cpp ├── config.h ├── data_adaptor.h ├── errorcode.h ├── page_cache.cpp ├── page_cache.h ├── read_cache.cpp ├── read_cache.h ├── throttle.cpp ├── throttle.h ├── write_cache.cpp └── write_cache.h ├── s3fs ├── CMakeLists.txt ├── addhead.cpp ├── addhead.h ├── autolock.cpp ├── autolock.h ├── cache.cpp ├── cache.h ├── common.h ├── common_auth.cpp ├── config.h ├── curl.cpp ├── curl.h ├── curl_handlerpool.cpp ├── curl_handlerpool.h ├── curl_multi.cpp ├── curl_multi.h ├── curl_util.cpp ├── curl_util.h ├── fdcache.cpp ├── fdcache.h ├── fdcache_auto.cpp ├── fdcache_auto.h ├── fdcache_entity.cpp ├── fdcache_entity.h ├── fdcache_fdinfo.cpp ├── fdcache_fdinfo.h ├── fdcache_page.cpp ├── fdcache_page.h ├── fdcache_pseudofd.cpp ├── fdcache_pseudofd.h ├── fdcache_stat.cpp ├── fdcache_stat.h ├── fdcache_untreated.cpp ├── fdcache_untreated.h ├── hybridcache_accessor_4_s3fs.cpp ├── hybridcache_accessor_4_s3fs.h ├── hybridcache_disk_data_adaptor.cpp ├── hybridcache_disk_data_adaptor.h ├── hybridcache_s3_data_adaptor.cpp ├── hybridcache_s3_data_adaptor.h ├── metaheader.cpp ├── metaheader.h ├── mpu_util.cpp ├── mpu_util.h ├── openssl_auth.cpp ├── psemaphore.h ├── s3fs.cpp ├── s3fs.h ├── s3fs_auth.h ├── s3fs_cred.cpp ├── s3fs_cred.h ├── s3fs_extcred.h ├── s3fs_global.cpp ├── s3fs_help.cpp ├── s3fs_help.h ├── s3fs_lib.cpp ├── s3fs_lib.h ├── s3fs_logger.cpp ├── s3fs_logger.h ├── s3fs_util.cpp ├── s3fs_util.h ├── s3fs_version.md ├── s3fs_xml.cpp ├── s3fs_xml.h ├── s3objlist.cpp ├── s3objlist.h ├── sighandlers.cpp ├── sighandlers.h ├── string_util.cpp ├── string_util.h ├── threadpoolman.cpp ├── threadpoolman.h └── types.h └── test ├── CMakeLists.txt ├── hybridcache.conf ├── test_config.cpp ├── test_future.cpp ├── test_global_read_cache.cpp ├── test_global_read_cache_perf.cpp ├── test_global_write_cache_perf.cpp ├── test_page_cache.cpp ├── test_read_cache.cpp └── test_write_cache.cpp /.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | thirdparties/ 3 | JYCache_Env/ 4 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | project(hybridcache) 2 | 3 | cmake_minimum_required(VERSION 3.7) 4 | cmake_policy(SET CMP0079 NEW) 5 | set(CMAKE_CXX_STANDARD 17) 6 | 7 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-PIE") 8 | set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fno-PIE") 9 | 10 | list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/thirdparties) 11 | list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/thirdparties/CmakeFiles) 12 | include(ThirdPartyConfig) 13 | 14 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DNDEBUG -O3 -g -D__const__=__unused__ -pipe -W -Wno-deprecated -Wno-sign-compare -Wno-unused-parameter -fPIC") 15 | 16 | include_directories(AFTER ${CMAKE_SOURCE_DIR}/local_cache ${CMAKE_SOURCE_DIR}/global_cache) 17 | include_directories(AFTER ${CMAKE_BINARY_DIR}/local_cache ${CMAKE_BINARY_DIR}/global_cache) 18 | 19 | # subdirectory 20 | add_subdirectory(local_cache) 21 | add_subdirectory(global_cache) 22 | add_subdirectory(s3fs) 23 | add_subdirectory(intercept) 24 | add_subdirectory(test) 25 | -------------------------------------------------------------------------------- /LinUCBSchedule/Logs/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aisoft9/JYCache/050c668dd9025e3b99c9e4d040e57b95fa15278f/LinUCBSchedule/Logs/.gitkeep -------------------------------------------------------------------------------- /LinUCBSchedule/ScheduleFrame.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | import numpy as np 3 | # from scipy.special import gamma 4 | 5 | class ScheduleFrame: 6 | def __init__(self): 7 | pass 8 | 9 | @abstractmethod 10 | def select_arm(self): 11 | pass 12 | 13 | @abstractmethod 14 | def update(self, reward, chosen_arm): 15 | pass 16 | 17 | @abstractmethod 18 | def get_now_reward(self, performance, context_info=None): 19 | pass 20 | 21 | 22 | def latin_hypercube_sampling(n, d, m, M): 23 | # Initialize the samples array 24 | samples = np.zeros((n, d)) 25 | 26 | for i in range(n): 27 | # Generate d random numbers and normalize them 28 | while True: 29 | x = np.random.uniform(m, M, d) 30 | if np.sum(x) >= M: 31 | break 32 | x = x / np.sum(x) * M 33 | 34 | # Check if all elements are in the range [m, M] 35 | if np.all(x >= m) and np.all(x <= M): 36 | samples[i] = x 37 | else: 38 | # Re-generate the sample if it doesn't satisfy the constraints 39 | i -= 1 40 | for i in range(len(samples)): 41 | samples[i] = [int(ele) for ele in samples[i]] 42 | curr_sum = sum(samples[i]) 43 | samples[i][-1] += M - curr_sum 44 | 45 | return samples 46 | 47 | 48 | if __name__ == '__main__': 49 | # Parameters 50 | num_samples = 20 # Number of samples 51 | num_dimensions = 3 # Number of dimensions 52 | min_value = 0 # Minimum value of each element 53 | total_sum = 240 # Total sum of elements in each sample 54 | 55 | # Generate samples 56 | samples = latin_hypercube_sampling(num_samples, num_dimensions, min_value, total_sum) 57 | print(samples) 58 | 59 | pass 60 | -------------------------------------------------------------------------------- /LinUCBSchedule/analyse/activate_fun.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | 4 | # 生成数据 5 | x = np.linspace(0, 20000, 1000) 6 | 7 | # Sigmoid 函数 8 | def sigmoid(x, k=0.0003, x0=13000): 9 | return 1 / (1 + np.exp(-k * (x - x0))) 10 | 11 | # Tanh 函数 12 | def tanh(x, k=0.0005, x0=3000): 13 | return (np.tanh(k * (x - x0)) + 1) / 2 14 | 15 | # 指数归一化函数 16 | def exponential_normalize(x, alpha=0.5): 17 | return (x / 6000) ** alpha 18 | 19 | # Arctan 函数 20 | def arctan(x, k=0.001, x0=3000): 21 | return 0.5 + (1 / np.pi) * np.arctan(k * (x - x0)) 22 | 23 | # 映射到 0-1 24 | y1 = sigmoid(x) 25 | y2 = tanh(x) 26 | y3 = exponential_normalize(x) 27 | y4 = arctan(x) 28 | 29 | # 绘制图像 30 | plt.plot(x, y1, label='sigmoid') 31 | plt.plot(x, y2, label='tanh') 32 | plt.plot(x, y3, label='exponential_normalize') 33 | plt.plot(x, y4, label='arctan') 34 | plt.legend() 35 | plt.title("Function Mapping") 36 | plt.xlabel("Original Values") 37 | plt.ylabel("Mapped Values (0-1)") 38 | plt.savefig('../figures/activate_fun.png') 39 | -------------------------------------------------------------------------------- /LinUCBSchedule/analyse/analyse_config.py: -------------------------------------------------------------------------------- 1 | import re 2 | import matplotlib.pyplot as plt 3 | 4 | with open('../Logs/linucb.log', 'r', encoding='utf-8') as file: 5 | text = file.read() 6 | 7 | result = [[float(x) for x in match] for match in re.findall(r'reward : ([\d\.]+) \[(\d+), (\d+)\]', text)] 8 | 9 | rewards = [item[0] for item in result] 10 | write_pool_sizes = [item[1] for item in result] 11 | read_pool_sizes = [item[2] for item in result] 12 | # flush_pool_sizes = [item[3] for item in result] 13 | 14 | def putAll(): 15 | plt.figure(figsize=(15, 6)) 16 | 17 | plt.plot(write_pool_sizes, label='WritePool Size', marker='o', markersize=5, alpha=0.5) 18 | plt.plot(read_pool_sizes, label='ReadPool Size', marker='s', markersize=5, alpha=0.8) 19 | # plt.plot(flush_pool_sizes, label='FlushPool Size', marker='^', markersize=5, alpha=0.8) 20 | 21 | plt.title('Pool Sizes Over Time') 22 | plt.xlabel('Time (Epoch)') 23 | plt.ylabel('Pool Size') 24 | 25 | plt.legend() 26 | 27 | plt.grid(True) 28 | 29 | # plt.show() 30 | plt.savefig('../figures/newPoolSize.png', dpi=300) 31 | 32 | def separate(): 33 | # Plotting the data in a 2x2 grid 34 | fig, axs = plt.subplots(2, 2, figsize=(18, 10)) 35 | 36 | # Write Pool size plot 37 | axs[0, 0].plot(write_pool_sizes, marker='o', linestyle='-', color='blue') 38 | axs[0, 0].set_title('Write Pool Size Over Time') 39 | axs[0, 0].set_xlabel('Iteration') 40 | axs[0, 0].set_ylabel('Write Pool Size') 41 | axs[0, 0].grid(True) 42 | 43 | # Read Pool size plot 44 | axs[0, 1].plot(read_pool_sizes, marker='^', linestyle='-', color='green') 45 | axs[0, 1].set_title('Read Pool Size Over Time') 46 | axs[0, 1].set_xlabel('Iteration') 47 | axs[0, 1].set_ylabel('Read Pool Size') 48 | axs[0, 1].grid(True) 49 | 50 | # Flush Pool size plot 51 | # axs[1, 0].plot(flush_pool_sizes, marker='+', linestyle='-', color='red') 52 | # axs[1, 0].set_title('Flush Pool Size Over Time') 53 | # axs[1, 0].set_xlabel('Iteration') 54 | # axs[1, 0].set_ylabel('Flush Pool Size') 55 | # axs[1, 0].grid(True) 56 | 57 | # Reward plot 58 | axs[1, 1].plot(rewards, marker='*', linestyle='-', color='purple') 59 | axs[1, 1].set_title('Reward Over Time') 60 | axs[1, 1].set_xlabel('Iteration') 61 | axs[1, 1].set_ylabel('Reward') 62 | axs[1, 1].grid(True) 63 | 64 | plt.tight_layout() 65 | 66 | plt.savefig('../figures/pool_sizes_separate.png', dpi=300) 67 | 68 | def calculate_greater_than_ratio(pool_sizes, average_size=80): 69 | count_greater= sum(1 for size in pool_sizes if size > average_size) 70 | total_count = len(pool_sizes) 71 | ratio = count_greater / total_count 72 | return ratio 73 | 74 | def calculate_all_ratio(): 75 | average_size = 120 # 单机unet3d专属 76 | train_point = 20 # 训练开始的节点 77 | write_pool_ratio = calculate_greater_than_ratio(write_pool_sizes[train_point:], average_size) 78 | read_pool_ratio = calculate_greater_than_ratio(read_pool_sizes[train_point:], average_size) 79 | # flush_pool_ratio = calculate_greater_than_ratio(flush_pool_sizes[train_point:], average_size) 80 | 81 | print(f'WritePool > 120 Ratio: {write_pool_ratio:.2%}') 82 | print(f'ReadPool > 120 Ratio: {read_pool_ratio:.2%}') 83 | # print(f'FlushPool > 120 Ratio: {flush_pool_ratio:.2%}') 84 | 85 | if __name__ == '__main__': 86 | separate() 87 | for i in result: 88 | print(i) -------------------------------------------------------------------------------- /LinUCBSchedule/analyse/analyse_data.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | 3 | if __name__ == '__main__': 4 | log_name = '../Logs/linucb.log' 5 | index = [] 6 | reward = [] 7 | count = 1 8 | # with open(log_name, 'r') as log_file: 9 | # for line in log_file.readlines(): 10 | # # idx, rew = line.split(' ') 11 | # rew = line 12 | # idx = count 13 | # index.append(idx) 14 | # reward.append(rew) 15 | # count += 1 16 | with open(log_name, 'r') as log_file: 17 | for line in log_file.readlines(): 18 | if 'reward' in line: 19 | idx = count 20 | reward_str = line.split(':')[1].split('[')[0].strip() # 提取 reward 数值部分 21 | reward.append(float(reward_str)) 22 | index.append(idx) 23 | count += 1 24 | reward = [float(i) for i in reward] 25 | # best_score = 0.32 26 | best_score = reward[0] * 1.2 27 | target = [best_score] * len(index) 28 | init_score = [reward[0]] * len(index) 29 | 30 | plt.plot(index, reward) 31 | plt.plot(index, target) 32 | plt.plot(index, init_score) 33 | # 设置刻度 34 | plt.xticks(list(range(0, int(index[-1]), int(int(index[-1])/20))), rotation = 45) 35 | plt.yticks([i * 0.2 for i in range(0, int(best_score // 0.2) + 2, 1)]) 36 | plt.title('OLUCB') 37 | # plt.show() 38 | plt.savefig('../figures/OLUCB_JYCache.png') 39 | 40 | -------------------------------------------------------------------------------- /LinUCBSchedule/config.py: -------------------------------------------------------------------------------- 1 | # The number of hypercube samples in the initial stage 2 | SAMPLE_TIMES = 20 3 | 4 | # Ratio of sample, if ratio is 10 and num_cache is 100, valid solution in sample is 0, 10, 20, 30 ...... 100 5 | SAMPLE_RATIO = 10 6 | 7 | # If no better config is found within threshold, 8 | # it is determined that the model has entered a state of approximate convergence 9 | APPROXIMATE_CONVERGENCE_THRESHOLD = 150 10 | 11 | # In approximate convergence state,There is a certain probability to directly choose the optimal solution 12 | # that has been explored, otherwise continue to explore 13 | PROBABILITY_THRESHOLD = 80 14 | 15 | # It takes time for rewards to level off, only after this threshold, detection begins for load changes 16 | LOAD_CHANGE_THRESHOLD = 50 17 | 18 | # The size of history reward sliding window 19 | HISTORY_REWARD_WINDOW = 50 20 | 21 | # Basis for workload changes 22 | WORKLOAD_CHANGE = 0.30 23 | 24 | -------------------------------------------------------------------------------- /LinUCBSchedule/figures/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aisoft9/JYCache/050c668dd9025e3b99c9e4d040e57b95fa15278f/LinUCBSchedule/figures/.gitkeep -------------------------------------------------------------------------------- /LinUCBSchedule/server_util.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | # unet3d x0 设置为 3000，对于 resnet50 x0 设置为 30000 4 | # s3fs x0 设置为13000, k = 0.0003 5 | def sigmoid(x, k=0.0003, x0=13000): 6 | return 1 / (1 + np.exp(-k * (x - x0))) 7 | # 接收curve发送的request，解析为pool_and_size, pool_and_throughput 8 | def process_request(request): 9 | pairs = request.split(';')[:-1] 10 | pool_and_size = {} 11 | pool_and_throughput = {} 12 | pool_and_invoke = {} 13 | pair1 = pairs[:2] 14 | pair2 = pairs[2:4] 15 | pair3 = pairs[4:] 16 | for pair in pair1: 17 | key,value = pair.split(':') 18 | pool_and_size[key] = int(value) 19 | for pair in pair2: 20 | key,value = pair.split(':') 21 | pool_and_throughput[key] = float(value) 22 | for pair in pair3: 23 | key,value = pair.split(':') 24 | pool_and_invoke[key] = sigmoid(float(value)) 25 | return pool_and_size, pool_and_throughput, pool_and_invoke -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | if [ ! -d "./thirdparties" ]; then 3 | case $(uname -m) in 4 | x86_64) 5 | wget https://madstorage.s3.cn-north-1.jdcloud-oss.com/JYCache_Dendepency_x64.tgz 6 | md5=`md5sum JYCache_Dendepency_x64.tgz | awk {'print $1'}` 7 | if [ "$md5" != "48f67dd9b7bcb1b2bdd6be9f2283b714" ]; then 8 | echo 'JYCache_Dendepency version inconsistency!' 9 | exit 1 10 | fi 11 | tar -zxvf JYCache_Dendepency_x64.tgz 12 | ;; 13 | aarch64) 14 | wget https://madstorage.s3.cn-north-1.jdcloud-oss.com/JYCache_Dendepency_arm64.tgz 15 | md5=`md5sum JYCache_Dendepency_arm64.tgz | awk {'print $1'}` 16 | if [ "$md5" != "5c90ddd6b0849336adeccbdadf42f065" ]; then 17 | echo 'JYCache_Dendepency version inconsistency!' 18 | exit 1 19 | fi 20 | tar -zxvf JYCache_Dendepency_arm64.tgz 21 | ;; 22 | esac 23 | fi 24 | 25 | mkdir -p build && cd build 26 | cmake .. && cmake --build . -j 16 27 | -------------------------------------------------------------------------------- /doc/Deployment_doc.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aisoft9/JYCache/050c668dd9025e3b99c9e4d040e57b95fa15278f/doc/Deployment_doc.pdf -------------------------------------------------------------------------------- /doc/cache_framework_design.md: -------------------------------------------------------------------------------- 1 | # 缓存系统设计 2 | 3 | ### 设计背景 4 | 5 | 在用户和数据服务器之间构建一套缓存系统，该缓存系统可以让用户以本地文件的形式透明且高效地访问数据服务器中的数据。其中，数据服务器的类型有对象存储、自建全局缓存等。以数据服务器为对象存储为例，用户可以通过fuse以本地文件形式访问存储在远端的对象，且远端的对象索引是用户可懂的。 6 | ![](image/system_purpose.png) 7 | 8 | ### 系统定位 9 | 该缓存系统支持多种数据源，包括S3对象存储、自建全局缓存等，故称为HybridCache。同时借助S3FS对fuse的支持，以及其在元数据管理方面的能力，实现fuse模式下的文件管理操作。HybridCache的定位如下图所示： 10 | ![](image/system_positioning.png) 11 | 12 | ### 系统架构 13 | HybridCache架构如下图所示： 14 | ![](image/HybridCache_architecture.PNG) 15 | 16 | 1.写缓存模块 17 | 18 | 写缓存模块的定位是本地写缓存，写缓存中的key是文件的path，不理解远端数据源（对象存储和全局缓存等），从write->flush的过程由上层去做。 19 | 20 | 2.读缓存模块 21 | 22 | 读缓存模块的定位是文件（以远端数据源为对象存储为例）的只读缓存，读缓存中的key是对象的key。读缓存需要用到本地缓存，以及远端缓存（对象存储和全局缓存等）。 23 | 24 | 3.数据源访问组件 25 | 26 | 数据源访问组件负责和远端数据源进行交互，涉及数据的上传下载等。以Adaptor的形式支持多种数据源，包括对象存储和全局缓存等。 27 | 28 | 4.缓存管理组件 29 | 30 | 内存管理组件管理本地缓存，写缓存模块和读缓存模块中实际的本地缓存就是用的该组件。 31 | 在本地缓存中，我们直接将文件切分为固定大小的page（page大小可配置，下文以64KB为例），并使用CacheLib来维护这些page。page在CacheLib中以KV形式进行存储，其存储结构如下： 32 | - key为 cacheKey_pageid。读写模块各自维护自己的本地缓存，cacheKey在写缓存模块中就是文件的path，在读缓存模块中就是S3上对象的key。pageid即为页号，通过offset/64KB计算得来。 33 | - value的数据结构如下： 34 | ![](image/page_structure.jpg) 35 | 36 | 通过 cacheKey+offset+size 即可接操作指定文件中的特定page。page并发操作的安全性是通过CacheLib自身的机制以及page内的lock和新旧版号位来保证。 37 | 38 | 5.HybridCache访问组件 39 | 40 | HybridCache访问组件定位在胶水层，要根据上层调用方的特性定制化实现，其内需要理解到上层调用方的逻辑。 41 | -------------------------------------------------------------------------------- /doc/conf_spec/newcache.conf_spec: -------------------------------------------------------------------------------- 1 | # ReadCache 2 | ReadCacheConfig.CacheConfig.CacheName # 读缓存名称 3 | ReadCacheConfig.CacheConfig.MaxCacheSize # 读缓存内存容量限制 4 | ReadCacheConfig.CacheConfig.PageBodySize # 读缓存page大小 5 | ReadCacheConfig.CacheConfig.PageMetaSize # 读缓存page元数据大小 6 | ReadCacheConfig.CacheConfig.EnableCAS # 读缓存是否启用CAS 7 | ReadCacheConfig.CacheConfig.SafeMode # 读缓存是否启用 write/delete 原子锁 8 | ReadCacheConfig.CacheConfig.CacheLibConfig.EnableNvmCache # 读缓存是否开启nvm缓存 9 | ReadCacheConfig.CacheConfig.CacheLibConfig.RaidPath # nvm缓存文件目录 10 | ReadCacheConfig.CacheConfig.CacheLibConfig.RaidFileNum # nvm缓存文件数量限制 11 | ReadCacheConfig.CacheConfig.CacheLibConfig.RaidFileSize # nvm单个缓存文件大小限制 12 | ReadCacheConfig.CacheConfig.CacheLibConfig.DataChecksum # nvm缓存是否进行数据校验 13 | ReadCacheConfig.DownloadNormalFlowLimit # 读缓存内存未命中从远端下载时的平峰流控 14 | ReadCacheConfig.DownloadBurstFlowLimit # 读缓存内存未命中从远端下载时的顶峰流控 15 | 16 | # WriteCache 17 | WriteCacheConfig.CacheConfig.CacheName # 写缓存名称 18 | WriteCacheConfig.CacheConfig.MaxCacheSize # 写缓存内存容量限制 19 | WriteCacheConfig.CacheConfig.PageBodySize # 写缓存page大小 20 | WriteCacheConfig.CacheConfig.PageMetaSize # 写缓存page元数据大小 21 | WriteCacheConfig.CacheConfig.EnableCAS # 写缓存是否启用CAS 22 | WriteCacheConfig.CacheConfig.SafeMode # 写缓存是否启用 write/delete 原子锁 23 | WriteCacheConfig.CacheSafeRatio # 写缓存安全容量阈值(百分比), 缓存达到阈值时阻塞待异步flush释放空间 24 | WriteCacheConfig.EnableThrottle # 写缓存开启限流 25 | 26 | # GlobalCache 27 | UseGlobalCache # 全局缓存开关 28 | GlobalCacheConfig.EnableWriteCache # 全局缓存是否启用写缓存 29 | GlobalCacheConfig.EtcdAddress # etcd地址，例如 http://127.0.0.1:2379 30 | GlobalCacheConfig.GlobalServers # 全局缓存服务端地址，例如 127.0.0.1:8000 31 | GlobalCacheConfig.GflagFile # 全局缓存gflag文件形式输入 32 | 33 | ThreadNum # 线程数 34 | BackFlushCacheRatio # 写缓存异步flush阈值(百分比) 35 | UploadNormalFlowLimit # 上传平峰流控 36 | UploadBurstFlowLimit # 上传顶峰流控 37 | LogPath # 日志文件路径 38 | LogLevel # 日志级别，INFO=0, WARNING=1, ERROR=2, FATAL=3 39 | EnableLog # 是否启用日志打印 40 | FlushToRead # 文件flush完成后是否写入读缓存 41 | CleanCacheByOpen # 文件open时是否清理读缓存 42 | EnableResize # 是否开启普通的Resize策略 43 | EnableLinUCB # 是否开启LinUCB 44 | -------------------------------------------------------------------------------- /doc/frequently_asked_questions.md: -------------------------------------------------------------------------------- 1 | **1. 如何切换挂载目录？** 2 | 3 | 在start_s3fs.sh中 4 | ```bash 5 | LD_LIBRARY_PATH=./libs/:$LD_LIBRARY_PATH nohup ./s3fs -o passwd_file=./conf/passwd -o use_path_request_style -o endpoint=us-east-1 -o url=http://127.0.0.1:9000 -o bucket=test ./mnt -o dbglevel=err -o use_cache=./diskcache -o del_cache -o newcache_conf=./conf/newcache.conf -f >> ./log/s3fs.log 2>&1 & 6 | ``` 7 | 更换其中的 `./mnt` 即可 -------------------------------------------------------------------------------- /doc/image/HybridCache_architecture.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aisoft9/JYCache/050c668dd9025e3b99c9e4d040e57b95fa15278f/doc/image/HybridCache_architecture.PNG -------------------------------------------------------------------------------- /doc/image/JYCache_architecture.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aisoft9/JYCache/050c668dd9025e3b99c9e4d040e57b95fa15278f/doc/image/JYCache_architecture.PNG -------------------------------------------------------------------------------- /doc/image/JYCache_architecture_en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aisoft9/JYCache/050c668dd9025e3b99c9e4d040e57b95fa15278f/doc/image/JYCache_architecture_en.png -------------------------------------------------------------------------------- /doc/image/LocalWriteCache_throttle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aisoft9/JYCache/050c668dd9025e3b99c9e4d040e57b95fa15278f/doc/image/LocalWriteCache_throttle.png -------------------------------------------------------------------------------- /doc/image/OLUCB_CS.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aisoft9/JYCache/050c668dd9025e3b99c9e4d040e57b95fa15278f/doc/image/OLUCB_CS.png -------------------------------------------------------------------------------- /doc/image/OLUCB_frame.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aisoft9/JYCache/050c668dd9025e3b99c9e4d040e57b95fa15278f/doc/image/OLUCB_frame.png -------------------------------------------------------------------------------- /doc/image/page_structure.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aisoft9/JYCache/050c668dd9025e3b99c9e4d040e57b95fa15278f/doc/image/page_structure.jpg -------------------------------------------------------------------------------- /doc/image/result_LinUCB.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aisoft9/JYCache/050c668dd9025e3b99c9e4d040e57b95fa15278f/doc/image/result_LinUCB.png -------------------------------------------------------------------------------- /doc/image/result_throttle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aisoft9/JYCache/050c668dd9025e3b99c9e4d040e57b95fa15278f/doc/image/result_throttle.png -------------------------------------------------------------------------------- /doc/image/serial_read.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aisoft9/JYCache/050c668dd9025e3b99c9e4d040e57b95fa15278f/doc/image/serial_read.png -------------------------------------------------------------------------------- /doc/image/system_positioning.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aisoft9/JYCache/050c668dd9025e3b99c9e4d040e57b95fa15278f/doc/image/system_positioning.png -------------------------------------------------------------------------------- /doc/image/system_purpose.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aisoft9/JYCache/050c668dd9025e3b99c9e4d040e57b95fa15278f/doc/image/system_purpose.png -------------------------------------------------------------------------------- /global_cache/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include(FindThreads) 2 | include(FindProtobuf) 3 | protobuf_generate_cpp(PROTO_SRC PROTO_HEADER gcache.proto) 4 | include_directories(${CMAKE_CURRENT_BINARY_DIR} /usr/local/include/jerasure) 5 | 6 | add_library(madfs_global 7 | Common.h 8 | Common.cpp 9 | FileSystemDataAdaptor.h 10 | EtcdClient.h 11 | Placement.h 12 | GlobalCacheClient.h 13 | GlobalCacheClient.cpp 14 | S3DataAdaptor.h 15 | S3DataAdaptor.cpp 16 | GlobalDataAdaptor.h 17 | GlobalDataAdaptor.cpp 18 | ReadCacheClient.h 19 | ReadCacheClient.cpp 20 | ReplicationWriteCacheClient.h 21 | ReplicationWriteCacheClient.cpp 22 | GlobalCacheServer.h 23 | GlobalCacheServer.cpp 24 | ReadCache.h 25 | ReadCache.cpp 26 | WriteCache.h 27 | WriteCache.cpp 28 | WriteCacheClient.h 29 | ErasureCodingWriteCacheClient.h 30 | ErasureCodingWriteCacheClient.cpp 31 | ${PROTO_SRC} 32 | ${PROTO_HEADER} 33 | ) 34 | 35 | option(ENABLE_EC "Enable Erasure Coding" OFF) 36 | target_link_libraries(madfs_global PUBLIC hybridcache_local aio) 37 | if(ENABLE_EC) 38 | add_definitions(-DCONFIG_JERASURE) 39 | target_link_libraries(madfs_global PUBLIC Jerasure) 40 | endif() 41 | 42 | add_executable(madfs_global_server GlobalCacheServerMain.cpp) 43 | target_link_libraries(madfs_global_server PUBLIC madfs_global) 44 | 45 | add_executable(madfs_gc GarbageCollectorMain.cpp) 46 | target_link_libraries(madfs_gc PUBLIC madfs_global) 47 | -------------------------------------------------------------------------------- /global_cache/ErasureCodingWriteCacheClient.h: -------------------------------------------------------------------------------- 1 | #ifndef MADFS_EC_WRITE_CACHE_CLIENT_H 2 | #define MADFS_EC_WRITE_CACHE_CLIENT_H 3 | 4 | #include "WriteCacheClient.h" 5 | 6 | using HybridCache::ByteBuffer; 7 | 8 | class GlobalDataAdaptor; 9 | 10 | using PutResult = WriteCacheClient::PutResult; 11 | 12 | class ErasureCodingWriteCacheClient : public WriteCacheClient { 13 | friend class GetChunkContext; 14 | 15 | public: 16 | ErasureCodingWriteCacheClient(GlobalDataAdaptor *parent) : parent_(parent) {} 17 | 18 | ~ErasureCodingWriteCacheClient() {} 19 | 20 | virtual folly::Future Put(const std::string &key, 21 | size_t size, 22 | const ByteBuffer &buffer, 23 | const std::map &headers, 24 | size_t off = 0); 25 | 26 | virtual folly::Future Get(const std::string &key, 27 | size_t start, 28 | size_t size, 29 | ByteBuffer &buffer, 30 | Json::Value &root); 31 | 32 | virtual folly::Future GetDecode(const std::string &key, 33 | size_t start, 34 | size_t size, 35 | ByteBuffer &buffer, 36 | Json::Value &root); 37 | 38 | public: 39 | std::vector GetReplica(const std::string &key); 40 | 41 | struct GetChunkRequestV2 { 42 | std::string user_key; 43 | size_t chunk_id; 44 | size_t chunk_start; 45 | size_t chunk_len; 46 | size_t chunk_granularity; 47 | ByteBuffer buffer; 48 | }; 49 | 50 | static void GenerateGetChunkRequestsV2(const std::string &key, 51 | size_t start, 52 | size_t size, 53 | ByteBuffer &buffer, 54 | std::vector &requests, 55 | size_t chunk_size); 56 | 57 | private: 58 | GlobalDataAdaptor *parent_; 59 | }; 60 | 61 | #endif // MADFS_EC_WRITE_CACHE_CLIENT_H -------------------------------------------------------------------------------- /global_cache/GarbageCollectorMain.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include "S3DataAdaptor.h" 8 | #include "FileSystemDataAdaptor.h" 9 | #include "GlobalDataAdaptor.h" 10 | #include "ReadCacheClient.h" 11 | 12 | #include "GlobalCacheServer.h" 13 | #include "S3DataAdaptor.h" 14 | 15 | DEFINE_string(data_server, "0.0.0.0:8000", "IP address of global data servers"); 16 | DEFINE_string(etcd_server, "http://127.0.0.1:2379", "Location of etcd server"); 17 | DEFINE_string(prefix, "", "Key prefix for garbage collection"); 18 | DEFINE_bool(use_s3, false, "Use S3 storage"); 19 | 20 | std::vector SplitString(const std::string &input) { 21 | std::vector result; 22 | std::stringstream ss(input); 23 | std::string item; 24 | while (std::getline(ss, item, ',')) { 25 | result.push_back(item); 26 | } 27 | return result; 28 | } 29 | 30 | int main(int argc, char *argv[]) { 31 | std::cerr << YELLOW << "MADFS GC TOOL" << WHITE << std::endl; 32 | 33 | gflags::ParseCommandLineFlags(&argc, &argv, true); 34 | auto etcd_client = std::make_shared(FLAGS_etcd_server); 35 | std::shared_ptr base_adaptor; 36 | if (FLAGS_use_s3) { 37 | base_adaptor = std::make_shared(); 38 | } else { 39 | base_adaptor = std::make_shared(); 40 | } 41 | 42 | auto global_adaptor = std::make_shared(base_adaptor, SplitString(FLAGS_data_server), etcd_client); 43 | if (global_adaptor->PerformGarbageCollection(FLAGS_prefix)) { 44 | std::cerr << RED << "Garbage collection failed!" << WHITE << std::endl; 45 | exit(EXIT_FAILURE); 46 | } else { 47 | std::cerr << GREEN << "Garbage collection successfully" << WHITE << std::endl; 48 | exit(EXIT_SUCCESS); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /global_cache/GlobalCacheClient.h: -------------------------------------------------------------------------------- 1 | #ifndef MADFS_GLOBAL_CACHE_CLIENT_H 2 | #define MADFS_GLOBAL_CACHE_CLIENT_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include "Common.h" 11 | #include "common.h" 12 | 13 | using HybridCache::ByteBuffer; 14 | 15 | class GlobalCacheClient { 16 | public: 17 | GlobalCacheClient(const std::string &group = ""); 18 | 19 | ~GlobalCacheClient(); 20 | 21 | int RegisterServer(int server_id, const char *hostname); 22 | 23 | Future GetEntryFromReadCache(int server_id, const std::string &key, uint64_t start, uint64_t length) { 24 | return GetEntry(server_id, key, start, length, true); 25 | } 26 | 27 | Future PutEntryFromReadCache(int server_id, const std::string &key, const ByteBuffer &buf, uint64_t length) { 28 | return PutEntry(server_id, key, buf, length, true); 29 | } 30 | 31 | Future DeleteEntryFromReadCache(int server_id, const std::string &key, uint64_t chunk_size, uint64_t max_chunk_id); 32 | 33 | Future GetEntryFromWriteCache(int server_id, const std::string &key, uint64_t start, uint64_t length){ 34 | return GetEntry(server_id, key, start, length, false); 35 | } 36 | 37 | Future PutEntryFromWriteCache(int server_id, const std::string &key, const ByteBuffer &buf, uint64_t length){ 38 | return PutEntry(server_id, key, buf, length, false); 39 | } 40 | 41 | Future QueryTsFromWriteCache(int server_id); 42 | 43 | Future DeleteEntryFromWriteCache(int server_id, 44 | const std::string &key_prefix, 45 | uint64_t max_ts, 46 | std::vector &except_keys); 47 | 48 | private: 49 | brpc::Channel *GetChannelByServerId(int server_id); 50 | 51 | Future GetEntry(int server_id, const std::string &key, uint64_t start, uint64_t length, bool is_read_cache); 52 | 53 | Future PutEntry(int server_id, const std::string &key, const ByteBuffer &buf, uint64_t length, bool is_read_cache); 54 | 55 | private: 56 | std::mutex mutex_; 57 | const std::string group_; 58 | std::map server_map_; 59 | std::atomic inflight_payload_size_; 60 | }; 61 | 62 | #endif // MADFS_GLOBAL_CACHE_CLIENT_H -------------------------------------------------------------------------------- /global_cache/GlobalCacheServerMain.cpp: -------------------------------------------------------------------------------- 1 | #include "GlobalCacheServer.h" 2 | #include "S3DataAdaptor.h" 3 | 4 | #include 5 | 6 | DEFINE_int32(port, 8000, "TCP Port of global cache server"); 7 | DEFINE_bool(fetch_s3_if_miss, false, "Allow fetch data from S3 if cache miss"); 8 | 9 | int main(int argc, char *argv[]) { 10 | LOG(INFO) << "MADFS Global Cache Server"; 11 | gflags::ParseCommandLineFlags(&argc, &argv, true); 12 | brpc::Server server; 13 | 14 | folly::SingletonVault::singleton()->registrationComplete(); 15 | 16 | brpc::ServerOptions options; 17 | options.num_threads = GetGlobalConfig().rpc_threads; 18 | options.use_rdma = GetGlobalConfig().use_rdma; 19 | 20 | std::shared_ptr base_adaptor = nullptr; 21 | if (FLAGS_fetch_s3_if_miss) { 22 | base_adaptor = std::make_shared(); 23 | } 24 | 25 | auto executor = std::make_shared(GetGlobalConfig().folly_threads); 26 | auto gcache_service = std::make_shared(executor, base_adaptor); 27 | 28 | if (server.AddService(gcache_service.get(), brpc::SERVER_DOESNT_OWN_SERVICE)) { 29 | PLOG(ERROR) << "Failed to register global cache service"; 30 | return -1; 31 | } 32 | 33 | butil::EndPoint point = butil::EndPoint(butil::IP_ANY, FLAGS_port); 34 | if (server.Start(point, &options) != 0) { 35 | PLOG(ERROR) << "Failed to start global cache server"; 36 | return -1; 37 | } 38 | 39 | server.RunUntilAskedToQuit(); 40 | return 0; 41 | } 42 | -------------------------------------------------------------------------------- /global_cache/Placement.h: -------------------------------------------------------------------------------- 1 | #ifndef MADFS_PLACEMENT_H 2 | #define MADFS_PLACEMENT_H 3 | 4 | #include 5 | #include "Common.h" 6 | 7 | inline static std::vector Placement(const std::string &key, int num_available, int num_choose) { 8 | uint64_t seed = std::hash < std::string > {}(key); 9 | std::vector output; 10 | for (int i = 0; i < std::min(num_available, num_choose); ++i) 11 | output.push_back((seed + i) % num_available); 12 | return output; 13 | } 14 | 15 | #endif // MADFS_PLACEMENT_H -------------------------------------------------------------------------------- /global_cache/ReadCache.h: -------------------------------------------------------------------------------- 1 | #ifndef MADFS_READ_CACHE_H 2 | #define MADFS_READ_CACHE_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | #include "Common.h" 11 | #include "data_adaptor.h" 12 | #include "read_cache.h" 13 | 14 | using HybridCache::DataAdaptor; 15 | 16 | class ReadCacheImpl { 17 | public: 18 | virtual Future Get(const std::string &key, uint64_t start, uint64_t length) = 0; 19 | virtual int Put(const std::string &key, uint64_t length, const butil::IOBuf &buf) = 0; 20 | virtual int Delete(const std::string &key) = 0; 21 | virtual int Delete(const std::string &key, uint64_t chunk_size, uint64_t max_chunk_id) = 0; 22 | }; 23 | 24 | class ReadCache { 25 | public: 26 | explicit ReadCache(std::shared_ptr executor, 27 | std::shared_ptr base_adaptor = nullptr); 28 | 29 | ~ReadCache() { 30 | delete impl_; 31 | } 32 | 33 | Future Get(const std::string &key, uint64_t start, uint64_t length) { 34 | return impl_->Get(key, start, length); 35 | } 36 | 37 | int Put(const std::string &key, uint64_t length, const butil::IOBuf &buf) { 38 | return impl_->Put(key, length, buf); 39 | } 40 | 41 | int Delete(const std::string &key) { 42 | return impl_->Delete(key); 43 | } 44 | 45 | int Delete(const std::string &key, uint64_t chunk_size, uint64_t max_chunk_id) { 46 | return impl_->Delete(key, chunk_size, max_chunk_id); 47 | } 48 | 49 | private: 50 | ReadCacheImpl *impl_; 51 | }; 52 | 53 | #endif // MADFS_READ_CACHE_H -------------------------------------------------------------------------------- /global_cache/ReadCacheClient.h: -------------------------------------------------------------------------------- 1 | #ifndef MADFS_READ_CACHE_CLIENT_H 2 | #define MADFS_READ_CACHE_CLIENT_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include "Common.h" 9 | #include "Placement.h" 10 | #include "data_adaptor.h" 11 | 12 | using HybridCache::ByteBuffer; 13 | 14 | class GlobalDataAdaptor; 15 | 16 | class ReadCacheClient { 17 | friend class GetChunkContext; 18 | 19 | public: 20 | ReadCacheClient(GlobalDataAdaptor *parent); 21 | 22 | ~ReadCacheClient(); 23 | 24 | virtual folly::Future Get(const std::string &key, 25 | size_t start, 26 | size_t size, 27 | ByteBuffer &buffer); 28 | 29 | virtual folly::Future Invalidate(const std::string &key, size_t size); 30 | 31 | // for testing only 32 | public: 33 | struct GetChunkRequestV2 { 34 | std::string user_key; 35 | std::string internal_key; 36 | size_t chunk_id; 37 | size_t chunk_start; 38 | size_t chunk_len; 39 | size_t chunk_granularity; 40 | ByteBuffer buffer; 41 | }; 42 | 43 | static void GenerateGetChunkRequestsV2(const std::string &key, 44 | size_t start, 45 | size_t size, 46 | ByteBuffer &buffer, 47 | std::vector &requests, 48 | size_t chunk_size); 49 | 50 | folly::Future GetChunkAsync(int server_id, GetChunkRequestV2 context); 51 | 52 | folly::Future GetChunkFromGlobalCache(int server_id, GetChunkRequestV2 context); 53 | 54 | std::vector GetReplica(const std::string &key, int num_choose); 55 | 56 | private: 57 | GlobalDataAdaptor *parent_; 58 | }; 59 | 60 | #endif // MADFS_READ_CACHE_CLIENT_H -------------------------------------------------------------------------------- /global_cache/ReplicationWriteCacheClient.h: -------------------------------------------------------------------------------- 1 | #ifndef MADFS_REPLICATION_WRITE_CACHE_CLIENT_H 2 | #define MADFS_REPLICATION_WRITE_CACHE_CLIENT_H 3 | 4 | #include "WriteCacheClient.h" 5 | 6 | using HybridCache::ByteBuffer; 7 | 8 | class GlobalDataAdaptor; 9 | 10 | using PutResult = WriteCacheClient::PutResult; 11 | 12 | class ReplicationWriteCacheClient : public WriteCacheClient { 13 | friend class GetChunkContext; 14 | 15 | public: 16 | ReplicationWriteCacheClient(GlobalDataAdaptor *parent) : parent_(parent) {} 17 | 18 | ~ReplicationWriteCacheClient() {} 19 | 20 | virtual folly::Future Put(const std::string &key, 21 | size_t size, 22 | const ByteBuffer &buffer, 23 | const std::map &headers, 24 | size_t off = 0); 25 | 26 | virtual folly::Future Get(const std::string &key, 27 | size_t start, 28 | size_t size, 29 | ByteBuffer &buffer, 30 | Json::Value &root); 31 | 32 | public: 33 | std::vector GetReplica(const std::string &key); 34 | 35 | struct GetChunkRequestV2 { 36 | std::string user_key; 37 | size_t chunk_id; 38 | size_t chunk_start; 39 | size_t chunk_len; 40 | size_t chunk_granularity; 41 | ByteBuffer buffer; 42 | }; 43 | 44 | static void GenerateGetChunkRequestsV2(const std::string &key, 45 | size_t start, 46 | size_t size, 47 | ByteBuffer &buffer, 48 | std::vector &requests, 49 | size_t chunk_size); 50 | 51 | folly::Future GetChunkAsync(int server_id, GetChunkRequestV2 context, std::string &internal_key); 52 | 53 | private: 54 | GlobalDataAdaptor *parent_; 55 | }; 56 | 57 | #endif // MADFS_REPLICATION_WRITE_CACHE_CLIENT_H -------------------------------------------------------------------------------- /global_cache/S3DataAdaptor.h: -------------------------------------------------------------------------------- 1 | #ifndef MADFS_S3_DATA_ADAPTOR_H 2 | #define MADFS_S3_DATA_ADAPTOR_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "data_adaptor.h" 10 | 11 | #include "Common.h" 12 | 13 | using HybridCache::ByteBuffer; 14 | using HybridCache::DataAdaptor; 15 | 16 | class S3DataAdaptor : public DataAdaptor { 17 | public: 18 | S3DataAdaptor(); 19 | 20 | ~S3DataAdaptor(); 21 | 22 | // 从数据服务器加载数据 23 | virtual folly::Future DownLoad(const std::string &key, 24 | size_t start, 25 | size_t size, 26 | ByteBuffer &buffer); 27 | 28 | // 上传数据到数据服务器 29 | virtual folly::Future UpLoad(const std::string &key, 30 | size_t size, 31 | const ByteBuffer &buffer, 32 | const std::map &headers); 33 | 34 | // 删除数据服务器的数据 35 | virtual folly::Future Delete(const std::string &key); 36 | 37 | // 获取数据的元数据 38 | virtual folly::Future Head(const std::string &key, 39 | size_t &size, 40 | std::map &headers); 41 | 42 | private: 43 | Aws::Client::ClientConfiguration *clientCfg_; 44 | Aws::S3::S3Client *s3Client_; 45 | }; 46 | 47 | #endif // MADFS_S3_DATA_ADAPTOR_H -------------------------------------------------------------------------------- /global_cache/WriteCache.h: -------------------------------------------------------------------------------- 1 | #ifndef MADFS_WRITE_CACHE_H 2 | #define MADFS_WRITE_CACHE_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | #include 12 | 13 | #include "Common.h" 14 | 15 | class WriteCacheImpl { 16 | public: 17 | WriteCacheImpl(std::shared_ptr executor) : executor_(executor), next_object_id_(0) {} 18 | virtual GetOutput Get(const std::string &internal_key, uint64_t start, uint64_t length) = 0; 19 | virtual PutOutput Put(const std::string &key, uint64_t length, const butil::IOBuf &buf) = 0; 20 | virtual uint64_t QueryTS() { return next_object_id_.load(); } 21 | virtual int Delete(const std::string &key_prefix, uint64_t ts, const std::unordered_set &except_keys) = 0; 22 | 23 | std::shared_ptr executor_; 24 | std::atomic next_object_id_; 25 | }; 26 | 27 | class WriteCache { 28 | public: 29 | explicit WriteCache(std::shared_ptr executor); 30 | 31 | ~WriteCache() { 32 | delete impl_; 33 | } 34 | 35 | GetOutput Get(const std::string &internal_key, uint64_t start, uint64_t length) { 36 | return impl_->Get(internal_key, start, length); 37 | } 38 | 39 | PutOutput Put(const std::string &key, uint64_t length, const butil::IOBuf &buf) { 40 | return impl_->Put(key, length, buf); 41 | } 42 | 43 | uint64_t QueryTS() { return impl_->QueryTS(); } 44 | 45 | int Delete(const std::string &key_prefix, uint64_t ts, const std::unordered_set &except_keys) { 46 | return impl_->Delete(key_prefix, ts, except_keys); 47 | } 48 | 49 | private: 50 | WriteCacheImpl *impl_; 51 | }; 52 | 53 | #endif // MADFS_WRITE_CACHE_H -------------------------------------------------------------------------------- /global_cache/WriteCacheClient.h: -------------------------------------------------------------------------------- 1 | #ifndef MADFS_WRITE_CACHE_CLIENT_H 2 | #define MADFS_WRITE_CACHE_CLIENT_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include "Common.h" 9 | #include "Placement.h" 10 | #include "data_adaptor.h" 11 | #include "EtcdClient.h" 12 | 13 | using HybridCache::ByteBuffer; 14 | 15 | class GlobalDataAdaptor; 16 | 17 | class WriteCacheClient { 18 | public: 19 | struct PutResult { 20 | int status; 21 | Json::Value root; 22 | }; 23 | 24 | public: 25 | WriteCacheClient() {} 26 | 27 | ~WriteCacheClient() {} 28 | 29 | virtual folly::Future Put(const std::string &key, 30 | size_t size, 31 | const ByteBuffer &buffer, 32 | const std::map &headers, 33 | size_t off = 0) = 0; 34 | 35 | virtual folly::Future Get(const std::string &key, 36 | size_t start, 37 | size_t size, 38 | ByteBuffer &buffer, 39 | Json::Value &root) = 0; 40 | }; 41 | 42 | #endif // MADFS_WRITE_CACHE_CLIENT_H -------------------------------------------------------------------------------- /global_cache/gcache.proto: -------------------------------------------------------------------------------- 1 | syntax="proto2"; 2 | package gcache; 3 | 4 | option cc_generic_services = true; 5 | 6 | message GetEntryRequest { 7 | required string key = 1; 8 | required uint64 start = 2; 9 | required uint64 length = 3; 10 | }; 11 | 12 | message GetEntryResponse { 13 | required int32 status_code = 1; 14 | optional bytes data = 2; 15 | }; 16 | 17 | message PutEntryRequest { 18 | required string key = 1; 19 | required uint64 length = 2; 20 | optional bytes data = 3; 21 | }; 22 | 23 | message PutEntryResponse { 24 | required int32 status_code = 1; 25 | optional string internal_key = 2; // for write cache 26 | }; 27 | 28 | message DeleteEntryRequest { 29 | required string key = 1; // actually 'prefix' 30 | optional uint64 chunk_size = 2; 31 | optional uint64 max_chunk_id = 3; 32 | }; 33 | 34 | message DeleteEntryRequestForWriteCache { 35 | required string key_prefix = 1; 36 | required uint64 max_ts = 2; 37 | repeated string except_keys = 3; 38 | }; 39 | 40 | message DeleteEntryResponse { 41 | required int32 status_code = 1; 42 | }; 43 | 44 | message RegisterRequest { 45 | // nothing 46 | }; 47 | 48 | message QueryTsRequest { 49 | // nothing 50 | }; 51 | 52 | message QueryTsResponse { 53 | required int32 status_code = 1; 54 | required uint64 timestamp = 2; 55 | }; 56 | 57 | message RegisterResponse { 58 | required int32 status_code = 1; 59 | }; 60 | 61 | service GlobalCacheService { 62 | rpc GetEntryFromReadCache(GetEntryRequest) returns (GetEntryResponse); 63 | rpc PutEntryFromReadCache(PutEntryRequest) returns (PutEntryResponse); 64 | rpc DeleteEntryFromReadCache(DeleteEntryRequest) returns (DeleteEntryResponse); 65 | 66 | rpc GetEntryFromWriteCache(GetEntryRequest) returns (GetEntryResponse); 67 | rpc PutEntryFromWriteCache(PutEntryRequest) returns (PutEntryResponse); 68 | rpc DeleteEntryFromWriteCache(DeleteEntryRequestForWriteCache) returns (DeleteEntryResponse); 69 | rpc QueryTsFromWriteCache(QueryTsRequest) returns (QueryTsResponse); 70 | 71 | rpc Register(RegisterRequest) returns (RegisterResponse); 72 | }; 73 | -------------------------------------------------------------------------------- /hadoop_posix/.bazelrc: -------------------------------------------------------------------------------- 1 | build --verbose_failures 2 | 3 | build --define=with_glog=true --define=libunwind=true 4 | build --copt -DHAVE_ZLIB=1 --copt -DGFLAGS_NS=google --copt -DUSE_BTHREAD_MUTEX 5 | build --cxxopt -Wno-error=format-security 6 | build:gcc7-later --cxxopt -faligned-new 7 | build --incompatible_blacklisted_protos_requires_proto_info=false 8 | build --copt=-fdiagnostics-color=always 9 | 10 | build:sanitize-common --strip=never 11 | build:sanitize-common --copt -O1 12 | build:sanitize-common --copt -g 13 | build:sanitize-common --copt -fno-omit-frame-pointer 14 | 15 | build:asan --config=sanitize-common 16 | build:asan --copt -fsanitize=address 17 | build:asan --copt -DADDRESS_SANITIZER 18 | build:asan --linkopt -fsanitize=address 19 | 20 | build:asan --config=sanitize-common 21 | build:msan --copt -fsanitize=memory 22 | build:msan --copt -fsanitize=undefined 23 | build:msan --linkopt -fsanitize=address 24 | build:msan --linkopt -fsanitize=undefined 25 | 26 | run --copt=-fdiagnostics-color=always 27 | 28 | -------------------------------------------------------------------------------- /hadoop_posix/.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files 2 | *.slo 3 | *.lo 4 | *.o 5 | *.obj 6 | 7 | # Precompiled Headers 8 | *.gch 9 | *.pch 10 | 11 | # Compiled Dynamic libraries 12 | *.so 13 | *.dylib 14 | *.dll 15 | 16 | # Fortran module files 17 | *.mod 18 | *.smod 19 | 20 | # Compiled Static libraries 21 | *.lai 22 | *.la 23 | *.a 24 | *.lib 25 | 26 | # Executables 27 | *.exe 28 | *.out 29 | *.app 30 | 31 | # clion 32 | .idea 33 | cmake-build-debug 34 | CMakeLists.txt 35 | bin 36 | *.temp 37 | .clwb 38 | *.swp 39 | 40 | # cscope 41 | cscope.out 42 | cscope.in.out 43 | cscope.po.out 44 | cscope.files 45 | tags 46 | 47 | .sconsign.dblite 48 | build.err 49 | build.log 50 | 51 | # python 52 | *.pyc 53 | 54 | 55 | # protobuf 56 | *.pb.h 57 | *.pb.cc 58 | 59 | # bazel 60 | bazel-* 61 | bazel-bin 62 | hadoop_posix 63 | bazel-genfiles 64 | bazel-out 65 | bazel-testlogs 66 | 67 | # vscode 68 | .vscode 69 | 70 | # vimprj 71 | .vimprj/ 72 | 73 | #log 74 | *.log 75 | runlog/ 76 | 77 | # history 78 | .gdb_history 79 | 80 | build 81 | 82 | *.class 83 | hadoop_sdk/java/target/ 84 | hadoop_sdk/java/native/build 85 | hadoop_sdk/libjycachefs/examples/bin 86 | hadoop_sdk/java/dependency-reduced-pom.xml 87 | hadoop_sdk/output/ 88 | hadoop-test/ 89 | nnbench-test/ 90 | META-INF/ 91 | com/ 92 | 93 | -------------------------------------------------------------------------------- /hadoop_posix/BUILD: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2020 # Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | # 18 | # Copyright 2017 The Abseil Authors. 19 | # 20 | # Licensed under the Apache License, Version 2.0 (the "License"); 21 | # you may not use this file except in compliance with the License. 22 | # You may obtain a copy of the License at 23 | # 24 | # https://www.apache.org/licenses/LICENSE-2.0 25 | # 26 | # Unless required by applicable law or agreed to in writing, software 27 | # distributed under the License is distributed on an "AS IS" BASIS, 28 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 29 | # See the License for the specific language governing permissions and 30 | # limitations under the License. 31 | 32 | package(default_visibility = ["//visibility:public"]) 33 | 34 | config_setting( 35 | name = "clang_compiler", 36 | flag_values = { 37 | "@bazel_tools//tools/cpp:compiler": "clang", 38 | }, 39 | ) 40 | 41 | config_setting( 42 | name = "gcc_compiler_arm64", 43 | define_values = { 44 | "compile_flags": "gcc_arm64", 45 | } 46 | ) 47 | 48 | config_setting( 49 | name = "base_flags_arm64", 50 | define_values = { 51 | "compile_base_flags": "base_flags_arm64", 52 | } 53 | ) 54 | 55 | config_setting( 56 | name = "base_flags", 57 | define_values = { 58 | "compile_base_flags": "base_flags", 59 | } 60 | ) 61 | 62 | -------------------------------------------------------------------------------- /hadoop_posix/README.md: -------------------------------------------------------------------------------- 1 | ## 简介 2 | 3 | 此项目为JYCacheFS 提供hadoop支持，支持hadoop的应用，可以通过hadoop接口在JYCacheFS上存取数据。 4 | 5 | 6 | 7 | ## 部署 8 | 9 | ### 前置要求 10 | 11 | 1. 已经获取到此项目的发行包：jycachefs-hadoop-1.0-SNAPSHOT.jar 12 | 2. 已经部署并启动JYCacheFS，挂载路径：/home/jycachefs/mnt/ 13 | 14 | ### 安装 15 | 16 | 1. 复制 jycachefs-hadoop-1.0-SNAPSHOT.jar 包到相应hadoop程序的目录下 17 | 18 | ```bash 19 | cp jycachefs-hadoop-1.0-SNAPSHOT.jar /opt/hadoop/share/hadoop/common/lib/ 20 | cp jycachefs-hadoop-1.0-SNAPSHOT.jar /opt/hadoop/share/hadoop/yarn/lib/ 21 | cp jycachefs-hadoop-1.0-SNAPSHOT.jar /opt/spark/jars/ 22 | ``` 23 | 24 | 2. 修改hadoop的配置文件(/opt/hadoop/etc/hadoop/core-site.xml) 25 | 26 | ```xml 27 | 28 | 29 | 42 | 43 | 44 | 45 | 46 | 47 | fs.jycachefs.impl 48 | io.aisoft9.jycache.fs.hadoop.JYCacheFileSystem 49 | 50 | 51 | fs.AbstractFileSystem.jycachefs.impl 52 | io.aisoft9.jycache.fs.hadoop.JYCacheFS 53 | 54 | 55 | jycachefs.name 56 | jycachefs 57 | 58 | 59 | fs.defaultFS 60 | jycachefs://jycachefs 61 | 62 | 63 | ``` 64 | 65 | 3. 在相应应用中修改文件的访问路径uri为: jycachefs://jycachefs 66 | 67 | ### 验证 68 | 69 | 1. 查看是否能通过hadoop接口正常访问JYCacheFS 70 | 71 | ```bash 72 | hadoop fs -ls / 73 | ``` 74 | 75 | 76 | 77 | ## 构建 78 | 79 | 构建jycachefs-hadoop-1.0-SNAPSHOT.jar, 若已获取jar包，则无需关注此项。 80 | 81 | ### 前置要求 82 | 83 | 1. 已安装 gcc-10，bazel， maven 等编译构建所需软件。 84 | 85 | ### 启动构建 86 | 87 | 1. 若GCC-10不在默认路径(/usr/local/gcc-10.2.0)，则需要手动指定路径。执行构建命令： 88 | 89 | ```bash 90 | cd hadoop_posix 91 | GCC_10_DIR=/home/programs/gcc-10.2.0/ bash util/sdk.sh 92 | ``` 93 | 94 | ### 构建完成 95 | 96 | 1. 构建完成，jar包生成的路径： 97 | 98 | ```bash 99 | cd hadoop_posix 100 | ls -al hadoop_sdk/output/jycachefs-hadoop-1.0-SNAPSHOT.jar 101 | ``` 102 | 103 | -------------------------------------------------------------------------------- /hadoop_posix/hadoop_sdk/.clang-format: -------------------------------------------------------------------------------- 1 | DisableFormat: true 2 | SortIncludes: Never 3 | -------------------------------------------------------------------------------- /hadoop_posix/hadoop_sdk/README.md: -------------------------------------------------------------------------------- 1 | Hadoop SDK 2 | === 3 | 4 | How to build 5 | --- 6 | 7 | ``` bash 8 | $ git clone git@github.com:aisoft9/jycache.git 9 | $ make playground 10 | $ make ci-dep stor=fs 11 | $ make sdk 12 | ``` 13 | 14 | It will generate a jar package after build success: 15 | 16 | ``` 17 | Build SDK success => /jycache/jycachefs/sdk/output/jycachefs-hadoop-1.0-SNAPSHOT.jar 18 | ``` 19 | -------------------------------------------------------------------------------- /hadoop_posix/hadoop_sdk/java/native/BUILD: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2023 # Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | load("//:copts.bzl", "JYCACHE_DEFAULT_COPTS") 18 | 19 | cc_binary( 20 | name = "jycachefs_jni", 21 | srcs = glob([ 22 | "*.h", 23 | "*.cpp", 24 | ]), 25 | visibility = ["//visibility:public"], 26 | copts = JYCACHE_DEFAULT_COPTS, 27 | linkopts = [ 28 | "-Wl,-rpath=/tmp/libjycachefs,--disable-new-dtags", 29 | "-L/usr/lib/x86_64-linux-gnu/", 30 | # "-lboost_filesystem" 31 | # "-lhashkit", 32 | ], 33 | deps = [ 34 | "@com_google_absl//absl/cleanup", 35 | "@jni//:copy_jni_hdr_lib", 36 | "//hadoop_sdk/libjycachefs:jycachefs_lib", 37 | ], 38 | linkshared = True, 39 | ) 40 | -------------------------------------------------------------------------------- /hadoop_posix/hadoop_sdk/java/src/main/java/io/aisoft9/jycache/fs/flink/JYCacheFileSystemFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023 # Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aisoft9.jycache.fs.flink; 18 | 19 | import io.aisoft9.jycache.fs.hadoop.JYCacheFileSystem; 20 | import org.apache.flink.core.fs.FileSystem; 21 | import org.apache.flink.core.fs.FileSystemFactory; 22 | import org.apache.flink.runtime.fs.hdfs.HadoopFileSystem; 23 | import org.apache.hadoop.conf.Configuration; 24 | 25 | import java.io.IOException; 26 | import java.net.URI; 27 | 28 | public class JYCacheFileSystemFactory implements FileSystemFactory { 29 | private org.apache.hadoop.conf.Configuration conf = new Configuration(); 30 | private static final String JYCACHE_FS_CONFIG_PREFIXES = "jycachefs."; 31 | private static final String FLINK_CONFIG_PREFIXES = "fs."; 32 | public static String SCHEME = "jycachefs"; 33 | 34 | @Override 35 | public void configure(org.apache.flink.configuration.Configuration config) { 36 | config.keySet() 37 | .stream() 38 | .filter(key -> key.startsWith(JYCACHE_FS_CONFIG_PREFIXES) || key.startsWith(FLINK_CONFIG_PREFIXES)) 39 | .forEach(key -> conf.set(key, config.getString(key, ""))); 40 | } 41 | 42 | @Override 43 | public String getScheme() { 44 | return SCHEME; 45 | } 46 | 47 | @Override 48 | public FileSystem create(URI uri) throws IOException { 49 | System.out.println("JYCacheFileSystemFactory is : " + this); 50 | System.out.println("create filesystem in JYCacheFileSystemFactory"); 51 | JYCacheFileSystem fs = new JYCacheFileSystem(); 52 | fs.initialize(uri, conf); 53 | return new HadoopFileSystem(fs); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /hadoop_posix/hadoop_sdk/java/src/main/java/io/aisoft9/jycache/fs/flink/JYCacheFileSystemTableFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023 # Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aisoft9.jycache.fs.flink; 18 | 19 | import org.apache.flink.connector.file.table.FileSystemTableFactory; 20 | 21 | public class JYCacheFileSystemTableFactory extends FileSystemTableFactory { 22 | @Override 23 | public String factoryIdentifier() { 24 | return JYCacheFileSystemFactory.SCHEME; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /hadoop_posix/hadoop_sdk/java/src/main/java/io/aisoft9/jycache/fs/hadoop/JYCacheFS.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023 # Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /* 18 | * Project: JYCache 19 | * Created Date: 2023-08-01 20 | * Author: Media Bigdata 21 | */ 22 | 23 | package io.aisoft9.jycache.fs.hadoop; 24 | 25 | import org.apache.hadoop.conf.Configuration; 26 | import org.apache.hadoop.fs.AbstractFileSystem; 27 | import org.apache.hadoop.fs.DelegateToFileSystem; 28 | import io.aisoft9.jycache.fs.flink.JYCacheFileSystemFactory; 29 | 30 | import java.io.IOException; 31 | import java.net.URI; 32 | import java.net.URISyntaxException; 33 | 34 | /** 35 | * The JYCacheFS implementation of AbstractFileSystem. 36 | * This impl delegates to the old FileSystem 37 | */ 38 | public class JYCacheFS extends DelegateToFileSystem { 39 | /** 40 | * This constructor has the signature needed by 41 | * {@link AbstractFileSystem#createFileSystem(URI, Configuration)}. 42 | * 43 | * @param theUri which must be that of localFs 44 | * @param conf 45 | * @throws IOException 46 | * @throws URISyntaxException 47 | */ 48 | JYCacheFS(final URI theUri, final Configuration conf) throws IOException, 49 | URISyntaxException { 50 | super(theUri, new JYCacheFileSystem(conf), conf, JYCacheFileSystemFactory.SCHEME, false); 51 | System.out.println("JYCacheFS new..."); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /hadoop_posix/hadoop_sdk/java/src/main/java/io/aisoft9/jycache/fs/hadoop/JYCacheFSProto.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023 # Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /* 18 | * Project: JYCache 19 | * Created Date: 2023-08-01 20 | * Author: Media Bigdata 21 | */ 22 | 23 | package io.aisoft9.jycache.fs.hadoop; 24 | 25 | import org.apache.hadoop.conf.Configuration; 26 | import org.apache.hadoop.fs.Path; 27 | import io.aisoft9.jycache.fs.libfs.JYCacheFSStat; 28 | import io.aisoft9.jycache.fs.libfs.JYCacheFSStatVFS; 29 | 30 | import java.io.IOException; 31 | import java.net.URI; 32 | 33 | abstract class JYCacheFSProto { 34 | // init* 35 | abstract void initialize(URI uri, Configuration conf) throws IOException; 36 | abstract void shutdown() throws IOException; 37 | // directory* 38 | abstract void mkdirs(Path path, int mode) throws IOException; 39 | abstract void rmdir(Path path) throws IOException; 40 | abstract String[] listdir(Path path) throws IOException; 41 | // file* 42 | abstract int open(Path path, int flags, int mode) throws IOException; 43 | abstract long lseek(int fd, long offset, int whence) throws IOException; 44 | abstract int write(int fd, byte[] buf, long size, long offset) throws IOException; 45 | abstract int read(int fd, byte[] buf, long size, long offset) throws IOException; 46 | abstract void fsync(int fd) throws IOException; 47 | abstract void close(int fd) throws IOException; 48 | abstract void unlink(Path path) throws IOException; 49 | // others 50 | abstract void statfs(Path path, JYCacheFSStatVFS stat) throws IOException; 51 | abstract void lstat(Path path, JYCacheFSStat stat) throws IOException; 52 | abstract void fstat(int fd, JYCacheFSStat stat) throws IOException; 53 | abstract void setattr(Path path, JYCacheFSStat stat, int mask) throws IOException; 54 | abstract void chmod(Path path, int mode) throws IOException; 55 | abstract void chown(Path path, int uid, int gid) throws IOException; 56 | abstract void rename(Path src, Path dst) throws IOException; 57 | } 58 | -------------------------------------------------------------------------------- /hadoop_posix/hadoop_sdk/java/src/main/java/io/aisoft9/jycache/fs/hadoop/permission/Permission.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023 # Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /* 18 | * Project: JYCache 19 | * Created Date: 2023-08-01 20 | * Author: Xianfei Cao (caoxianfei1) 21 | */ 22 | 23 | package io.aisoft9.jycache.fs.hadoop.permission; 24 | 25 | import org.apache.hadoop.conf.Configuration; 26 | 27 | import java.io.IOException; 28 | 29 | public class Permission { 30 | private static User user = null; 31 | private static Group group = null; 32 | 33 | public Permission() { 34 | user = new User(); 35 | group = new Group(); 36 | } 37 | 38 | public void initialize(Configuration conf) throws IOException { 39 | user.initialize(null); 40 | group.initialize(null); 41 | } 42 | 43 | public int getCurrentUid() { 44 | return user.getCurrentUid(); 45 | } 46 | 47 | public int getUid(String username) { 48 | return user.getUid(username); 49 | } 50 | 51 | public String getUsername(int uid) { 52 | return user.getUsername(uid); 53 | } 54 | 55 | public int[] getCurrentGids() { 56 | return group.getCurrentGids(); 57 | } 58 | 59 | public int getGid(String groupname) { 60 | return group.getGid(groupname); 61 | } 62 | 63 | public String getGroupname(int gid) { 64 | return group.getGroupname(gid); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /hadoop_posix/hadoop_sdk/java/src/main/java/io/aisoft9/jycache/fs/hadoop/permission/User.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023 # Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /* 18 | * Project: JYCache 19 | * Created Date: 2023-08-01 20 | * Author: Xianfei Cao (caoxianfei1) 21 | */ 22 | 23 | package io.aisoft9.jycache.fs.hadoop.permission; 24 | 25 | import org.apache.hadoop.fs.Path; 26 | import org.apache.hadoop.security.UserGroupInformation; 27 | 28 | import java.util.HashMap; 29 | import java.io.IOException; 30 | import java.util.List; 31 | 32 | public class User { 33 | private HashMap usernames; 34 | private HashMap userIDs; 35 | private UserGroupInformation ugi; 36 | 37 | private static final String SUPER_USERNAME = "hdfs"; 38 | private static final int SUPER_UID = 0; 39 | 40 | public User() { 41 | usernames = new HashMap(); 42 | userIDs = new HashMap(); 43 | } 44 | 45 | private int finger(String groupname) { 46 | return Math.abs(groupname.hashCode()); 47 | } 48 | 49 | private void addUser(String username, int uid) { 50 | usernames.put(username, uid); 51 | userIDs.put(uid, username); 52 | } 53 | 54 | private void loadUserFromSystem() throws IOException { 55 | List users = Helper.getAllUsers(); 56 | for (Entry user : users) { 57 | if (user.id == 0) { 58 | user.id = finger(user.name); 59 | } 60 | addUser(user.name, user.id); 61 | } 62 | addUser(SUPER_USERNAME, SUPER_UID); 63 | } 64 | 65 | private void loadUserFromFile() throws IOException { 66 | // TODO: implement it 67 | } 68 | 69 | public void initialize(Path path) throws IOException { 70 | this.ugi = UserGroupInformation.getCurrentUser(); 71 | if (path != null) { 72 | loadUserFromFile(); 73 | } else { 74 | loadUserFromSystem(); 75 | } 76 | } 77 | 78 | public int getCurrentUid() { 79 | String username = ugi.getShortUserName(); 80 | return getUid(username); 81 | } 82 | 83 | public int getUid(String username) { 84 | Integer gid = usernames.get(username); 85 | if (null == gid) { 86 | gid = finger(username); 87 | addUser(username, gid); 88 | } 89 | return gid; 90 | } 91 | 92 | public String getUsername(int uid) { 93 | String username = userIDs.get(uid); 94 | if (null == username || username.isEmpty()) { 95 | return String.valueOf(uid); 96 | } 97 | return username; 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /hadoop_posix/hadoop_sdk/java/src/main/java/io/aisoft9/jycache/fs/libfs/JYCacheFSStat.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023 # Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /* 18 | * Project: JYCache 19 | * Created Date: 2023-07-07 20 | * Author: Jingli Chen (Wine93) 21 | */ 22 | 23 | package io.aisoft9.jycache.fs.libfs; 24 | 25 | public class JYCacheFSStat { 26 | public boolean is_file; /* S_ISREG */ 27 | public boolean is_directory; /* S_ISDIR */ 28 | public boolean is_symlink; /* S_ISLNK */ 29 | 30 | public int mode; 31 | public int uid; 32 | public int gid; 33 | public long size; 34 | public long blksize; 35 | public long blocks; 36 | public long a_time; 37 | public long m_time; 38 | 39 | public boolean isFile() { 40 | return is_file; 41 | } 42 | 43 | public boolean isDir() { 44 | return is_directory; 45 | } 46 | 47 | public boolean isSymlink() { 48 | return is_symlink; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /hadoop_posix/hadoop_sdk/java/src/main/java/io/aisoft9/jycache/fs/libfs/JYCacheFSStatVFS.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023 # Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /* 18 | * Project: JYCache 19 | * Created Date: 2023-07-07 20 | * Author: Jingli Chen (Wine93) 21 | */ 22 | 23 | package io.aisoft9.jycache.fs.libfs; 24 | 25 | public class JYCacheFSStatVFS { 26 | public long bsize; 27 | public long frsize; 28 | public long blocks; 29 | public long bavail; 30 | public long files; 31 | public long fsid; 32 | public long namemax; 33 | } 34 | -------------------------------------------------------------------------------- /hadoop_posix/hadoop_sdk/java/src/test/java/io/aisoft9/jycache/fs/others/TestOthers.java: -------------------------------------------------------------------------------- 1 | package io.aisoft9.jycache.fs.others; 2 | 3 | import junit.framework.TestCase; 4 | import io.aisoft9.jycache.fs.hadoop.JYCacheFileSystem; 5 | 6 | import java.io.IOException; 7 | import java.net.URL; 8 | 9 | public class TestOthers extends TestCase { 10 | public void testHelloWorld() throws IOException { 11 | URL location = JYCacheFileSystem.class.getProtectionDomain().getCodeSource().getLocation(); 12 | System.out.println("Hello World"); 13 | System.out.println(location.getPath()); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /hadoop_posix/hadoop_sdk/libjycachefs/BUILD: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2023 # Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | load("//:copts.bzl", "JYCACHE_DEFAULT_COPTS") 18 | 19 | 20 | 21 | cc_binary( 22 | name = "jycachefs", 23 | srcs = [ 24 | "libjycachefs.h", 25 | "libjycachefs.cpp", 26 | ], 27 | copts = JYCACHE_DEFAULT_COPTS, 28 | visibility = ["//visibility:public"], 29 | deps = [ 30 | # "//jycachefs/src/client/vfs:vfs", 31 | ], 32 | linkshared = True, 33 | ) 34 | 35 | cc_library( 36 | name = "jycachefs_lib", 37 | srcs = [ 38 | "libjycachefs.h", 39 | "libjycachefs.cpp", 40 | ], 41 | includes = [ 42 | "include", 43 | ], 44 | copts = JYCACHE_DEFAULT_COPTS, 45 | visibility = ["//visibility:public"], 46 | deps = [ 47 | # "@client//:jy_s3fs_lib" 48 | ], 49 | ) 50 | -------------------------------------------------------------------------------- /hadoop_posix/hadoop_sdk/libjycachefs/README.md: -------------------------------------------------------------------------------- 1 | libjycachefs 2 | === 3 | 4 | SDK C/C++ library for JYCacheFS. 5 | 6 | Example 7 | === 8 | 9 | ```c 10 | #include "libjycachefs.h" 11 | 12 | int instance = jycachefs_create(); 13 | jycachefs_conf_set(instance, "s3.ak", "xxx") 14 | jycachefs_conf_set(instance, "s3.sk", "xxx") 15 | 16 | ... 17 | 18 | int rc = jycachefs_mount(instance, "fsname", "/); 19 | if (rc != 0) { 20 | // mount failed 21 | } 22 | 23 | rc = jycachefs_mkdir(instance_ptr, "/mydir") 24 | if (rc != 0) { 25 | // mkdir failed 26 | } 27 | ``` 28 | 29 | See [examples](examples) for more examples. 30 | -------------------------------------------------------------------------------- /hadoop_posix/hadoop_sdk/libjycachefs/client.BUILD: -------------------------------------------------------------------------------- 1 | cc_library( 2 | name = "jy_s3fs_lib", 3 | srcs = glob([ 4 | "s3fs/lib/libs3fs_lib.so", 5 | ]), 6 | hdrs = glob([ 7 | "s3fs/include/s3fs_lib.h", 8 | ]), 9 | visibility = [ 10 | "//visibility:public" 11 | ] 12 | ) -------------------------------------------------------------------------------- /hadoop_posix/hadoop_sdk/libjycachefs/examples/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2024 Jingli Chen (Wine93), # Inc. 2 | 3 | root= $(shell echo $${PWD%/jycachefs/sdk/libjycachefs/examples}) 4 | so_path?= $(root)/bazel-bin/jycachefs/sdk/libjycachefs 5 | hrd_opt?= -I/jycache 6 | build_opt?= -L$(so_path) 7 | link_opt?= -Wl,-rpath=$(so_path) -Wall 8 | flags?= $(hrd_opt) $(build_opt) $(link_opt) 9 | targets= mkdir rmdir ls touch read write unlink append rename stat statfs fstat chmod 10 | build_copt= 11 | 12 | host_arch := $(shell uname -m | \ 13 | sed -e s/i.86/x86/ \ 14 | -e s/sun4u/sparc64/ \ 15 | -e s/arm.*/arm/ \ 16 | -e s/sa110/arm/ \ 17 | -e s/ppc64/powerpc/ \ 18 | -e s/ppc/powerpc/ \ 19 | -e s/macppc/powerpc/\ 20 | -e s/sh.*/sh/) 21 | ifeq ($(host_arch), aarch64) 22 | build_copt= --define compile_base_flags=base_flags_arm64 --define compile_flags=gcc_arm64 --copt=-fno-gcse --copt=-fno-cse-follow-jumps --copt=-fno-move-loop-invariants 23 | endif 24 | 25 | 26 | libjycachefs: 27 | mkdir -p bin 28 | bazel build \ 29 | --config=gcc7-later \ 30 | --compilation_mode=opt \ 31 | --copt -g \ 32 | --copt -DUNINSTALL_SIGSEGV=1 \ 33 | --copt -DCLIENT_CONF_PATH="\"$(root)/jycachefs/conf/client.conf\"" \ 34 | $(build_copt) \ 35 | //jycachefs/sdk/java/native:jycachefs_jni 36 | 37 | $(targets): libjycachefs 38 | gcc ${flags} $@.c -o bin/$@ -ljycachefs 39 | 40 | all: $(targets) 41 | 42 | 43 | -------------------------------------------------------------------------------- /hadoop_posix/hadoop_sdk/libjycachefs/examples/append.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "common.h" 4 | 5 | int 6 | main(int argc, char** argv) { 7 | exact_args(argc, 2); 8 | 9 | uintptr_t instance = jycachefs_create(); 10 | load_cfg_from_environ(instance); 11 | 12 | char* fsname = get_filesystem_name(); 13 | char* mountpoint = get_mountpoint(); 14 | int rc = jycachefs_mount(instance, fsname, mountpoint); 15 | if (rc != 0) { 16 | fprintf(stderr, "mount failed: retcode = %d\n", rc); 17 | return rc; 18 | } 19 | 20 | int fd = jycachefs_open(instance, argv[1], O_WRONLY, 0777); 21 | if (fd != 0) { 22 | rc = fd; 23 | fprintf(stderr, "open failed: retcode = %d\n", rc); 24 | return rc; 25 | } 26 | 27 | rc = jycachefs_lseek(instance, fd, 0, SEEK_END); 28 | if (rc != 0) { 29 | fprintf(stderr, "lseek failed: retcode = %d\n", rc); 30 | return rc; 31 | } 32 | 33 | ssize_t n = jycachefs_write(instance, fd, argv[2], strlen(argv[2])); 34 | if (n < 0) { 35 | rc = n; 36 | fprintf(stderr, "write failed: retcode = %d\n", rc); 37 | return rc; 38 | } else if (n != strlen(argv[2])) { 39 | fprintf(stderr, "write failed: %zd != %zu\n", n, strlen(argv[2])); 40 | return -1; 41 | } 42 | 43 | rc = jycachefs_close(instance, fd); 44 | if (rc != 0) { 45 | fprintf(stderr, "close failed: retcode = %d\n", rc); 46 | } 47 | return 0; 48 | } 49 | -------------------------------------------------------------------------------- /hadoop_posix/hadoop_sdk/libjycachefs/examples/chmod.c: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | 3 | int 4 | main(int argc, char** argv) { 5 | exact_args(argc, 2); 6 | 7 | uintptr_t instance = jycachefs_create(); 8 | load_cfg_from_environ(instance); 9 | 10 | char* fsname = get_filesystem_name(); 11 | char* mountpoint = get_mountpoint(); 12 | int rc = jycachefs_mount(instance, fsname, mountpoint); 13 | if (rc != 0) { 14 | fprintf(stderr, "mount failed: retcode = %d\n", rc); 15 | return rc; 16 | } 17 | 18 | rc = jycachefs_chmod(instance, argv[1], atoi(argv[2])); 19 | if (rc != 0) { 20 | fprintf(stderr, "chmod failed: retcode = %d\n", rc); 21 | return rc; 22 | } 23 | return 0; 24 | } 25 | -------------------------------------------------------------------------------- /hadoop_posix/hadoop_sdk/libjycachefs/examples/common.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023 # Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef JYCACHEFS_SDK_LIBJYCACHEFS_EXAMPLES_COMMON_H_ 18 | #define JYCACHEFS_SDK_LIBJYCACHEFS_EXAMPLES_COMMON_H_ 19 | 20 | #include 21 | #include 22 | #include 23 | 24 | #include "jycachefs/sdk/libjycachefs/libjycachefs.h" 25 | 26 | const char* KEY_FSNAME = "JYCACHEFS_FSNAME"; 27 | const char* KEY_S3_AK = "s3.ak"; 28 | const char* KEY_S3_SK = "s3.sk"; 29 | const char* KEY_S3_ENDPOINT = "s3.endpoint"; 30 | const char* KEY_S3_BUCKET = "s3.bucket_name"; 31 | const char* KEY_MDS_ADDRS = "mdsOpt.rpcRetryOpt.addrs"; 32 | 33 | char* 34 | require_string(const char* name) { 35 | char* value = getenv(name); 36 | if (strlen(value) == 0) { 37 | fprintf(stderr, "require %s\n", name); 38 | exit(1); 39 | } 40 | return value; 41 | } 42 | 43 | char* 44 | get_filesystem_name() { 45 | return require_string(KEY_FSNAME); 46 | } 47 | 48 | char* 49 | get_mountpoint() { 50 | return "/"; 51 | } 52 | 53 | void 54 | load_cfg_from_environ(uintptr_t instance) { 55 | jycachefs_conf_set(instance, KEY_S3_AK, require_string(KEY_S3_AK)); 56 | jycachefs_conf_set(instance, KEY_S3_SK, require_string(KEY_S3_SK)); 57 | jycachefs_conf_set(instance, KEY_S3_ENDPOINT, 58 | require_string(KEY_S3_ENDPOINT)); 59 | jycachefs_conf_set(instance, KEY_S3_BUCKET, require_string(KEY_S3_BUCKET)); 60 | jycachefs_conf_set(instance, KEY_MDS_ADDRS, require_string(KEY_MDS_ADDRS)); 61 | jycachefs_conf_set(instance, "fs.accessLogging", "true"); 62 | jycachefs_conf_set(instance, "client.loglevel", "6"); 63 | jycachefs_conf_set(instance, "diskCache.diskCacheType", "0"); 64 | } 65 | 66 | void 67 | exact_args(int argc, int number) { 68 | if (--argc == number) { 69 | return; 70 | } 71 | 72 | fprintf(stderr, "requires exactly %d argument[s]\n", number); 73 | exit(1); 74 | } 75 | 76 | #endif // JYCACHEFS_SDK_LIBJYCACHEFS_EXAMPLES_COMMON_H_ 77 | -------------------------------------------------------------------------------- /hadoop_posix/hadoop_sdk/libjycachefs/examples/fstat.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "common.h" 4 | 5 | int 6 | main(int argc, char** argv) { 7 | exact_args(argc, 1); 8 | 9 | uintptr_t instance = jycachefs_create(); 10 | load_cfg_from_environ(instance); 11 | 12 | char* fsname = get_filesystem_name(); 13 | char* mountpoint = get_mountpoint(); 14 | int rc = jycachefs_mount(instance, fsname, mountpoint); 15 | if (rc != 0) { 16 | fprintf(stderr, "mount failed: retcode = %d\n", rc); 17 | return rc; 18 | } 19 | 20 | int fd = jycachefs_open(instance, argv[1], O_WRONLY, 0777); 21 | if (fd != 0) { 22 | rc = fd; 23 | fprintf(stderr, "open failed: retcode = %d\n", rc); 24 | return rc; 25 | } 26 | 27 | struct stat stat; 28 | rc = jycachefs_fstat(instance, fd, &stat); 29 | if (rc != 0) { 30 | fprintf(stderr, "fstat failed: retcode = %d\n", rc); 31 | return rc; 32 | } 33 | 34 | printf("ino=%d, size=%d\n", stat.st_ino, stat.st_size); 35 | return 0; 36 | } 37 | -------------------------------------------------------------------------------- /hadoop_posix/hadoop_sdk/libjycachefs/examples/ls.c: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | 3 | int 4 | main(int argc, char** argv) { 5 | exact_args(argc, 1); 6 | 7 | uintptr_t instance = jycachefs_create(); 8 | load_cfg_from_environ(instance); 9 | 10 | // mount 11 | char* fsname = get_filesystem_name(); 12 | char* mountpoint = get_mountpoint(); 13 | int rc = jycachefs_mount(instance, fsname, mountpoint); 14 | if (rc != 0) { 15 | fprintf(stderr, "mount failed: retcode = %d\n", rc); 16 | return rc; 17 | } 18 | 19 | // opendir 20 | dir_stream_t dir_stream; 21 | rc = jycachefs_opendir(instance, argv[1], &dir_stream); 22 | if (rc != 0) { 23 | fprintf(stderr, "opendir failed: retcode = %d\n", rc); 24 | return rc; 25 | } 26 | 27 | // readdir 28 | dirent_t dirent; 29 | for ( ;; ) { 30 | ssize_t n = jycachefs_readdir(instance, &dir_stream, &dirent); 31 | if (n < 0) { 32 | rc = n; 33 | fprintf(stderr, "readdir failed: retcode = %d\n", rc); 34 | break; 35 | } else if (n == 0) { 36 | break; 37 | } 38 | 39 | printf("%s: ino=%d size=%d\n", dirent.name, 40 | dirent.stat.st_ino, 41 | dirent.stat.st_size); 42 | } 43 | 44 | rc = jycachefs_closedir(instance, &dir_stream); 45 | if (rc != 0) { 46 | fprintf(stderr, "closedir failed: retcode = %d\n", rc); 47 | } 48 | return rc; 49 | } 50 | -------------------------------------------------------------------------------- /hadoop_posix/hadoop_sdk/libjycachefs/examples/mkdir.c: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | 3 | int 4 | main(int argc, char** argv) { 5 | exact_args(argc, 1); 6 | 7 | uintptr_t instance = jycachefs_create(); 8 | load_cfg_from_environ(instance); 9 | 10 | char* fsname = get_filesystem_name(); 11 | char* mountpoint = get_mountpoint(); 12 | int rc = jycachefs_mount(instance, fsname, mountpoint); 13 | if (rc != 0) { 14 | fprintf(stderr, "mount failed: retcode = %d\n", rc); 15 | return rc; 16 | } 17 | 18 | rc = jycachefs_mkdir(instance, argv[1], 0755); 19 | if (rc != 0) { 20 | fprintf(stderr, "mkdir failed: retcode = %d\n", rc); 21 | return rc; 22 | } 23 | 24 | return 0; 25 | } 26 | -------------------------------------------------------------------------------- /hadoop_posix/hadoop_sdk/libjycachefs/examples/read.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "common.h" 4 | 5 | #define MAX_BUFFER_SIZE 4096 + 5 6 | 7 | int 8 | main(int argc, char** argv) { 9 | exact_args(argc, 1); 10 | 11 | uintptr_t instance = jycachefs_create(); 12 | load_cfg_from_environ(instance); 13 | 14 | char* fsname = get_filesystem_name(); 15 | char* mountpoint = get_mountpoint(); 16 | int rc = jycachefs_mount(instance, fsname, mountpoint); 17 | if (rc != 0) { 18 | fprintf(stderr, "mount failed: retcode = %d\n", rc); 19 | return rc; 20 | } 21 | 22 | int fd = jycachefs_open(instance, argv[1], O_WRONLY, 0777); 23 | if (fd != 0) { 24 | rc = fd; 25 | fprintf(stderr, "open failed: retcode = %d\n", rc); 26 | return rc; 27 | } 28 | 29 | char buffer[MAX_BUFFER_SIZE]; 30 | for ( ;; ) { 31 | ssize_t n = jycachefs_read(instance, fd, buffer, sizeof(buffer)); 32 | if (n < 0) { 33 | rc = n; 34 | fprintf(stderr, "read failed: retcode = %d\n", rc); 35 | return rc; 36 | } else if (n == 0) { 37 | break; 38 | } 39 | 40 | buffer[n] = '\0'; 41 | printf("%s", buffer); 42 | } 43 | return 0; 44 | } 45 | -------------------------------------------------------------------------------- /hadoop_posix/hadoop_sdk/libjycachefs/examples/rename.c: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | 3 | int 4 | main(int argc, char** argv) { 5 | exact_args(argc, 2); 6 | 7 | uintptr_t instance = jycachefs_create(); 8 | load_cfg_from_environ(instance); 9 | 10 | char* fsname = get_filesystem_name(); 11 | char* mountpoint = get_mountpoint(); 12 | int rc = jycachefs_mount(instance, fsname, mountpoint); 13 | if (rc != 0) { 14 | fprintf(stderr, "mount failed: retcode = %d\n", rc); 15 | return rc; 16 | } 17 | 18 | rc = jycachefs_rename(instance, argv[1], argv[2]); 19 | if (rc != 0) { 20 | fprintf(stderr, "rename failed: retcode = %d\n", rc); 21 | return rc; 22 | } 23 | return 0; 24 | } 25 | -------------------------------------------------------------------------------- /hadoop_posix/hadoop_sdk/libjycachefs/examples/rmdir.c: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | 3 | int 4 | main(int argc, char** argv) { 5 | exact_args(argc, 1); 6 | 7 | uintptr_t instance = jycachefs_create(); 8 | load_cfg_from_environ(instance); 9 | 10 | char* fsname = get_filesystem_name(); 11 | char* mountpoint = get_mountpoint(); 12 | int rc = jycachefs_mount(instance, fsname, mountpoint); 13 | if (rc != 0) { 14 | fprintf(stderr, "mount failed: retcode = %d\n", rc); 15 | return rc; 16 | } 17 | 18 | rc = jycachefs_rmdir(instance, argv[1]); 19 | if (rc != 0) { 20 | fprintf(stderr, "rmdir failed: retcode = %d\n", rc); 21 | return rc; 22 | } 23 | 24 | return 0; 25 | } 26 | -------------------------------------------------------------------------------- /hadoop_posix/hadoop_sdk/libjycachefs/examples/stat.c: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | 3 | int 4 | main(int argc, char** argv) { 5 | exact_args(argc, 1); 6 | 7 | uintptr_t instance = jycachefs_create(); 8 | load_cfg_from_environ(instance); 9 | 10 | char* fsname = get_filesystem_name(); 11 | char* mountpoint = get_mountpoint(); 12 | int rc = jycachefs_mount(instance, fsname, mountpoint); 13 | if (rc != 0) { 14 | fprintf(stderr, "mount failed: retcode = %d\n", rc); 15 | return rc; 16 | } 17 | 18 | struct stat stat; 19 | rc = jycachefs_lstat(instance, argv[1], &stat); 20 | if (rc != 0) { 21 | fprintf(stderr, "stat failed: retcode = %d\n", rc); 22 | return rc; 23 | } 24 | 25 | printf("ino=%d, size=%d\n", stat.st_ino, stat.st_size); 26 | return 0; 27 | } 28 | -------------------------------------------------------------------------------- /hadoop_posix/hadoop_sdk/libjycachefs/examples/statfs.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "common.h" 4 | 5 | int 6 | main(int argc, char** argv) { 7 | exact_args(argc, 0); 8 | 9 | uintptr_t instance = jycachefs_create(); 10 | load_cfg_from_environ(instance); 11 | 12 | char* fsname = get_filesystem_name(); 13 | char* mountpoint = get_mountpoint(); 14 | int rc = jycachefs_mount(instance, fsname, mountpoint); 15 | if (rc != 0) { 16 | fprintf(stderr, "mount failed: retcode = %d\n", rc); 17 | return rc; 18 | } 19 | 20 | struct statvfs statvfs; 21 | rc = jycachefs_statfs(instance, &statvfs); 22 | if (rc != 0) { 23 | fprintf(stderr, "statvfs failed: retcode = %d\n", rc); 24 | return rc; 25 | } 26 | 27 | printf("fsid = %d\n", statvfs.f_fsid); 28 | return 0; 29 | } 30 | -------------------------------------------------------------------------------- /hadoop_posix/hadoop_sdk/libjycachefs/examples/touch.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "common.h" 4 | 5 | int 6 | main(int argc, char** argv) { 7 | exact_args(argc, 1); 8 | 9 | uintptr_t instance = jycachefs_create(); 10 | load_cfg_from_environ(instance); 11 | 12 | char* fsname = get_filesystem_name(); 13 | char* mountpoint = get_mountpoint(); 14 | int rc = jycachefs_mount(instance, fsname, mountpoint); 15 | if (rc != 0) { 16 | fprintf(stderr, "mount failed: retcode = %d\n", rc); 17 | return rc; 18 | } 19 | 20 | int fd = jycachefs_open(instance, argv[1], O_CREAT, 0644); 21 | if (fd < 0) { 22 | rc = fd; 23 | fprintf(stderr, "open failed: retcode = %d\n", rc); 24 | } 25 | return rc; 26 | } 27 | -------------------------------------------------------------------------------- /hadoop_posix/hadoop_sdk/libjycachefs/examples/unlink.c: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | 3 | int 4 | main(int argc, char** argv) { 5 | exact_args(argc, 1); 6 | 7 | uintptr_t instance = jycachefs_create(); 8 | load_cfg_from_environ(instance); 9 | 10 | char* fsname = get_filesystem_name(); 11 | char* mountpoint = get_mountpoint(); 12 | int rc = jycachefs_mount(instance, fsname, mountpoint); 13 | if (rc != 0) { 14 | fprintf(stderr, "mount failed: retcode = %d\n", rc); 15 | return rc; 16 | } 17 | 18 | rc = jycachefs_unlink(instance, argv[1]); 19 | if (rc != 0) { 20 | fprintf(stderr, "unlink failed: retcode = %d\n", rc); 21 | return rc; 22 | } 23 | 24 | return 0; 25 | } 26 | -------------------------------------------------------------------------------- /hadoop_posix/hadoop_sdk/libjycachefs/examples/write.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "common.h" 4 | 5 | int 6 | main(int argc, char** argv) { 7 | exact_args(argc, 2); 8 | 9 | uintptr_t instance = jycachefs_create(); 10 | load_cfg_from_environ(instance); 11 | 12 | char* fsname = get_filesystem_name(); 13 | char* mountpoint = get_mountpoint(); 14 | int rc = jycachefs_mount(instance, fsname, mountpoint); 15 | if (rc != 0) { 16 | fprintf(stderr, "mount failed: retcode = %d\n", rc); 17 | return rc; 18 | } 19 | 20 | int fd = jycachefs_open(instance, argv[1], O_WRONLY, 0777); 21 | if (fd != 0) { 22 | rc = fd; 23 | fprintf(stderr, "open failed: retcode = %d\n", rc); 24 | return rc; 25 | } 26 | 27 | ssize_t n = jycachefs_write(instance, fd, argv[2], strlen(argv[2])); 28 | if (n < 0) { 29 | rc = n; 30 | fprintf(stderr, "write failed: retcode = %d\n", rc); 31 | return rc; 32 | } else if (n != strlen(argv[2])) { 33 | fprintf(stderr, "write failed: %zd != %zu\n", n, strlen(argv[2])); 34 | return -1; 35 | } 36 | 37 | rc = jycachefs_close(instance, fd); 38 | if (rc != 0) { 39 | fprintf(stderr, "close failed: retcode = %d\n", rc); 40 | } 41 | return 0; 42 | } 43 | -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ ! -d "./JYCache_Env" ]; then 4 | case $(uname -m) in 5 | x86_64) 6 | wget https://madstorage.s3.cn-north-1.jdcloud-oss.com/JYCache_Env_x64_20241101.tgz 7 | md5=`md5sum JYCache_Env_x64_20241101.tgz | awk {'print $1'}` 8 | if [ "$md5" != "4c685f0465bbe08191c0308889b59826" ]; then 9 | echo 'JYCache_Env version inconsistency!' 10 | exit 1 11 | fi 12 | tar -zxvf JYCache_Env_x64_20241101.tgz 13 | ;; 14 | aarch64) 15 | wget https://madstorage.s3.cn-north-1.jdcloud-oss.com/JYCache_Env_arm64_20241101.tgz 16 | md5=`md5sum JYCache_Env_arm64_20241101.tgz | awk {'print $1'}` 17 | if [ "$md5" != "4fcb8fec4217869e66747cd7841de8dc" ]; then 18 | echo 'JYCache_Env version inconsistency!' 19 | exit 1 20 | fi 21 | tar -zxvf JYCache_Env_arm64_20241101.tgz 22 | ;; 23 | esac 24 | fi 25 | 26 | cp ./build/intercept/intercept_server JYCache_Env/ 27 | cp ./build/intercept/libintercept_client.so JYCache_Env/ 28 | cp ./build/global_cache/madfs_gc JYCache_Env/ 29 | cp ./build/global_cache/madfs_global_server JYCache_Env/ 30 | cp ./build/bin/s3fs JYCache_Env/ 31 | -------------------------------------------------------------------------------- /intercept/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | link_libraries(-lrt) 2 | 3 | find_library(ICEORYX_POSH_LIB iceoryx_posh PATHS ../thirdparties/iceoryx/lib) 4 | find_library(ICEORYX_HOOFS_LIB iceoryx_hoofs PATHS ../thirdparties/iceoryx/lib) 5 | find_library(ICEORYX_PLATFORM_LIB iceoryx_platform PATHS ../thirdparties/iceoryx/lib) 6 | 7 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") 8 | 9 | include_directories(${CMAKE_CURRENT_SOURCE_DIR}) 10 | include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../thirdparties/spdlog/include) 11 | 12 | add_subdirectory(common) 13 | add_subdirectory(internal) 14 | add_subdirectory(discovery) 15 | add_subdirectory(filesystem) 16 | add_subdirectory(registry) 17 | add_subdirectory(middleware) 18 | add_subdirectory(posix) 19 | 20 | add_executable(intercept_server server.cpp) 21 | target_link_libraries(intercept_server PUBLIC intercept_discovery intercept_internal common_lib intercept_filesystem intercept_middleware intercept_registry hybridcache_local madfs_global s3fs_lib ${ICEORYX_POSH_LIB} ${ICEORYX_HOOFS_LIB} ${ICEORYX_PLATFORM_LIB} -pthread -lcurl -lxml2 -lcrypto -ldl -laio -lrt) 22 | 23 | add_library(intercept_client SHARED client.cpp) 24 | target_link_libraries(intercept_client PUBLIC 25 | intercept_posix_interface_client 26 | -lsyscall_intercept 27 | -pthread 28 | ${ICEORYX_POSH_LIB} 29 | ${ICEORYX_HOOFS_LIB} 30 | ${ICEORYX_PLATFORM_LIB} 31 | -lrt 32 | -L${CMAKE_CURRENT_SOURCE_DIR}/../thirdparties/intercept/lib 33 | ) 34 | target_compile_options(intercept_client PUBLIC -DCLIENT_BUILD) 35 | -------------------------------------------------------------------------------- /intercept/common/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB COMMON_SOURCES *.cpp) 2 | 3 | add_library(common_lib ${COMMON_SOURCES}) 4 | target_include_directories(common_lib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) 5 | 6 | add_library(common_lib_client ${COMMON_SOURCES}) 7 | target_compile_options(common_lib_client PUBLIC -fPIC -DCLIENT_BUILD) 8 | target_include_directories(common_lib_client PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) 9 | -------------------------------------------------------------------------------- /intercept/discovery/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # discovery/CMakeLists.txt 2 | 3 | file(GLOB DISCOVERY_SOURCES *.cpp) 4 | 5 | find_library(ICEORYX_POSH_LIB NAMES iceoryx_posh PATHS ../../thirdparties/iceoryx/lib) 6 | find_library(ICEORYX_HOOFS_LIB NAMES iceoryx_hoofs PATHS ../../thirdparties/iceoryx/lib) 7 | find_library(ICEORYX_PLATFORM_LIB NAMES iceoryx_platform PATHS ../../thirdparties/iceoryx/lib) 8 | 9 | add_library(intercept_discovery ${DISCOVERY_SOURCES}) 10 | target_include_directories(intercept_discovery PUBLIC 11 | ${CMAKE_CURRENT_SOURCE_DIR} 12 | ${CMAKE_CURRENT_SOURCE_DIR}/../../thirdparties/iceoryx/include 13 | ) 14 | target_link_libraries(intercept_discovery PUBLIC intercept_internal ${ICEORYX_POSH_LIB}) 15 | 16 | add_library(intercept_discovery_client ${DISCOVERY_SOURCES}) 17 | target_include_directories(intercept_discovery_client PUBLIC 18 | ${CMAKE_CURRENT_SOURCE_DIR} 19 | ${CMAKE_CURRENT_SOURCE_DIR}/../../thirdparties/iceoryx/include 20 | ) 21 | target_compile_options(intercept_discovery_client PUBLIC -fPIC ) 22 | target_link_libraries(intercept_discovery_client PUBLIC -lrt intercept_internal_client ${ICEORYX_POSH_LIB} ${ICEORYX_HOOFS_LIB} ${ICEORYX_PLATFORM_LIB} ) -------------------------------------------------------------------------------- /intercept/discovery/discovery.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include "internal/metainfo.h" 8 | 9 | namespace intercept { 10 | namespace discovery { 11 | 12 | using intercept::internal::ServiceMetaInfo; 13 | 14 | 15 | 16 | // Discovery : use to discover the existing servers 17 | // and the servers to be deleted 18 | class Discovery { 19 | public: 20 | // Constructor 21 | Discovery() { 22 | // Initialization code 23 | } 24 | 25 | // Initialize the discovery 26 | virtual void Init() = 0; 27 | 28 | // Start the discovery loop 29 | virtual void Start() = 0; 30 | 31 | // Stop the discovery loop 32 | virtual void Stop() = 0; 33 | 34 | // Get the existing servers 35 | virtual std::vector GetServers() const { 36 | // Return the existing servers 37 | return std::vector(); 38 | } 39 | 40 | // Get the servers to be deleted 41 | virtual std::set GetServersToDelete() const { 42 | // Return the servers to be deleted 43 | return std::set(); 44 | } 45 | 46 | virtual std::vector FindServices(const ServiceMetaInfo& info) = 0; 47 | 48 | // Create a new server 49 | virtual void CreateServer(const ServiceMetaInfo& serverInfo) { 50 | // Create a new server using the serverInfo 51 | } 52 | 53 | // Delete a server 54 | virtual void DeleteServer(const ServiceMetaInfo& serverInfo) { 55 | // Delete a server using the serverInfo 56 | } 57 | 58 | protected: 59 | std::vector existingServers; 60 | std::set serversToDelete; 61 | bool DISCOVERY_RUNNING; 62 | }; 63 | 64 | } 65 | } 66 | 67 | -------------------------------------------------------------------------------- /intercept/discovery/iceoryx_discovery.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "discovery.h" 3 | #include "iceoryx_posh/runtime/service_discovery.hpp" 4 | 5 | namespace intercept { 6 | namespace discovery { 7 | 8 | class IceoryxDiscovery : public Discovery 9 | { 10 | public: 11 | IceoryxDiscovery(); 12 | 13 | virtual ~IceoryxDiscovery(); 14 | 15 | virtual void Init(); 16 | 17 | virtual void Start(); 18 | 19 | virtual void Stop(); 20 | 21 | virtual std::vector GetServers() const; 22 | 23 | virtual std::vector GetNewServers(const std::vector& oldservers, 24 | const std::vector& newservers); 25 | 26 | virtual std::set GetRemovedServers( 27 | const std::vector& oldservers, const std::vector& newservers); 28 | 29 | virtual std::vector FindServices(const ServiceMetaInfo& info); 30 | 31 | virtual void CreateServer(const ServiceMetaInfo& serverInfo); 32 | 33 | virtual void DeleteServer(const ServiceMetaInfo& serverInfo); 34 | 35 | private: 36 | iox::runtime::ServiceDiscovery serviceDiscovery_; 37 | }; 38 | 39 | } // namespace discovery 40 | } // namespace intercept 41 | 42 | -------------------------------------------------------------------------------- /intercept/filesystem/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | find_library(ICEORYX_POSH_LIB iceoryx_posh PATHS ../thirdparties/iceoryx/lib) 2 | find_library(ICEORYX_HOOFS_LIB iceoryx_hoofs PATHS ../thirdparties/iceoryx/lib) 3 | find_library(ICEORYX_PLATFORM_LIB iceoryx_platform PATHS ../thirdparties/iceoryx/lib) 4 | 5 | file(GLOB FILESYSTEM_SOURCES *.cpp) 6 | file(GLOB FILESYSTEM_HEADERS *.h) 7 | 8 | add_library(intercept_filesystem ${FILESYSTEM_SOURCES}) 9 | target_include_directories(intercept_filesystem PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) 10 | target_link_libraries(intercept_filesystem PUBLIC 11 | ${ICEORYX_POSH_LIB} ${ICEORYX_HOOFS_LIB} ${ICEORYX_PLATFORM_LIB} 12 | hybridcache_local madfs_global s3fs_lib ${THIRD_PARTY_LIBRARIES} common_lib 13 | -pthread 14 | -lcurl 15 | -lxml2 16 | -lcrypto 17 | -ldl 18 | -laio 19 | -lrt 20 | ) 21 | 22 | add_library(intercept_filesystem_client INTERFACE) 23 | target_include_directories(intercept_filesystem_client INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}) 24 | target_link_libraries(intercept_filesystem_client INTERFACE 25 | common_lib_client 26 | -lrt 27 | ) 28 | target_compile_options(intercept_filesystem_client INTERFACE -DCLIENT_BUILD) -------------------------------------------------------------------------------- /intercept/filesystem/abstract_filesystem.h: -------------------------------------------------------------------------------- 1 | #ifndef ABSTRACT_FILESYSTEM_H 2 | #define ABSTRACT_FILESYSTEM_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include "common/common.h" 11 | 12 | namespace intercept { 13 | namespace filesystem { 14 | 15 | using intercept::common::DirStream; 16 | class AbstractFileSystem { 17 | public: 18 | virtual ~AbstractFileSystem() {} 19 | virtual void Init() = 0; 20 | virtual void Shutdown() = 0; 21 | virtual int Open(const char* path, int flags, int mode) = 0; 22 | virtual ssize_t Read(int fd, void* buf, size_t count) = 0; 23 | virtual ssize_t Write(int fd, const void* buf, size_t count) = 0; 24 | virtual int Close(int fd) = 0; 25 | virtual off_t Lseek(int fd, off_t offset, int whence) = 0; 26 | virtual int Stat(const char* path, struct stat* st) = 0; 27 | virtual int Fstat(int fd, struct stat* st) = 0; 28 | virtual int Fsync(int fd) = 0; 29 | virtual int Truncate(const char* path, off_t length) = 0; 30 | virtual int Ftruncate(int fd, off_t length) = 0; 31 | virtual int Unlink(const char* path) = 0; 32 | virtual int Mkdir(const char* path, mode_t mode) = 0; 33 | virtual int Opendir(const char* path, DirStream* dirstream) = 0; 34 | virtual int Getdents(DirStream* dirstream, char* contents, size_t maxread, ssize_t* realbytes) = 0; 35 | virtual int Closedir(DirStream* dirstream) = 0; 36 | virtual int Rmdir(const char* path) = 0; 37 | virtual int Chmod(const char* path, mode_t mode) = 0; 38 | virtual int Chown(const char* path, uid_t owner, gid_t group) = 0; 39 | virtual int Rename(const char* oldpath, const char* newpath) = 0; 40 | virtual int Link(const char* oldpath, const char* newpath) = 0; 41 | virtual int Symlink(const char* oldpath, const char* newpath) = 0; 42 | virtual int Readlink(const char* path, char* buf, size_t bufsize) = 0; 43 | virtual int Utime(const char* path, const struct utimbuf* times) = 0; 44 | 45 | virtual ssize_t MultiRead(int fd, void* buf, size_t count) {} 46 | virtual ssize_t MultiWrite(int fd, const void* buf, size_t count) {} 47 | 48 | protected: 49 | virtual std::string NormalizePath(const std::string& path) = 0; 50 | }; 51 | 52 | } // namespace filesystem 53 | } // namespace intercept 54 | 55 | 56 | 57 | #endif // ABSTRACT_FILESYSTEM_H 58 | -------------------------------------------------------------------------------- /intercept/filesystem/curve_filesystem.h: -------------------------------------------------------------------------------- 1 | #ifndef CURVE_FILESYSTEM_H 2 | #define CURVE_FILESYSTEM_H 3 | 4 | #include "abstract_filesystem.h" 5 | namespace intercept { 6 | namespace filesystem { 7 | class CurveFileSystem : public AbstractFileSystem { 8 | public: 9 | CurveFileSystem(); 10 | ~CurveFileSystem() override; 11 | void Init() override; 12 | void Shutdown() override; 13 | int Open(const char* path, int flags, int mode) override; 14 | ssize_t Read(int fd, void* buf, size_t count) override; 15 | ssize_t Write(int fd, const void* buf, size_t count) override; 16 | int Close(int fd) override; 17 | off_t Lseek(int fd, off_t offset, int whence) override; 18 | int Stat(const char* path, struct stat* st) override; 19 | int Fstat(int fd, struct stat* st) override; 20 | int Fsync(int fd) override; 21 | int Ftruncate(int fd, off_t length) override; 22 | int Unlink(const char* path) override; 23 | int Mkdir(const char* path, mode_t mode) override; 24 | int Opendir(const char* path, DirStream* dirstream); 25 | int Getdents(DirStream* dirstream, char* contents, size_t maxread, ssize_t* realbytes); 26 | int Closedir(DirStream* dirstream); 27 | int Rmdir(const char* path) override; 28 | int Rename(const char* from, const char* to) override; 29 | int Link(const char* from, const char* to) override; 30 | int Symlink(const char* from, const char* to) override; 31 | int Readlink(const char* path, char* buf, size_t bufsize) override; 32 | int Chmod(const char* path, mode_t mode) override; 33 | int Chown(const char* path, uid_t uid, gid_t gid) override; 34 | int Truncate(const char* path, off_t length) override; 35 | int Utime(const char* path, const struct utimbuf* times) override; 36 | 37 | 38 | protected: 39 | std::string NormalizePath(const std::string& path) override; 40 | uintptr_t instance_; 41 | }; 42 | 43 | } // namespace filesystem 44 | } // namespace intercept 45 | 46 | 47 | #endif // CURVE_FILESYSTEM_H 48 | -------------------------------------------------------------------------------- /intercept/filesystem/dummy_filesystem.h: -------------------------------------------------------------------------------- 1 | #ifndef DUMMY_FILESYSTEM_H 2 | #define DUMMY_FILESYSTEM_H 3 | #include 4 | 5 | #include "abstract_filesystem.h" 6 | namespace intercept { 7 | namespace filesystem { 8 | class DummyFileSystem : public AbstractFileSystem { 9 | public: 10 | DummyFileSystem(); 11 | ~DummyFileSystem() override; 12 | void Init() override; 13 | void Shutdown() override; 14 | int Open(const char* path, int flags, int mode) override; 15 | ssize_t Read(int fd, void* buf, size_t count) override; 16 | ssize_t Write(int fd, const void* buf, size_t count) override; 17 | int Close(int fd) override; 18 | off_t Lseek(int fd, off_t offset, int whence) override; 19 | int Stat(const char* path, struct stat* st) override; 20 | int Fstat(int fd, struct stat* st) override; 21 | int Fsync(int fd) override; 22 | int Ftruncate(int fd, off_t length) override; 23 | int Unlink(const char* path) override; 24 | int Mkdir(const char* path, mode_t mode) override; 25 | int Opendir(const char* path, DirStream* dirstream); 26 | int Getdents(DirStream* dirstream, char* contents, size_t maxread, ssize_t* realbytes); 27 | int Closedir(DirStream* dirstream); 28 | int Rmdir(const char* path) override; 29 | int Rename(const char* from, const char* to) override; 30 | int Link(const char* from, const char* to) override; 31 | int Symlink(const char* from, const char* to) override; 32 | int Readlink(const char* path, char* buf, size_t bufsize) override; 33 | int Chmod(const char* path, mode_t mode) override; 34 | int Chown(const char* path, uid_t uid, gid_t gid) override; 35 | int Truncate(const char* path, off_t length) override; 36 | int Utime(const char* path, const struct utimbuf* times) override; 37 | 38 | 39 | protected: 40 | std::string NormalizePath(const std::string& path) override; 41 | uintptr_t instance_; 42 | std::atomic fd_ = 0; 43 | off_t offset_ = 0; 44 | long copynum_ = 0; 45 | static char* memory_; 46 | }; 47 | 48 | } // namespace filesystem 49 | } // namespace intercept 50 | #endif // DUMMY_FILESYSTEM_H -------------------------------------------------------------------------------- /intercept/filesystem/s3fs_filesystem.h: -------------------------------------------------------------------------------- 1 | #ifndef S3FS_FILESYSTEM_H 2 | #define S3FS_FILESYSTEM_H 3 | 4 | #include "abstract_filesystem.h" 5 | namespace intercept { 6 | namespace filesystem { 7 | 8 | class S3fsFileSystem : public AbstractFileSystem { 9 | public: 10 | S3fsFileSystem(); 11 | ~S3fsFileSystem() override; 12 | void Init() override; 13 | void Shutdown() override; 14 | int Open(const char* path, int flags, int mode) override; 15 | ssize_t Read(int fd, void* buf, size_t count) override; 16 | ssize_t Write(int fd, const void* buf, size_t count) override; 17 | int Close(int fd) override; 18 | off_t Lseek(int fd, off_t offset, int whence) override; 19 | int Stat(const char* path, struct stat* st) override; 20 | int Fstat(int fd, struct stat* st) override; 21 | int Fsync(int fd) override; 22 | int Ftruncate(int fd, off_t length) override; 23 | int Unlink(const char* path) override; 24 | int Mkdir(const char* path, mode_t mode) override; 25 | int Opendir(const char* path, DirStream* dirstream); 26 | int Getdents(DirStream* dirstream, char* contents, size_t maxread, ssize_t* realbytes); 27 | int Closedir(DirStream* dirstream); 28 | int Rmdir(const char* path) override; 29 | int Rename(const char* from, const char* to) override; 30 | int Link(const char* from, const char* to) override; 31 | int Symlink(const char* from, const char* to) override; 32 | int Readlink(const char* path, char* buf, size_t bufsize) override; 33 | int Chmod(const char* path, mode_t mode) override; 34 | int Chown(const char* path, uid_t uid, gid_t gid) override; 35 | int Truncate(const char* path, off_t length) override; 36 | int Utime(const char* path, const struct utimbuf* times) override; 37 | 38 | ssize_t MultiRead(int fd, void* buf, size_t count) override; 39 | ssize_t MultiWrite(int fd, const void* buf, size_t count) override; 40 | 41 | protected: 42 | std::string NormalizePath(const std::string& path) override; 43 | 44 | }; 45 | 46 | } // namespace filesystem 47 | } // namespace intercept 48 | 49 | #endif -------------------------------------------------------------------------------- /intercept/filesystem/s3fs_lib.h: -------------------------------------------------------------------------------- 1 | #ifndef S3FS_S3FS_LIB_H_ 2 | #define S3FS_S3FS_LIB_H_ 3 | 4 | #ifdef S3FS_MALLOC_TRIM 5 | #ifdef HAVE_MALLOC_TRIM 6 | #include 7 | #define S3FS_MALLOCTRIM(pad) malloc_trim(pad) 8 | #else // HAVE_MALLOC_TRIM 9 | #define S3FS_MALLOCTRIM(pad) 10 | #endif // HAVE_MALLOC_TRIM 11 | #else // S3FS_MALLOC_TRIM 12 | #define S3FS_MALLOCTRIM(pad) 13 | #endif // S3FS_MALLOC_TRIM 14 | 15 | 16 | //------------------------------------------------------------------- 17 | // posix interface functions 18 | //------------------------------------------------------------------- 19 | #ifdef __cplusplus 20 | extern "C" { 21 | #endif 22 | 23 | struct S3DirStream; 24 | 25 | void s3fs_global_init(); 26 | 27 | void s3fs_global_uninit(); 28 | 29 | int posix_s3fs_create(const char* _path, int flags, mode_t mode); 30 | 31 | int posix_s3fs_open(const char* _path, int flags, mode_t mode); 32 | 33 | int posix_s3fs_multiread(int fd, void* buf, size_t size, off_t file_offset); 34 | 35 | int posix_s3fs_read(int fd, void* buf, size_t size); 36 | 37 | int posix_s3fs_multiwrite(int fd, const void* buf, size_t size, off_t file_offset); 38 | 39 | int posix_s3fs_write(int fd, const void* buf, size_t size); 40 | 41 | off_t posix_s3fs_lseek(int fd, off_t offset, int whence); 42 | 43 | int posix_s3fs_close(int fd); 44 | 45 | int posix_s3fs_stat(const char* _path, struct stat* stbuf); 46 | 47 | int posix_s3fs_fstat(int fd, struct stat* stbuf) ; 48 | 49 | int posix_s3fs_mkdir(const char* _path, mode_t mode); 50 | 51 | int posix_s3fs_opendir(const char* _path, S3DirStream* dirstream); 52 | 53 | int posix_s3fs_getdents(S3DirStream* dirstream, char* contents, size_t maxread, ssize_t* realbytes); 54 | 55 | int posix_s3fs_closedir(S3DirStream* dirstream); 56 | 57 | int posix_s3fs_unlink(const char* _path); 58 | 59 | #ifdef __cplusplus 60 | } 61 | #endif 62 | 63 | #endif // S3FS_S3FS_LIB_H_ -------------------------------------------------------------------------------- /intercept/internal/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # internal/CMakeLists.txt 2 | 3 | file(GLOB INTERNAL_SOURCES *.cpp) 4 | 5 | add_library(intercept_internal ${INTERNAL_SOURCES}) 6 | target_include_directories(intercept_internal PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) 7 | target_link_libraries(intercept_internal PUBLIC common_lib) 8 | 9 | add_library(intercept_internal_client ${INTERNAL_SOURCES}) 10 | target_include_directories(intercept_internal_client PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) 11 | target_compile_options(intercept_internal_client PUBLIC -fPIC) 12 | target_link_libraries(intercept_internal_client PUBLIC common_lib_client) 13 | -------------------------------------------------------------------------------- /intercept/internal/metainfo.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 by Apex.AI Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // SPDX-License-Identifier: Apache-2.0 16 | 17 | #ifndef IOX_EXAMPLES_REQUEST_AND_RESPONSE_TYPES_HPP 18 | #define IOX_EXAMPLES_REQUEST_AND_RESPONSE_TYPES_HPP 19 | 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | #include 26 | 27 | 28 | 29 | #define SERVICE_FLAG "interceptservice" 30 | #define DUMMY_INSTANCE_FLAG "dummyserver" 31 | #define INTERCEPT_INSTANCE_FLAG "interceptserver" 32 | 33 | #define ICEORYX "ICEORYX" 34 | 35 | namespace intercept { 36 | namespace internal { 37 | //! [request] 38 | struct AddRequest 39 | { 40 | uint64_t augend{0}; 41 | uint64_t addend{0}; 42 | }; 43 | //! [request] 44 | 45 | //! [response] 46 | struct AddResponse 47 | { 48 | uint64_t sum{0}; 49 | }; 50 | //! [response] 51 | 52 | struct UserRequest 53 | { 54 | uint64_t pid{0}; 55 | uint64_t threadid{0}; 56 | }; 57 | 58 | struct UserResponse 59 | { 60 | uint64_t pid{0}; 61 | uint64_t threadid{0}; 62 | }; 63 | 64 | 65 | struct Metainfo { 66 | int type = 0; 67 | int fd = 0; 68 | size_t count = 0; 69 | }; 70 | 71 | 72 | 73 | struct ServiceMetaInfo { 74 | std::string service = ""; 75 | std::string instance = ""; 76 | std::string event = ""; 77 | std::string serverType = ""; // server类型 : normal dummy 78 | }; 79 | 80 | } // namespace internal 81 | } // namespace intercept 82 | 83 | 84 | #define MAX_LENGTH 2000000 85 | // 生成随机字符，不包括 '\0' 86 | // char randomChar() { 87 | // const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; 88 | // return charset[rand() % (sizeof(charset) - 1)]; 89 | // } 90 | 91 | // // 生成随机字符串 92 | // char* generateRandomString(size_t length) { 93 | // if (length > MAX_LENGTH) { 94 | // fprintf(stderr, "String length is too long.\n"); 95 | // } 96 | 97 | // char *str = (char*)malloc((length + 1) * sizeof(char)); // +1 为字符串的终止符 '\0' 预留空间 98 | // if (str == NULL) { 99 | // perror("malloc"); 100 | // } 101 | 102 | // for (size_t i = 0; i < length; ++i) { 103 | // str[i] = randomChar(); 104 | // } 105 | // str[length] = '\0'; // 确保字符串以空字符结尾 106 | 107 | // return str; 108 | // } 109 | 110 | 111 | 112 | #endif // IOX_EXAMPLES_REQUEST_AND_RESPONSE_TYPES_HPP 113 | -------------------------------------------------------------------------------- /intercept/middleware/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # src/middleware/CMakeLists.txt 2 | 3 | find_library(ICEORYX_POSH_LIB NAMES iceoryx_posh PATHS ../../thirdparties/iceoryx/lib) 4 | find_library(ICEORYX_HOOFS_LIB NAMES iceoryx_hoofs PATHS ../../thirdparties/iceoryx/lib) 5 | 6 | file(GLOB MIDDLEWARE_SOURCES *.cpp) 7 | file(GLOB MIDDLEWARE_HEADERS *.h) 8 | 9 | add_library(intercept_middleware ${MIDDLEWARE_SOURCES}) 10 | target_include_directories(intercept_middleware PUBLIC 11 | ${CMAKE_CURRENT_SOURCE_DIR} 12 | ${CMAKE_CURRENT_SOURCE_DIR}/../../thirdparties/iceoryx/include 13 | ) 14 | target_link_libraries(intercept_middleware PUBLIC 15 | intercept_internal 16 | intercept_filesystem 17 | ${ICEORYX_HOOFS_LIB} 18 | ${ICEORYX_POSH_LIB} 19 | 20 | ) 21 | 22 | 23 | set(CMAKE_FIND_LIBRARY_SUFFIXES ".so" ".a") 24 | 25 | find_library(ICEORYX_POSH_LIB NAMES iceoryx_posh PATHS ../../thirdparties/iceoryx/lib) 26 | find_library(ICEORYX_HOOFS_LIB NAMES iceoryx_hoofs PATHS ../../thirdparties/iceoryx/lib) 27 | find_library(ICEORYX_PLATFORM_LIB NAMES iceoryx_hoofs PATHS ../../thirdparties/iceoryx/lib) 28 | 29 | file(GLOB CLIENT_MIDDLEWARE_SOURCES *.cpp) 30 | file(GLOB CLIENT_MIDDLEWARE_HEADERS *.h) 31 | 32 | add_library(intercept_middleware_client ${CLIENT_MIDDLEWARE_SOURCES}) 33 | target_include_directories(intercept_middleware_client PUBLIC 34 | ${CMAKE_CURRENT_SOURCE_DIR} 35 | ${CMAKE_CURRENT_SOURCE_DIR}/../../thirdparties/iceoryx/include 36 | ) 37 | target_link_libraries(intercept_middleware_client PUBLIC 38 | -lrt 39 | intercept_internal_client 40 | intercept_filesystem_client 41 | ${ICEORYX_POSH_LIB} 42 | ${ICEORYX_HOOFS_LIB} 43 | ${ICEORYX_PLATFORM_LIB} 44 | ) 45 | target_compile_options(intercept_middleware_client PUBLIC -DCLIENT_BUILD -fPIC ) -------------------------------------------------------------------------------- /intercept/middleware/iceoryx_wrapper.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "req_res_middleware_wrapper.h" 4 | 5 | #include "iceoryx_posh/popo/untyped_server.hpp" 6 | #include "iceoryx_posh/popo/untyped_client.hpp" 7 | 8 | namespace intercept { 9 | namespace filesystem { 10 | class AbstractFileSystem; // Forward declaration 11 | } 12 | } 13 | 14 | namespace intercept { 15 | namespace middleware { 16 | 17 | class IceoryxWrapper : public ReqResMiddlewareWrapper { 18 | public: 19 | explicit IceoryxWrapper(const ServiceMetaInfo& info); 20 | 21 | ~IceoryxWrapper() override; 22 | 23 | virtual void Init() override; 24 | 25 | virtual void InitClient() override; 26 | 27 | virtual void InitServer() override; 28 | 29 | virtual void InitDummyServer() override; 30 | 31 | virtual void StartServer(); 32 | 33 | virtual void StartClient(); 34 | 35 | virtual void StopServer() override; 36 | 37 | virtual void StopClient() override; 38 | 39 | virtual void OnRequest(PosixOpReqRes& reqRes) override; 40 | 41 | virtual void OnResponse() override; 42 | 43 | virtual void Shutdown() override; 44 | 45 | virtual ServiceMetaInfo GetServiceMetaInfo() override {return info_;} 46 | 47 | private: 48 | void HandleOpenRequest(const auto& requestPayload); 49 | void HandleReadRequest(const auto& requestPayload); 50 | void HandleWriteRequest(const auto& requestPayload); 51 | void HandleCloseRequest(const auto& requestPayload); 52 | void HandleLseekRequest(const auto& requestPayload); 53 | void HandleFsyncRequest(const auto& requestPayload); 54 | void HandleStatRequest(const auto& requestPayload); 55 | void HandleFstatRequest(const auto& requestPayload); 56 | void HandleMkdirRequest(const auto& requestPayload); 57 | void HandleOpendirRequest(const auto& requestPayload); 58 | void HandleGetdentsRequest(const auto& requestPayload); 59 | void HandleClosedirRequest(const auto& requestPayload); 60 | void HandleUnlinkRequest(const auto& requestPayload); 61 | void HandleRenameRequest(const auto& requestPayload); 62 | void HandleTruncateRequest(const auto& requestPayload); 63 | void HandleTerminalRequest(const auto& requestPayload); 64 | 65 | private: 66 | std::shared_ptr server_; 67 | 68 | std::shared_ptr client_; 69 | 70 | int64_t requestSequenceId_ = 0; 71 | bool running_ = false; 72 | }; 73 | 74 | 75 | } // namespace middleware 76 | } // namespace intercept 77 | -------------------------------------------------------------------------------- /intercept/middleware/req_res_middleware_wrapper.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "middleware/req_res_middleware_wrapper.h" 4 | #ifndef CLIENT_BUILD 5 | #include "filesystem/curve_filesystem.h" 6 | #include "filesystem/s3fs_filesystem.h" 7 | #include "filesystem/dummy_filesystem.h" 8 | #endif 9 | #include "filesystem/abstract_filesystem.h" 10 | 11 | 12 | namespace intercept { 13 | namespace middleware { 14 | using intercept::common::Configure; 15 | void ReqResMiddlewareWrapper::Init() { 16 | 17 | } 18 | 19 | void ReqResMiddlewareWrapper::InitServer() { 20 | if (info_.serverType == "dummy") { 21 | spdlog::info("dont create fileSystem in ReqResMiddlewareWrapper::InitServer"); 22 | return; 23 | } 24 | if (!fileSystem_) { 25 | #ifndef CLIENT_BUILD 26 | if (Configure::getInstance().getConfig("backendFilesystem") == "s3fs") { 27 | fileSystem_.reset(new intercept::filesystem::S3fsFileSystem); 28 | } else if (Configure::getInstance().getConfig("backendFilesystem") == "curvefs") { 29 | fileSystem_.reset(new intercept::filesystem::CurveFileSystem); 30 | } else if (Configure::getInstance().getConfig("backendFilesystem") == "dummyfs") { 31 | fileSystem_.reset(new intercept::filesystem::DummyFileSystem); 32 | } else { 33 | spdlog::error("dont create fileSystem in ReqResMiddlewareWrapper::InitServer"); 34 | return; 35 | } 36 | fileSystem_->Init(); 37 | spdlog::info("Initserver, filesystem: {}", Configure::getInstance().getConfig("backendFilesystem")); 38 | #endif 39 | } else { 40 | spdlog::info("ReqResMiddlewareWrapper::InitServer, have inited, donot need to init again"); 41 | } 42 | } 43 | 44 | void ReqResMiddlewareWrapper::InitClient() { 45 | 46 | } 47 | 48 | } // namespace middleware 49 | } // namespace intercept -------------------------------------------------------------------------------- /intercept/middleware/req_res_middleware_wrapper.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | #include "internal/posix_op_req_res.h" 5 | #include "internal/metainfo.h" 6 | 7 | namespace intercept { 8 | namespace filesystem { 9 | class AbstractFileSystem; // Forward declaration 10 | } 11 | } 12 | 13 | namespace intercept 14 | { 15 | namespace middleware 16 | { 17 | using intercept::internal::ServiceMetaInfo; 18 | using intercept::internal::PosixOpReqRes; 19 | 20 | enum class ServiceType { 21 | CLIENT = 0, 22 | SERVER = 1, 23 | DUMMYSERVER = 2, 24 | }; 25 | 26 | class ReqResMiddlewareWrapper { 27 | public: 28 | ReqResMiddlewareWrapper() { 29 | spdlog::info("construct ReqResMiddlewareWrapper"); 30 | } 31 | 32 | ReqResMiddlewareWrapper(ServiceMetaInfo info) : info_(info) { 33 | spdlog::info("construct ReqResMiddlewareWrapper"); 34 | 35 | } 36 | 37 | virtual ~ReqResMiddlewareWrapper() { 38 | spdlog::info("deconstruct ReqResMiddlewareWrapper"); 39 | 40 | } 41 | 42 | virtual void Init(); 43 | 44 | virtual void InitClient(); 45 | 46 | virtual void InitServer(); 47 | 48 | virtual void SetServiceType(ServiceType type) { 49 | servicetype_ = type; 50 | } 51 | 52 | virtual void InitDummyServer() {} 53 | 54 | virtual void StartServer() = 0; 55 | 56 | virtual void StartClient() = 0; 57 | 58 | virtual void StopServer() = 0; 59 | 60 | virtual void StopClient() = 0; 61 | 62 | // 对外request接口 63 | virtual void OnRequest(PosixOpReqRes& reqRes) = 0; 64 | 65 | // 对外response接口 66 | virtual void OnResponse() = 0; 67 | 68 | virtual void Shutdown() = 0; 69 | 70 | virtual ServiceMetaInfo GetServiceMetaInfo() = 0; 71 | 72 | protected: 73 | static std::shared_ptr fileSystem_; 74 | ServiceMetaInfo info_; 75 | ServiceType servicetype_; 76 | }; 77 | 78 | } // namespace middleware 79 | } // namespace intercept 80 | 81 | -------------------------------------------------------------------------------- /intercept/posix/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # src/posix/CMakeLists.txt 2 | 3 | file(GLOB POSIX_SOURCES *.cpp) 4 | file(GLOB POSIX_HEADERS *.h) 5 | 6 | add_library(intercept_posix_interface_client ${POSIX_SOURCES}) 7 | target_include_directories(intercept_posix_interface_client PUBLIC 8 | ${CMAKE_CURRENT_SOURCE_DIR} 9 | ) 10 | target_link_libraries(intercept_posix_interface_client PUBLIC 11 | intercept_registry_client 12 | ) 13 | target_compile_options(intercept_posix_interface_client PUBLIC -DCLIENT_BUILD -fPIC) 14 | -------------------------------------------------------------------------------- /intercept/posix/posix_helper.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021 NetEase Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | 18 | /* 19 | * Project: curve 20 | * Created Date: Thur May 27 2021 21 | * Author: xuchaojie 22 | */ 23 | 24 | #include 25 | #include 26 | #include 27 | 28 | #include 29 | #include 30 | 31 | #include "posix_op.h" 32 | #include "syscall_client.h" 33 | 34 | // 仅用于联编 35 | int help(int argc, char *argv[]) { 36 | return 0; 37 | } 38 | -------------------------------------------------------------------------------- /intercept/posix/syscall_client.h: -------------------------------------------------------------------------------- 1 | #ifndef CURVEFS_SRC_CLIENT_SYSCALL_CLIENT_ 2 | #define CURVEFS_SRC_CLIENT_SYSCALL_CLIENT_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "posix/libsyscall_intercept_hook_point.h" 10 | //#include "syscall_interception.h" 11 | #include "posix/posix_op.h" 12 | #include 13 | #include 14 | #include 15 | 16 | // 拦截函数 17 | 18 | static int hook(long syscallNumber, 19 | long arg0, long arg1, 20 | long arg2, long arg3, 21 | long arg4, long arg5, 22 | long* result) { 23 | 24 | long args[6] = {arg0, arg1, arg2, arg3, arg4, arg5}; 25 | const struct syscall_desc* desc = GetSyscallDesc(syscallNumber, args); 26 | if (desc != nullptr) { 27 | int ret = desc->syscallFunction(args, result); 28 | //return 0; // 接管 29 | return ret; 30 | } 31 | 32 | return 1; // 如果不需要拦截，返回1 33 | } 34 | 35 | // 初始化函数 36 | static __attribute__((constructor)) void start(void) { 37 | InitSyscall(); 38 | intercept_hook_point = &hook; 39 | } 40 | 41 | #endif 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /intercept/registry/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # src/registry/CMakeLists.txt 2 | 3 | find_library(ICEORYX_POSH_LIB NAMES iceoryx_posh PATHS ../../thirdparties/iceoryx/lib) 4 | find_library(ICEORYX_HOOFS_LIB iceoryx_hoofs PATHS ../thirdparties/iceoryx/lib) 5 | find_library(ICEORYX_PLATFORM_LIB iceoryx_platform PATHS ../thirdparties/iceoryx/lib) 6 | 7 | file(GLOB REGISTRY_SOURCES *.cpp) 8 | file(GLOB REGISTRY_HEADERS *.h) 9 | 10 | add_library(intercept_registry ${REGISTRY_SOURCES}) 11 | target_include_directories(intercept_registry PUBLIC 12 | ${CMAKE_CURRENT_SOURCE_DIR} 13 | ${CMAKE_CURRENT_SOURCE_DIR}/../../thirdparties/iceoryx/include 14 | ) 15 | target_link_libraries(intercept_registry PUBLIC 16 | intercept_middleware 17 | intercept_discovery 18 | ${ICEORYX_HOOFS_LIB} 19 | ${ICEORYX_PLATFORM_LIB} 20 | ${ICEORYX_POSH_LIB} 21 | ) 22 | 23 | 24 | file(GLOB CLIENT_REGISTRY_SOURCES *.cpp) 25 | file(GLOB CLIENT_REGISTRY_HEADERS *.h) 26 | 27 | add_library(intercept_registry_client ${CLIENT_REGISTRY_SOURCES}) 28 | target_include_directories(intercept_registry_client PUBLIC 29 | ${CMAKE_CURRENT_SOURCE_DIR} 30 | ${CMAKE_CURRENT_SOURCE_DIR}/../../thirdparties/iceoryx/include 31 | ) 32 | target_link_libraries(intercept_registry_client PUBLIC 33 | intercept_middleware_client 34 | intercept_discovery_client 35 | ${ICEORYX_POSH_LIB} 36 | ${ICEORYX_HOOFS_LIB} 37 | ${ICEORYX_PLATFORM_LIB} 38 | -lrt 39 | ) 40 | target_compile_options(intercept_registry_client PUBLIC -DCLIENT_BUILD ) -------------------------------------------------------------------------------- /intercept/registry/client_server_registry.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "middleware/req_res_middleware_wrapper.h" 4 | #include "discovery/iceoryx_discovery.h" 5 | #include "discovery/discovery.h" 6 | 7 | #define CREATE_FLAG "create" 8 | #define DESTROY_FLAG "destroy" 9 | #define SERVER_FLAG "server" 10 | 11 | namespace intercept { 12 | namespace registry { 13 | 14 | using intercept::middleware::ReqResMiddlewareWrapper; 15 | using intercept::discovery::Discovery; 16 | using intercept::internal::OpenOpReqRes; 17 | using intercept::internal::ServiceMetaInfo; 18 | 19 | 20 | class ClientServerRegistry { 21 | 22 | public: 23 | // ... 24 | ClientServerRegistry(const std::string& middlewareType, const ServiceMetaInfo& info); 25 | ~ClientServerRegistry(); 26 | // 创建临时的server，主要用于通过server创建数据交换的server 27 | std::shared_ptr CreateDummyServer(); 28 | void DestroyDummyServer(); 29 | 30 | // 返回一个已经初始化的middleWrapper_; 31 | std::shared_ptr CreateClient(const ServiceMetaInfo& info); 32 | std::shared_ptr CreateServer(const ServiceMetaInfo& info); 33 | 34 | // 在daemon端更新server 35 | void MonitorServers(); 36 | 37 | private: 38 | // 根据client传递的信息 39 | void CreateServers(); // 创建服务 40 | void DestroyServers(); // 销毁服务 41 | 42 | private: 43 | // ... 44 | std::string middlewareType_; 45 | ServiceMetaInfo serviceInfo_; // 这里一个service由：service instance构成 46 | std::shared_ptr discovery_; 47 | 48 | std::vector> clientWrapper_; 49 | std::vector> serverWrapper_; 50 | 51 | std::set dummyevent_; 52 | std::unordered_map> serverMap_; 53 | 54 | // 存放创建的线程 55 | std::vector threads_; 56 | 57 | }; 58 | 59 | /// 60 | // int client() { 61 | // ServiceMetaInfo info = {"Service", "Instance", "Event"}; 62 | // ClientServerRegistry registry("ICE", info); 63 | // registry.CreateDummyServer(); 64 | // auto client = registry.CreateClient(ServiceMetaInfo{"Service", "Instance", "Event"}); 65 | // OpenOpReqRes reqres("test", 1, 1); 66 | // client->OnRequest(reqres); 67 | // // 全局使用这一个client去操作请求 68 | 69 | // registry.DestroyDummyServer(); 70 | // return 0; 71 | // } 72 | 73 | } 74 | } 75 | 76 | 77 | 78 | 79 | -------------------------------------------------------------------------------- /intercept/server.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "registry/client_server_registry.h" 9 | 10 | using namespace intercept::internal; 11 | using namespace intercept::registry; 12 | std::mutex mtx; 13 | std::condition_variable cv; 14 | std::atomic discovery_thread_running{false}; 15 | 16 | int main() { 17 | constexpr char APP_NAME[] = "iox-intercept-server"; 18 | if (intercept::common::Configure::getInstance().loadConfig(intercept::common::CONFIG_FILE)) { 19 | std::cout << "Config file loaded" << std::endl; 20 | } else { 21 | std::cout << "Config file not loaded: server.conf" << std::endl; 22 | return 0; 23 | } 24 | intercept::common::InitLog(); 25 | iox::runtime::PoshRuntime::initRuntime(APP_NAME); 26 | ServiceMetaInfo info = {SERVICE_FLAG, "", ""}; 27 | std::string type = ICEORYX; 28 | ClientServerRegistry registry(type, info); 29 | spdlog::info("begin to monitor servers"); 30 | registry.MonitorServers(); 31 | return 0; 32 | } -------------------------------------------------------------------------------- /local_cache/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/bin) 2 | 3 | file (GLOB_RECURSE LOCAL_CACHE_SOURCES CONFIGURE_DEPENDS "*.cpp") 4 | add_library(hybridcache_local STATIC ${LOCAL_CACHE_SOURCES}) 5 | target_link_libraries(hybridcache_local PUBLIC ${THIRD_PARTY_LIBRARIES} -laio) 6 | -------------------------------------------------------------------------------- /local_cache/accessor.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Project: HybridCache 3 | * Created Date: 24-3-25 4 | * Author: lshb 5 | */ 6 | #ifndef HYBRIDCACHE_ACCESSOR_H_ 7 | #define HYBRIDCACHE_ACCESSOR_H_ 8 | 9 | #include "read_cache.h" 10 | #include "write_cache.h" 11 | 12 | namespace HybridCache { 13 | 14 | class HybridCacheAccessor { 15 | public: 16 | HybridCacheAccessor(const HybridCacheConfig& cfg) : cfg_(cfg) {} 17 | ~HybridCacheAccessor() {} 18 | 19 | // Put in write cache. 20 | // If the write cache is full, block waiting for asynchronous flush to release the write cache space 21 | virtual int Put(const std::string &key, size_t start, size_t len, const char* buf) = 0; 22 | 23 | // 1.Read from write cache. 2.Read from read cache. 24 | virtual int Get(const std::string &key, size_t start, size_t len, char* buf) = 0; 25 | 26 | // Get4ReadHandle(); 27 | 28 | // File flush. Need to handle flush/write concurrency. 29 | virtual int Flush(const std::string &key) = 0; 30 | 31 | // Flush to the final data source, such as global cache to s3. 32 | virtual int DeepFlush(const std::string &key) = 0; 33 | 34 | virtual int Delete(const std::string &key) = 0; 35 | 36 | // Invalidated the local read cache. 37 | // Delete read cache when open the file. That is a configuration item. 38 | virtual int Invalidate(const std::string &key) = 0; 39 | 40 | // Background asynchronous flush all files and releases write cache space. 41 | virtual int FsSync() = 0; 42 | 43 | protected: 44 | HybridCacheConfig cfg_; 45 | std::shared_ptr writeCache_; 46 | std::shared_ptr readCache_; 47 | std::shared_ptr dataAdaptor_; 48 | }; 49 | 50 | } // namespace HybridCache 51 | 52 | #endif // HYBRIDCACHE_ACCESSOR_H_ 53 | -------------------------------------------------------------------------------- /local_cache/common.cpp: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | 3 | #include 4 | 5 | namespace HybridCache { 6 | 7 | bool EnableLogging = true; 8 | 9 | void split(const std::string& str, const char delim, 10 | std::vector& items) { 11 | std::istringstream iss(str); 12 | std::string tmp; 13 | while (std::getline(iss, tmp, delim)) { 14 | if (!tmp.empty()) { 15 | items.emplace_back(std::move(tmp)); 16 | } 17 | } 18 | } 19 | 20 | std::string base64(const unsigned char* input, size_t length) { 21 | static constexpr char base[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="; 22 | 23 | std::string result; 24 | result.reserve(((length + 3 - 1) / 3) * 4 + 1); 25 | 26 | unsigned char parts[4]; 27 | size_t rpos; 28 | for(rpos = 0; rpos < length; rpos += 3){ 29 | parts[0] = (input[rpos] & 0xfc) >> 2; 30 | parts[1] = ((input[rpos] & 0x03) << 4) | ((((rpos + 1) < length ? input[rpos + 1] : 0x00) & 0xf0) >> 4); 31 | parts[2] = (rpos + 1) < length ? (((input[rpos + 1] & 0x0f) << 2) | ((((rpos + 2) < length ? input[rpos + 2] : 0x00) & 0xc0) >> 6)) : 0x40; 32 | parts[3] = (rpos + 2) < length ? (input[rpos + 2] & 0x3f) : 0x40; 33 | 34 | result += base[parts[0]]; 35 | result += base[parts[1]]; 36 | result += base[parts[2]]; 37 | result += base[parts[3]]; 38 | } 39 | 40 | return result; 41 | } 42 | 43 | std::string md5(const std::string& str) { 44 | std::array binary; 45 | unsigned int digestlen = static_cast(binary.size()); 46 | 47 | const EVP_MD* md = EVP_get_digestbyname("md5"); 48 | EVP_MD_CTX* mdctx = EVP_MD_CTX_create(); 49 | EVP_DigestInit_ex(mdctx, md, nullptr); 50 | EVP_DigestUpdate(mdctx, str.c_str(), str.length()); 51 | EVP_DigestFinal_ex(mdctx, binary.data(), &digestlen); 52 | EVP_MD_CTX_destroy(mdctx); 53 | 54 | return base64(binary.data(), binary.size()); 55 | } 56 | 57 | } // namespace HybridCache 58 | -------------------------------------------------------------------------------- /local_cache/common.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Project: HybridCache 3 | * Created Date: 24-2-21 4 | * Author: lshb 5 | */ 6 | #ifndef HYBRIDCACHE_COMMON_H_ 7 | #define HYBRIDCACHE_COMMON_H_ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include "folly/executors/CPUThreadPoolExecutor.h" 15 | 16 | namespace HybridCache { 17 | 18 | typedef folly::CPUThreadPoolExecutor ThreadPool; 19 | 20 | static const char PAGE_SEPARATOR = 26; 21 | 22 | static const uint32_t BYTE_LEN = 8; 23 | 24 | // ConcurrentSkipList height 25 | static const int SKIP_LIST_HEIGHT = 2; 26 | 27 | extern bool EnableLogging; 28 | 29 | struct ByteBuffer { 30 | char* data; 31 | size_t len; 32 | ByteBuffer(char* buf = nullptr, size_t bufLen = 0) : data(buf), len(bufLen) {} 33 | }; 34 | 35 | void split(const std::string& str, const char delim, 36 | std::vector& items); 37 | 38 | std::string base64(const unsigned char* input, size_t length); 39 | 40 | std::string md5(const std::string& str); 41 | 42 | } // namespace HybridCache 43 | 44 | #endif // HYBRIDCACHE_COMMON_H_ 45 | -------------------------------------------------------------------------------- /local_cache/config.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Project: HybridCache 3 | * Created Date: 24-2-21 4 | * Author: lshb 5 | */ 6 | #ifndef HYBRIDCACHE_CONFIG_H_ 7 | #define HYBRIDCACHE_CONFIG_H_ 8 | 9 | #include 10 | #include 11 | 12 | namespace HybridCache { 13 | 14 | struct CacheLibConfig { 15 | bool EnableNvmCache = false; 16 | std::string RaidPath; 17 | uint64_t RaidFileNum; 18 | size_t RaidFileSize; 19 | bool DataChecksum = false; 20 | }; 21 | 22 | struct CacheConfig { 23 | std::string CacheName; 24 | size_t MaxCacheSize; 25 | uint32_t PageBodySize; 26 | uint32_t PageMetaSize; 27 | bool EnableCAS; 28 | bool SafeMode; // atomic write/delete lock 29 | CacheLibConfig CacheLibCfg; 30 | }; 31 | 32 | struct ReadCacheConfig { 33 | CacheConfig CacheCfg; 34 | uint64_t DownloadNormalFlowLimit; 35 | uint64_t DownloadBurstFlowLimit; 36 | }; 37 | 38 | struct WriteCacheConfig { 39 | CacheConfig CacheCfg; 40 | uint32_t CacheSafeRatio; // cache safety concern threshold (percent) 41 | bool EnableThrottle; // added by tqy 42 | }; 43 | 44 | struct GlobalCacheConfig { 45 | bool EnableWriteCache; 46 | std::string EtcdAddress; 47 | std::vector GlobalServers; 48 | std::string GflagFile; 49 | }; 50 | 51 | struct HybridCacheConfig { 52 | ReadCacheConfig ReadCacheCfg; 53 | WriteCacheConfig WriteCacheCfg; 54 | GlobalCacheConfig GlobalCacheCfg; 55 | uint32_t ThreadNum; 56 | uint32_t BackFlushCacheRatio; 57 | uint64_t UploadNormalFlowLimit; 58 | uint64_t UploadBurstFlowLimit; 59 | std::string LogPath; 60 | uint32_t LogLevel; 61 | bool EnableLog = true; 62 | bool UseGlobalCache = false; 63 | bool FlushToRead = false; // write to read cache after flush 64 | bool CleanCacheByOpen = false; // clean read cache when open file 65 | // added by tqy 66 | bool EnableResize; // 是否开启普通的Resize策略 67 | bool EnableLinUCB; // 是否开启LinUCB 68 | }; 69 | 70 | bool GetHybridCacheConfig(const std::string& file, HybridCacheConfig& cfg); 71 | bool CheckConfig(const HybridCacheConfig& cfg); 72 | bool ParseFlagFromFile(const std::string& file); 73 | 74 | class Configuration { 75 | public: 76 | bool LoadConfig(const std::string& file); 77 | void PrintConfig(); 78 | 79 | /* 80 | * @brief GetValueFatalIfFail Get the value of the specified config item 81 | * log it if get error 82 | * 83 | * @param[in] key config name 84 | * @param[out] value config value 85 | * 86 | * @return 87 | */ 88 | template 89 | void GetValueFatalIfFail(const std::string& key, T& value); 90 | 91 | private: 92 | std::string confFile_; 93 | std::map config_; 94 | }; 95 | 96 | } // namespace HybridCache 97 | 98 | #endif // HYBRIDCACHE_CONFIG_H_ 99 | -------------------------------------------------------------------------------- /local_cache/data_adaptor.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Project: HybridCache 3 | * Created Date: 24-2-26 4 | * Author: lshb 5 | */ 6 | #ifndef HYBRIDCACHE_DATA_ADAPTOR_H_ 7 | #define HYBRIDCACHE_DATA_ADAPTOR_H_ 8 | 9 | #include 10 | 11 | #include "folly/futures/Future.h" 12 | #include "glog/logging.h" 13 | 14 | #include "common.h" 15 | #include "errorcode.h" 16 | 17 | namespace HybridCache { 18 | 19 | class DataAdaptor { 20 | public: 21 | virtual folly::Future DownLoad(const std::string &key, 22 | size_t start, 23 | size_t size, 24 | ByteBuffer &buffer) = 0; 25 | 26 | virtual folly::Future UpLoad(const std::string &key, 27 | size_t size, 28 | const ByteBuffer &buffer, 29 | const std::map& headers) = 0; 30 | 31 | virtual folly::Future Delete(const std::string &key) = 0; 32 | 33 | // for global cache 34 | virtual folly::Future DeepFlush(const std::string &key) { 35 | return folly::makeFuture(0); 36 | } 37 | 38 | virtual folly::Future Head(const std::string &key, 39 | size_t& size, 40 | std::map& headers) = 0; 42 | 43 | void SetExecutor(std::shared_ptr executor) { 44 | executor_ = executor; 45 | } 46 | 47 | protected: 48 | std::shared_ptr executor_; 49 | }; 50 | 51 | class DataAdaptor4Test : public DataAdaptor { 52 | public: 53 | folly::Future DownLoad(const std::string &key, 54 | size_t start, 55 | size_t size, 56 | ByteBuffer &buffer) { 57 | assert(executor_); 58 | return folly::via(executor_.get(), [key, start, size, buffer]() -> int { 59 | LOG(INFO) << "[DataAdaptor]DownLoad start, key:" << key 60 | << ", start:" << start << ", size:" << size; 61 | std::this_thread::sleep_for(std::chrono::seconds(3)); 62 | LOG(INFO) << "[DataAdaptor]DownLoad error, key:" << key 63 | << ", start:" << start << ", size:" << size; 64 | return REMOTE_FILE_NOT_FOUND; 65 | }); 66 | } 67 | 68 | folly::Future UpLoad(const std::string &key, 69 | size_t size, 70 | const ByteBuffer &buffer, 71 | const std::map& headers) { 72 | return folly::makeFuture(REMOTE_FILE_NOT_FOUND); 73 | } 74 | 75 | folly::Future Delete(const std::string &key) { 76 | return folly::makeFuture(REMOTE_FILE_NOT_FOUND); 77 | } 78 | 79 | folly::Future Head(const std::string &key, 80 | size_t& size, 81 | std::map& headers) { 83 | return folly::makeFuture(REMOTE_FILE_NOT_FOUND); 84 | } 85 | }; 86 | 87 | } // namespace HybridCache 88 | 89 | #endif // HYBRIDCACHE_DATA_ADAPTOR_H_ 90 | -------------------------------------------------------------------------------- /local_cache/errorcode.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Project: HybridCache 3 | * Created Date: 24-3-18 4 | * Author: lshb 5 | */ 6 | #ifndef HYBRIDCACHE_ERRORCODE_H_ 7 | #define HYBRIDCACHE_ERRORCODE_H_ 8 | 9 | namespace HybridCache { 10 | 11 | enum ErrCode { 12 | SUCCESS = 0, 13 | PAGE_NOT_FOUND = -1, 14 | PAGE_DEL_FAIL = -2, 15 | ADAPTOR_NOT_FOUND = -3, 16 | REMOTE_FILE_NOT_FOUND = -4, 17 | }; 18 | 19 | } // namespace HybridCache 20 | 21 | #endif // HYBRIDCACHE_ERRORCODE_H_ 22 | -------------------------------------------------------------------------------- /local_cache/read_cache.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Project: HybridCache 3 | * Created Date: 24-2-29 4 | * Author: lshb 5 | */ 6 | #ifndef HYBRIDCACHE_READ_CACHE_H_ 7 | #define HYBRIDCACHE_READ_CACHE_H_ 8 | 9 | #include "folly/TokenBucket.h" 10 | 11 | #include "page_cache.h" 12 | #include "data_adaptor.h" 13 | 14 | namespace HybridCache { 15 | 16 | class ReadCache { 17 | public: 18 | ReadCache(const ReadCacheConfig& cfg, 19 | std::shared_ptr dataAdaptor, 20 | std::shared_ptr executor, 21 | PoolId curr_id = NULL, 22 | std::shared_ptr curr_cache = nullptr); 23 | ReadCache() = default; 24 | ~ReadCache() { Close(); } 25 | 26 | // Read the local page cache first, and get it from the DataAdaptor if it misses 27 | folly::Future Get(const std::string &key, 28 | size_t start, 29 | size_t len, 30 | ByteBuffer &buffer // user buf 31 | ); 32 | 33 | int Put(const std::string &key, 34 | size_t start, 35 | size_t len, 36 | const ByteBuffer &buffer); 37 | 38 | int Delete(const std::string &key); 39 | 40 | int GetAllKeys(std::set& keys); 41 | 42 | void Close(); 43 | 44 | private: 45 | int Init(); 46 | 47 | // added by tqy 48 | int CombinedInit(PoolId curr_id, std::shared_ptr curr_cache); 49 | 50 | std::string GetPageKey(const std::string &key, size_t pageIndex); 51 | 52 | private: 53 | ReadCacheConfig cfg_; 54 | std::shared_ptr pageCache_; 55 | std::shared_ptr dataAdaptor_; 56 | std::shared_ptr executor_; 57 | std::shared_ptr tokenBucket_; // download flow limit 58 | }; 59 | 60 | } // namespace HybridCache 61 | 62 | #endif // HYBRIDCACHE_READ_CACHE_H_ 63 | -------------------------------------------------------------------------------- /local_cache/throttle.h: -------------------------------------------------------------------------------- 1 | #ifndef HYBRIDCACHE_THROTTLE_H_ 2 | #define HYBRIDCACHE_THROTTLE_H_ 3 | 4 | #include "folly/TokenBucket.h" 5 | #include "folly/concurrency/ConcurrentHashMap.h" 6 | 7 | namespace HybridCache { 8 | 9 | class Throttle { 10 | public: 11 | Throttle() {} 12 | 13 | // 带宽重分配 14 | int SetNewLimits(const std::string &input); 15 | int Put_Consume(const std::string&, size_t); 16 | void Del_File(const std::string&); 17 | void Close(); 18 | std::string Cal_New4Test(); 19 | void CleanBlockTime(); 20 | 21 | public: 22 | folly::ConcurrentHashMap> job_tokenlimit_; // 文件和每个任务的写上限 23 | folly::ConcurrentHashMap job_waiting_; // 文件和阻塞时间 24 | folly::ConcurrentHashMap job_bandwidth_; 25 | //std::atomic is_resetting_{false}; // 正在重新调整各个文件的带宽 26 | }; 27 | 28 | } 29 | 30 | #endif // HYBRIDCACHE_THROTTLE_H_ 31 | -------------------------------------------------------------------------------- /local_cache/write_cache.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Project: HybridCache 3 | * Created Date: 24-3-18 4 | * Author: lshb 5 | */ 6 | #ifndef HYBRIDCACHE_WRITE_CACHE_H_ 7 | #define HYBRIDCACHE_WRITE_CACHE_H_ 8 | 9 | #include "folly/concurrency/ConcurrentHashMap.h" 10 | 11 | #include "page_cache.h" 12 | #include "throttle.h" 13 | 14 | namespace HybridCache { 15 | 16 | class WriteCache { 17 | public: 18 | // added by tqy 19 | WriteCache(const WriteCacheConfig& cfg, 20 | PoolId curr_id = NULL, 21 | std::shared_ptr curr_cache = nullptr); 22 | 23 | WriteCache() = default; 24 | ~WriteCache() { Close(); } 25 | 26 | enum class LockType { 27 | NONE = 0, 28 | ALREADY_LOCKED = -1, 29 | }; 30 | 31 | int Put(const std::string &key, 32 | size_t start, 33 | size_t len, 34 | const ByteBuffer &buffer 35 | ); 36 | 37 | int Get(const std::string &key, 38 | size_t start, 39 | size_t len, 40 | ByteBuffer &buffer, 41 | std::vector>& dataBoundary // valid data segment boundar 42 | ); 43 | 44 | // lock to ensure the availability of the returned buf 45 | // After being locked, it can be read and written, but cannot be deleted 46 | int GetAllCacheWithLock(const std::string &key, 47 | std::vector>& dataSegments // ByteBuffer + off of key value(file) 48 | ); 49 | 50 | int Delete(const std::string &key, LockType type = LockType::NONE); 51 | 52 | int Truncate(const std::string &key, size_t len); 53 | 54 | void UnLock(const std::string &key); 55 | 56 | int GetAllKeys(std::map& keys); 57 | 58 | void Close(); 59 | 60 | size_t GetCacheSize(); 61 | size_t GetCacheMaxSize(); 62 | 63 | private: 64 | int Init(); 65 | 66 | void Lock(const std::string &key); 67 | 68 | std::string GetPageKey(const std::string &key, size_t pageIndex); 69 | 70 | // added by tqy 71 | int CombinedInit(PoolId curr_id, std::shared_ptr curr_cache); 72 | void Dealing_throttling(); 73 | 74 | private: 75 | WriteCacheConfig cfg_; 76 | std::shared_ptr pageCache_; 77 | folly::ConcurrentHashMap keys_; // 78 | StringSkipList::Accessor keyLocks_ = StringSkipList::create(SKIP_LIST_HEIGHT); // presence key indicates lock 79 | 80 | // added by tqy 81 | HybridCache::Throttle throttling_; 82 | std::thread throttling_thread_; // 改成一个单独的线程 83 | std::atomic throttling_thread_running_{false}; // 调度线程是否启用 84 | }; 85 | 86 | } // namespace HybridCache 87 | 88 | #endif // HYBRIDCACHE_WRITE_CACHE_H_ 89 | -------------------------------------------------------------------------------- /s3fs/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/bin) 2 | 3 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -O3 -D_FILE_OFFSET_BITS=64 -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=3") 4 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -O3 -D_FILE_OFFSET_BITS=64 -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=3") 5 | 6 | file(GLOB_RECURSE ALL_SOURCES CONFIGURE_DEPENDS "*.cpp") 7 | list(REMOVE_ITEM ALL_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/s3fs_lib.cpp") 8 | add_executable(s3fs ${ALL_SOURCES}) 9 | target_include_directories(s3fs PRIVATE /usr/include/fuse /usr/include/libxml2) 10 | target_link_libraries(s3fs PUBLIC hybridcache_local madfs_global -lfuse -pthread -lcurl -lxml2 -lcrypto -ldl) 11 | 12 | file(GLOB_RECURSE LIB_SOURCES CONFIGURE_DEPENDS "*.cpp") 13 | list(REMOVE_ITEM LIB_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/s3fs.cpp") 14 | add_library(s3fs_lib STATIC ${LIB_SOURCES}) 15 | target_include_directories(s3fs_lib PRIVATE /usr/include/fuse /usr/include/libxml2) 16 | target_link_libraries(s3fs_lib PUBLIC hybridcache_local madfs_global -pthread -lcurl -lxml2 -lcrypto -ldl) 17 | -------------------------------------------------------------------------------- /s3fs/addhead.h: -------------------------------------------------------------------------------- 1 | /* 2 | * s3fs - FUSE-based file system backed by Amazon S3 3 | * 4 | * Copyright(C) 2007 Randy Rizun 5 | * 6 | * This program is free software; you can redistribute it and/or 7 | * modify it under the terms of the GNU General Public License 8 | * as published by the Free Software Foundation; either version 2 9 | * of the License, or (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License 17 | * along with this program; if not, write to the Free Software 18 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | */ 20 | 21 | #ifndef S3FS_ADDHEAD_H_ 22 | #define S3FS_ADDHEAD_H_ 23 | 24 | #include 25 | #include 26 | #include 27 | 28 | #include "metaheader.h" 29 | 30 | //---------------------------------------------- 31 | // Structure / Typedef 32 | //---------------------------------------------- 33 | struct add_header{ 34 | add_header(std::unique_ptr pregex, std::string basestring, std::string headkey, std::string headvalue) 35 | : pregex(std::move(pregex)) 36 | , basestring(std::move(basestring)) 37 | , headkey(std::move(headkey)) 38 | , headvalue(std::move(headvalue)) 39 | {} 40 | ~add_header() { 41 | if(pregex){ 42 | regfree(pregex.get()); 43 | } 44 | } 45 | 46 | add_header(const add_header&) = delete; 47 | add_header(add_header&& val) = default; 48 | add_header& operator=(const add_header&) = delete; 49 | add_header& operator=(add_header&&) = delete; 50 | 51 | std::unique_ptr pregex; // not nullptr means using regex, nullptr means comparing suffix directly. 52 | std::string basestring; 53 | std::string headkey; 54 | std::string headvalue; 55 | }; 56 | 57 | typedef std::vector addheadlist_t; 58 | 59 | //---------------------------------------------- 60 | // Class AdditionalHeader 61 | //---------------------------------------------- 62 | class AdditionalHeader 63 | { 64 | private: 65 | static AdditionalHeader singleton; 66 | bool is_enable; 67 | addheadlist_t addheadlist; 68 | 69 | protected: 70 | AdditionalHeader(); 71 | ~AdditionalHeader(); 72 | AdditionalHeader(const AdditionalHeader&) = delete; 73 | AdditionalHeader(AdditionalHeader&&) = delete; 74 | AdditionalHeader& operator=(const AdditionalHeader&) = delete; 75 | AdditionalHeader& operator=(AdditionalHeader&&) = delete; 76 | 77 | public: 78 | // Reference singleton 79 | static AdditionalHeader* get() { return &singleton; } 80 | 81 | bool Load(const char* file); 82 | void Unload(); 83 | 84 | bool AddHeader(headers_t& meta, const char* path) const; 85 | struct curl_slist* AddHeader(struct curl_slist* list, const char* path) const; 86 | bool Dump() const; 87 | }; 88 | 89 | #endif // S3FS_ADDHEAD_H_ 90 | 91 | /* 92 | * Local variables: 93 | * tab-width: 4 94 | * c-basic-offset: 4 95 | * End: 96 | * vim600: expandtab sw=4 ts=4 fdm=marker 97 | * vim<600: expandtab sw=4 ts=4 98 | */ 99 | -------------------------------------------------------------------------------- /s3fs/autolock.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * s3fs - FUSE-based file system backed by Amazon S3 3 | * 4 | * Copyright(C) 2007 Takeshi Nakatani 5 | * 6 | * This program is free software; you can redistribute it and/or 7 | * modify it under the terms of the GNU General Public License 8 | * as published by the Free Software Foundation; either version 2 9 | * of the License, or (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License 17 | * along with this program; if not, write to the Free Software 18 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | */ 20 | 21 | #include 22 | #include 23 | 24 | #include "autolock.h" 25 | #include "s3fs_logger.h" 26 | 27 | //------------------------------------------------------------------- 28 | // Class AutoLock 29 | //------------------------------------------------------------------- 30 | AutoLock::AutoLock(pthread_mutex_t* pmutex, Type type) : auto_mutex(pmutex) 31 | { 32 | if (type == ALREADY_LOCKED) { 33 | is_lock_acquired = false; 34 | } else if (type == NO_WAIT) { 35 | int result = pthread_mutex_trylock(auto_mutex); 36 | if(result == 0){ 37 | is_lock_acquired = true; 38 | }else if(result == EBUSY){ 39 | is_lock_acquired = false; 40 | }else{ 41 | S3FS_PRN_CRIT("pthread_mutex_trylock returned: %d", result); 42 | abort(); 43 | } 44 | } else { 45 | int result = pthread_mutex_lock(auto_mutex); 46 | if(result == 0){ 47 | is_lock_acquired = true; 48 | }else{ 49 | S3FS_PRN_CRIT("pthread_mutex_lock returned: %d", result); 50 | abort(); 51 | } 52 | } 53 | } 54 | 55 | bool AutoLock::isLockAcquired() const 56 | { 57 | return is_lock_acquired; 58 | } 59 | 60 | AutoLock::~AutoLock() 61 | { 62 | if (is_lock_acquired) { 63 | int result = pthread_mutex_unlock(auto_mutex); 64 | if(result != 0){ 65 | S3FS_PRN_CRIT("pthread_mutex_unlock returned: %d", result); 66 | abort(); 67 | } 68 | } 69 | } 70 | 71 | /* 72 | * Local variables: 73 | * tab-width: 4 74 | * c-basic-offset: 4 75 | * End: 76 | * vim600: expandtab sw=4 ts=4 fdm=marker 77 | * vim<600: expandtab sw=4 ts=4 78 | */ 79 | -------------------------------------------------------------------------------- /s3fs/autolock.h: -------------------------------------------------------------------------------- 1 | /* 2 | * s3fs - FUSE-based file system backed by Amazon S3 3 | * 4 | * Copyright(C) 2007 Randy Rizun 5 | * 6 | * This program is free software; you can redistribute it and/or 7 | * modify it under the terms of the GNU General Public License 8 | * as published by the Free Software Foundation; either version 2 9 | * of the License, or (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License 17 | * along with this program; if not, write to the Free Software 18 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | */ 20 | 21 | #ifndef S3FS_AUTOLOCK_H_ 22 | #define S3FS_AUTOLOCK_H_ 23 | 24 | #include 25 | 26 | //------------------------------------------------------------------- 27 | // AutoLock Class 28 | //------------------------------------------------------------------- 29 | class AutoLock 30 | { 31 | public: 32 | enum Type { 33 | NO_WAIT = 1, 34 | ALREADY_LOCKED = 2, 35 | NONE = 0 36 | }; 37 | 38 | private: 39 | pthread_mutex_t* const auto_mutex; 40 | bool is_lock_acquired; 41 | 42 | private: 43 | AutoLock(const AutoLock&) = delete; 44 | AutoLock(AutoLock&&) = delete; 45 | AutoLock& operator=(const AutoLock&) = delete; 46 | AutoLock& operator=(AutoLock&&) = delete; 47 | 48 | public: 49 | explicit AutoLock(pthread_mutex_t* pmutex, Type type = NONE); 50 | ~AutoLock(); 51 | bool isLockAcquired() const; 52 | }; 53 | 54 | #endif // S3FS_AUTOLOCK_H_ 55 | 56 | /* 57 | * Local variables: 58 | * tab-width: 4 59 | * c-basic-offset: 4 60 | * End: 61 | * vim600: expandtab sw=4 ts=4 fdm=marker 62 | * vim<600: expandtab sw=4 ts=4 63 | */ 64 | -------------------------------------------------------------------------------- /s3fs/common.h: -------------------------------------------------------------------------------- 1 | /* 2 | * s3fs - FUSE-based file system backed by Amazon S3 3 | * 4 | * Copyright(C) 2007 Randy Rizun 5 | * 6 | * This program is free software; you can redistribute it and/or 7 | * modify it under the terms of the GNU General Public License 8 | * as published by the Free Software Foundation; either version 2 9 | * of the License, or (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License 17 | * along with this program; if not, write to the Free Software 18 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | */ 20 | 21 | #ifndef S3FS_COMMON_H_ 22 | #define S3FS_COMMON_H_ 23 | 24 | #include 25 | 26 | #include "config.h" 27 | #include "types.h" 28 | #include "hybridcache_accessor_4_s3fs.h" 29 | 30 | //------------------------------------------------------------------- 31 | // Global variables 32 | //------------------------------------------------------------------- 33 | // TODO: namespace these 34 | static constexpr int64_t FIVE_GB = 5LL * 1024LL * 1024LL * 1024LL; 35 | static constexpr off_t MIN_MULTIPART_SIZE = 5 * 1024 * 1024; 36 | static constexpr int NEW_CACHE_FAKE_FD = -2; 37 | static constexpr int BLOCK_SIZE = 4096; 38 | 39 | extern bool foreground; 40 | extern bool nomultipart; 41 | extern bool pathrequeststyle; 42 | extern bool complement_stat; 43 | extern bool noxmlns; 44 | extern bool use_newcache; 45 | extern std::string program_name; 46 | extern std::string service_path; 47 | extern std::string s3host; 48 | extern std::string mount_prefix; 49 | extern std::string endpoint; 50 | extern std::string cipher_suites; 51 | extern std::string instance_name; 52 | 53 | extern std::shared_ptr accessor; 54 | 55 | //------------------------------------------------------------------- 56 | // For weak attribute 57 | //------------------------------------------------------------------- 58 | #define S3FS_FUNCATTR_WEAK __attribute__ ((weak,unused)) 59 | 60 | #endif // S3FS_COMMON_H_ 61 | 62 | /* 63 | * Local variables: 64 | * tab-width: 4 65 | * c-basic-offset: 4 66 | * End: 67 | * vim600: expandtab sw=4 ts=4 fdm=marker 68 | * vim<600: expandtab sw=4 ts=4 69 | */ 70 | -------------------------------------------------------------------------------- /s3fs/common_auth.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * s3fs - FUSE-based file system backed by Amazon S3 3 | * 4 | * Copyright(C) 2007 Randy Rizun 5 | * 6 | * This program is free software; you can redistribute it and/or 7 | * modify it under the terms of the GNU General Public License 8 | * as published by the Free Software Foundation; either version 2 9 | * of the License, or (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License 17 | * along with this program; if not, write to the Free Software 18 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | */ 20 | 21 | #include 22 | #include 23 | 24 | #include "s3fs_auth.h" 25 | #include "string_util.h" 26 | 27 | //------------------------------------------------------------------- 28 | // Utility Function 29 | //------------------------------------------------------------------- 30 | std::string s3fs_get_content_md5(int fd) 31 | { 32 | md5_t md5; 33 | if(!s3fs_md5_fd(fd, 0, -1, &md5)){ 34 | // TODO: better return value? 35 | return ""; 36 | } 37 | return s3fs_base64(md5.data(), md5.size()); 38 | } 39 | 40 | std::string s3fs_sha256_hex_fd(int fd, off_t start, off_t size) 41 | { 42 | sha256_t sha256; 43 | 44 | if(!s3fs_sha256_fd(fd, start, size, &sha256)){ 45 | // TODO: better return value? 46 | return ""; 47 | } 48 | 49 | std::string sha256hex = s3fs_hex_lower(sha256.data(), sha256.size()); 50 | 51 | return sha256hex; 52 | } 53 | 54 | 55 | std::string s3fs_get_content_md5(off_t fsize, char* buf) { 56 | md5_t md5; 57 | if(!s3fs_md5(reinterpret_cast(buf), fsize, &md5)){ 58 | // TODO: better return value? 59 | return ""; 60 | } 61 | return s3fs_base64(md5.data(), md5.size()); 62 | } 63 | 64 | /* 65 | * Local variables: 66 | * tab-width: 4 67 | * c-basic-offset: 4 68 | * End: 69 | * vim600: expandtab sw=4 ts=4 fdm=marker 70 | * vim<600: expandtab sw=4 ts=4 71 | */ 72 | -------------------------------------------------------------------------------- /s3fs/config.h: -------------------------------------------------------------------------------- 1 | /* config.h. Generated from config.h.in by configure. */ 2 | /* config.h.in. Generated from configure.ac by autoheader. */ 3 | 4 | /* short commit hash value on github */ 5 | #define COMMIT_HASH_VAL "70a30d6" 6 | 7 | /* Define to 1 if you have the header file. */ 8 | #define HAVE_ATTR_XATTR_H 1 9 | 10 | /* Define to 1 if you have the `clock_gettime' function. */ 11 | #define HAVE_CLOCK_GETTIME 1 12 | 13 | /* Define to 1 if libcurl has CURLOPT_KEEP_SENDING_ON_ERROR CURLoption */ 14 | #define HAVE_CURLOPT_KEEP_SENDING_ON_ERROR 1 15 | 16 | /* Define to 1 if libcurl has CURLOPT_SSL_ENABLE_ALPN CURLoption */ 17 | #define HAVE_CURLOPT_SSL_ENABLE_ALPN 1 18 | 19 | /* Define to 1 if libcurl has CURLOPT_TCP_KEEPALIVE CURLoption */ 20 | #define HAVE_CURLOPT_TCP_KEEPALIVE 1 21 | 22 | /* Define to 1 if you have the `fallocate' function. */ 23 | #define HAVE_FALLOCATE 1 24 | 25 | /* Define to 1 if you have the header file. */ 26 | #define HAVE_INTTYPES_H 1 27 | 28 | /* Define to 1 if you have the `dl' library (-ldl). */ 29 | #define HAVE_LIBDL 1 30 | 31 | /* Define to 1 if you have the `malloc_trim' function. */ 32 | #define HAVE_MALLOC_TRIM 1 33 | 34 | /* Define to 1 if you have the header file. */ 35 | #define HAVE_MEMORY_H 1 36 | 37 | /* Define to 1 if you have the header file. */ 38 | #define HAVE_STDINT_H 1 39 | 40 | /* Define to 1 if you have the header file. */ 41 | #define HAVE_STDLIB_H 1 42 | 43 | /* Define to 1 if you have the header file. */ 44 | #define HAVE_STRINGS_H 1 45 | 46 | /* Define to 1 if you have the header file. */ 47 | #define HAVE_STRING_H 1 48 | 49 | /* Define to 1 if you have the header file. */ 50 | /* #undef HAVE_SYS_EXTATTR_H */ 51 | 52 | /* Define to 1 if you have the header file. */ 53 | #define HAVE_SYS_STAT_H 1 54 | 55 | /* Define to 1 if you have the header file. */ 56 | #define HAVE_SYS_TYPES_H 1 57 | 58 | /* Define to 1 if you have the header file. */ 59 | #define HAVE_SYS_XATTR_H 1 60 | 61 | /* Define to 1 if you have the header file. */ 62 | #define HAVE_UNISTD_H 1 63 | 64 | /* Name of package */ 65 | #define PACKAGE "s3fs" 66 | 67 | /* Define to the address where bug reports for this package should be sent. */ 68 | #define PACKAGE_BUGREPORT "" 69 | 70 | /* Define to the full name of this package. */ 71 | #define PACKAGE_NAME "s3fs" 72 | 73 | /* Define to the full name and version of this package. */ 74 | #define PACKAGE_STRING "s3fs 1.94" 75 | 76 | /* Define to the one symbol short name of this package. */ 77 | #define PACKAGE_TARNAME "s3fs" 78 | 79 | /* Define to the home page for this package. */ 80 | #define PACKAGE_URL "" 81 | 82 | /* Define to the version of this package. */ 83 | #define PACKAGE_VERSION "1.94" 84 | 85 | /* Define if you have PTHREAD_MUTEX_RECURSIVE_NP */ 86 | #define S3FS_MUTEX_RECURSIVE PTHREAD_MUTEX_RECURSIVE 87 | 88 | /* Define to 1 if you have the ANSI C header files. */ 89 | #define STDC_HEADERS 1 90 | 91 | /* Version number of package */ 92 | #define VERSION "1.94" 93 | -------------------------------------------------------------------------------- /s3fs/curl_handlerpool.h: -------------------------------------------------------------------------------- 1 | /* 2 | * s3fs - FUSE-based file system backed by Amazon S3 3 | * 4 | * Copyright(C) 2007 Randy Rizun 5 | * 6 | * This program is free software; you can redistribute it and/or 7 | * modify it under the terms of the GNU General Public License 8 | * as published by the Free Software Foundation; either version 2 9 | * of the License, or (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License 17 | * along with this program; if not, write to the Free Software 18 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | */ 20 | 21 | #ifndef S3FS_CURL_HANDLERPOOL_H_ 22 | #define S3FS_CURL_HANDLERPOOL_H_ 23 | 24 | #include 25 | #include 26 | #include 27 | 28 | //---------------------------------------------- 29 | // Typedefs 30 | //---------------------------------------------- 31 | typedef std::list hcurllist_t; 32 | 33 | //---------------------------------------------- 34 | // class CurlHandlerPool 35 | //---------------------------------------------- 36 | class CurlHandlerPool 37 | { 38 | public: 39 | explicit CurlHandlerPool(int maxHandlers) : mMaxHandlers(maxHandlers) 40 | { 41 | assert(maxHandlers > 0); 42 | } 43 | CurlHandlerPool(const CurlHandlerPool&) = delete; 44 | CurlHandlerPool(CurlHandlerPool&&) = delete; 45 | CurlHandlerPool& operator=(const CurlHandlerPool&) = delete; 46 | CurlHandlerPool& operator=(CurlHandlerPool&&) = delete; 47 | 48 | bool Init(); 49 | bool Destroy(); 50 | 51 | CURL* GetHandler(bool only_pool); 52 | void ReturnHandler(CURL* hCurl, bool restore_pool); 53 | void ResetHandler(CURL* hCurl); 54 | 55 | private: 56 | int mMaxHandlers; 57 | pthread_mutex_t mLock; 58 | hcurllist_t mPool; 59 | }; 60 | 61 | #endif // S3FS_CURL_HANDLERPOOL_H_ 62 | 63 | /* 64 | * Local variables: 65 | * tab-width: 4 66 | * c-basic-offset: 4 67 | * End: 68 | * vim600: expandtab sw=4 ts=4 fdm=marker 69 | * vim<600: expandtab sw=4 ts=4 70 | */ 71 | -------------------------------------------------------------------------------- /s3fs/curl_util.h: -------------------------------------------------------------------------------- 1 | /* 2 | * s3fs - FUSE-based file system backed by Amazon S3 3 | * 4 | * Copyright(C) 2007 Randy Rizun 5 | * 6 | * This program is free software; you can redistribute it and/or 7 | * modify it under the terms of the GNU General Public License 8 | * as published by the Free Software Foundation; either version 2 9 | * of the License, or (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License 17 | * along with this program; if not, write to the Free Software 18 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | */ 20 | 21 | #ifndef S3FS_CURL_UTIL_H_ 22 | #define S3FS_CURL_UTIL_H_ 23 | 24 | #include 25 | 26 | enum class sse_type_t; 27 | 28 | //---------------------------------------------- 29 | // Functions 30 | //---------------------------------------------- 31 | struct curl_slist* curl_slist_sort_insert(struct curl_slist* list, const char* key, const char* value); 32 | struct curl_slist* curl_slist_remove(struct curl_slist* list, const char* key); 33 | std::string get_sorted_header_keys(const struct curl_slist* list); 34 | std::string get_canonical_headers(const struct curl_slist* list, bool only_amz = false); 35 | std::string get_header_value(const struct curl_slist* list, const std::string &key); 36 | bool MakeUrlResource(const char* realpath, std::string& resourcepath, std::string& url); 37 | std::string prepare_url(const char* url); 38 | bool get_object_sse_type(const char* path, sse_type_t& ssetype, std::string& ssevalue); // implement in s3fs.cpp 39 | 40 | bool make_md5_from_binary(const char* pstr, size_t length, std::string& md5); 41 | std::string url_to_host(const std::string &url); 42 | std::string get_bucket_host(); 43 | const char* getCurlDebugHead(curl_infotype type); 44 | 45 | bool etag_equals(const std::string& s1, const std::string& s2); 46 | 47 | #endif // S3FS_CURL_UTIL_H_ 48 | 49 | /* 50 | * Local variables: 51 | * tab-width: 4 52 | * c-basic-offset: 4 53 | * End: 54 | * vim600: expandtab sw=4 ts=4 fdm=marker 55 | * vim<600: expandtab sw=4 ts=4 56 | */ 57 | -------------------------------------------------------------------------------- /s3fs/fdcache_auto.h: -------------------------------------------------------------------------------- 1 | /* 2 | * s3fs - FUSE-based file system backed by Amazon S3 3 | * 4 | * Copyright(C) 2007 Randy Rizun 5 | * 6 | * This program is free software; you can redistribute it and/or 7 | * modify it under the terms of the GNU General Public License 8 | * as published by the Free Software Foundation; either version 2 9 | * of the License, or (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License 17 | * along with this program; if not, write to the Free Software 18 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | */ 20 | 21 | #ifndef S3FS_FDCACHE_AUTO_H_ 22 | #define S3FS_FDCACHE_AUTO_H_ 23 | 24 | #include 25 | 26 | #include "autolock.h" 27 | #include "metaheader.h" 28 | 29 | class FdEntity; 30 | 31 | //------------------------------------------------ 32 | // class AutoFdEntity 33 | //------------------------------------------------ 34 | // A class that opens fdentry and closes it automatically. 35 | // This class object is used to prevent inconsistencies in 36 | // the number of references in fdentry. 37 | // The methods are wrappers to the method of the FdManager class. 38 | // 39 | class AutoFdEntity 40 | { 41 | private: 42 | FdEntity* pFdEntity; 43 | int pseudo_fd; 44 | 45 | private: 46 | AutoFdEntity(const AutoFdEntity&) = delete; 47 | AutoFdEntity(AutoFdEntity&&) = delete; 48 | AutoFdEntity& operator=(const AutoFdEntity&) = delete; 49 | AutoFdEntity& operator=(AutoFdEntity&&) = delete; 50 | 51 | public: 52 | AutoFdEntity(); 53 | ~AutoFdEntity(); 54 | 55 | bool Close(); 56 | int Detach(); 57 | FdEntity* Attach(const char* path, int existfd); 58 | int GetPseudoFd() const { return pseudo_fd; } 59 | 60 | FdEntity* Open(const char* path, const headers_t* pmeta, off_t size, const struct timespec& ts_mctime, int flags, bool force_tmpfile, bool is_create, bool ignore_modify, AutoLock::Type type, int* error = nullptr); 61 | FdEntity* GetExistFdEntity(const char* path, int existfd = -1); 62 | FdEntity* OpenExistFdEntity(const char* path, int flags = O_RDONLY); 63 | }; 64 | 65 | #endif // S3FS_FDCACHE_AUTO_H_ 66 | 67 | /* 68 | * Local variables: 69 | * tab-width: 4 70 | * c-basic-offset: 4 71 | * End: 72 | * vim600: expandtab sw=4 ts=4 fdm=marker 73 | * vim<600: expandtab sw=4 ts=4 74 | */ 75 | -------------------------------------------------------------------------------- /s3fs/fdcache_pseudofd.h: -------------------------------------------------------------------------------- 1 | /* 2 | * s3fs - FUSE-based file system backed by Amazon S3 3 | * 4 | * Copyright(C) 2007 Randy Rizun 5 | * 6 | * This program is free software; you can redistribute it and/or 7 | * modify it under the terms of the GNU General Public License 8 | * as published by the Free Software Foundation; either version 2 9 | * of the License, or (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License 17 | * along with this program; if not, write to the Free Software 18 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | */ 20 | 21 | #ifndef S3FS_FDCACHE_PSEUDOFD_H_ 22 | #define S3FS_FDCACHE_PSEUDOFD_H_ 23 | 24 | #include 25 | 26 | //------------------------------------------------ 27 | // Typdefs 28 | //------------------------------------------------ 29 | // List of pseudo fd in use 30 | // 31 | typedef std::vector pseudofd_list_t; 32 | 33 | //------------------------------------------------ 34 | // Class PseudoFdManager 35 | //------------------------------------------------ 36 | class PseudoFdManager 37 | { 38 | private: 39 | pseudofd_list_t pseudofd_list; 40 | bool is_lock_init; 41 | pthread_mutex_t pseudofd_list_lock; // protects pseudofd_list 42 | 43 | private: 44 | static PseudoFdManager& GetManager(); 45 | 46 | PseudoFdManager(); 47 | ~PseudoFdManager(); 48 | PseudoFdManager(const PseudoFdManager&) = delete; 49 | PseudoFdManager(PseudoFdManager&&) = delete; 50 | PseudoFdManager& operator=(const PseudoFdManager&) = delete; 51 | PseudoFdManager& operator=(PseudoFdManager&&) = delete; 52 | 53 | int GetUnusedMinPseudoFd() const; 54 | int CreatePseudoFd(); 55 | bool ReleasePseudoFd(int fd); 56 | 57 | public: 58 | static int Get(); 59 | static bool Release(int fd); 60 | }; 61 | 62 | #endif // S3FS_FDCACHE_PSEUDOFD_H_ 63 | 64 | /* 65 | * Local variables: 66 | * tab-width: 4 67 | * c-basic-offset: 4 68 | * End: 69 | * vim600: expandtab sw=4 ts=4 fdm=marker 70 | * vim<600: expandtab sw=4 ts=4 71 | */ 72 | -------------------------------------------------------------------------------- /s3fs/fdcache_stat.h: -------------------------------------------------------------------------------- 1 | /* 2 | * s3fs - FUSE-based file system backed by Amazon S3 3 | * 4 | * Copyright(C) 2007 Randy Rizun 5 | * 6 | * This program is free software; you can redistribute it and/or 7 | * modify it under the terms of the GNU General Public License 8 | * as published by the Free Software Foundation; either version 2 9 | * of the License, or (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License 17 | * along with this program; if not, write to the Free Software 18 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | */ 20 | 21 | #ifndef S3FS_FDCACHE_STAT_H_ 22 | #define S3FS_FDCACHE_STAT_H_ 23 | 24 | #include 25 | 26 | //------------------------------------------------ 27 | // CacheFileStat 28 | //------------------------------------------------ 29 | class CacheFileStat 30 | { 31 | private: 32 | std::string path; 33 | int fd; 34 | 35 | private: 36 | static int MakeCacheFileStatPath(const char* path, std::string& sfile_path, bool is_create_dir = true); 37 | 38 | bool RawOpen(bool readonly); 39 | 40 | public: 41 | static std::string GetCacheFileStatTopDir(); 42 | static int DeleteCacheFileStat(const char* path); 43 | static bool CheckCacheFileStatTopDir(); 44 | static bool DeleteCacheFileStatDirectory(); 45 | static bool RenameCacheFileStat(const char* oldpath, const char* newpath); 46 | 47 | explicit CacheFileStat(const char* tpath = nullptr); 48 | ~CacheFileStat(); 49 | 50 | bool Open(); 51 | bool ReadOnlyOpen(); 52 | bool Release(); 53 | bool SetPath(const char* tpath, bool is_open = true); 54 | int GetFd() const { return fd; } 55 | }; 56 | 57 | #endif // S3FS_FDCACHE_STAT_H_ 58 | 59 | /* 60 | * Local variables: 61 | * tab-width: 4 62 | * c-basic-offset: 4 63 | * End: 64 | * vim600: expandtab sw=4 ts=4 fdm=marker 65 | * vim<600: expandtab sw=4 ts=4 66 | */ 67 | -------------------------------------------------------------------------------- /s3fs/fdcache_untreated.h: -------------------------------------------------------------------------------- 1 | /* 2 | * s3fs - FUSE-based file system backed by Amazon S3 3 | * 4 | * Copyright(C) 2007 Randy Rizun 5 | * 6 | * This program is free software; you can redistribute it and/or 7 | * modify it under the terms of the GNU General Public License 8 | * as published by the Free Software Foundation; either version 2 9 | * of the License, or (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License 17 | * along with this program; if not, write to the Free Software 18 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | */ 20 | 21 | #ifndef S3FS_FDCACHE_UNTREATED_H_ 22 | #define S3FS_FDCACHE_UNTREATED_H_ 23 | 24 | #include "common.h" 25 | #include "types.h" 26 | 27 | //------------------------------------------------ 28 | // Class UntreatedParts 29 | //------------------------------------------------ 30 | class UntreatedParts 31 | { 32 | private: 33 | mutable pthread_mutex_t untreated_list_lock; // protects untreated_list 34 | bool is_lock_init; 35 | 36 | untreated_list_t untreated_list; 37 | long last_tag; // [NOTE] Use this to identify the latest updated part. 38 | 39 | private: 40 | bool RowGetPart(off_t& start, off_t& size, off_t max_size, off_t min_size, bool lastpart) const; 41 | 42 | public: 43 | UntreatedParts(); 44 | ~UntreatedParts(); 45 | UntreatedParts(const UntreatedParts&) = delete; 46 | UntreatedParts(UntreatedParts&&) = delete; 47 | UntreatedParts& operator=(const UntreatedParts&) = delete; 48 | UntreatedParts& operator=(UntreatedParts&&) = delete; 49 | 50 | bool empty(); 51 | 52 | bool AddPart(off_t start, off_t size); 53 | bool GetLastUpdatedPart(off_t& start, off_t& size, off_t max_size, off_t min_size = MIN_MULTIPART_SIZE) const { return RowGetPart(start, size, max_size, min_size, true); } 54 | 55 | bool ClearParts(off_t start, off_t size); 56 | bool ClearAll() { return ClearParts(0, 0); } 57 | 58 | bool GetLastUpdatePart(off_t& start, off_t& size) const; 59 | bool ReplaceLastUpdatePart(off_t start, off_t size); 60 | bool RemoveLastUpdatePart(); 61 | 62 | bool Duplicate(untreated_list_t& list); 63 | 64 | void Dump(); 65 | }; 66 | 67 | #endif // S3FS_FDCACHE_UNTREATED_H_ 68 | 69 | /* 70 | * Local variables: 71 | * tab-width: 4 72 | * c-basic-offset: 4 73 | * End: 74 | * vim600: expandtab sw=4 ts=4 fdm=marker 75 | * vim<600: expandtab sw=4 ts=4 76 | */ 77 | -------------------------------------------------------------------------------- /s3fs/hybridcache_accessor_4_s3fs.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Project: HybridCache 3 | * Created Date: 24-3-25 4 | * Author: lshb 5 | */ 6 | 7 | #ifndef HYBRIDCACHE_ACCESSOR_4_S3FS_H_ 8 | #define HYBRIDCACHE_ACCESSOR_4_S3FS_H_ 9 | 10 | #include 11 | 12 | #include "accessor.h" 13 | 14 | using atomic_ptr_t = std::shared_ptr>; 15 | 16 | // added by tqy referring to xyq 17 | using Cache = facebook::cachelib::LruAllocator; 18 | using facebook::cachelib::PoolId; 19 | #define MAXSIZE 1024 20 | #define IP_ADDR "127.0.0.1" 21 | #define IP_PORT 2333 // 服务器端口 22 | const int64_t FIX_SIZE = 1024 * 1024 * 256; // 256M为分配单位 23 | const int64_t RESERVE_SIZE = 1024 * 1024 * 512; // 512M保留空间 24 | 25 | class HybridCacheAccessor4S3fs : public HybridCache::HybridCacheAccessor { 26 | public: 27 | HybridCacheAccessor4S3fs(const HybridCache::HybridCacheConfig& cfg); 28 | ~HybridCacheAccessor4S3fs(); 29 | 30 | void Init(); 31 | 32 | // added by tqy referring to xyq 33 | int InitCache(); 34 | void LinUCBClient(); 35 | 36 | void Stop(); 37 | 38 | int Put(const std::string &key, size_t start, size_t len, const char* buf); 39 | 40 | int Get(const std::string &key, size_t start, size_t len, char* buf); 41 | 42 | int Flush(const std::string &key); 43 | 44 | int DeepFlush(const std::string &key); 45 | 46 | int Delete(const std::string &key); 47 | 48 | int Truncate(const std::string &key, size_t size); 49 | 50 | int Invalidate(const std::string &key); 51 | 52 | int Head(const std::string &key, size_t& size, 53 | std::map& headers); 54 | 55 | // async full files flush in background 56 | int FsSync(); 57 | 58 | bool UseGlobalCache(); 59 | 60 | HybridCache::ThreadPool* GetExecutor() { 61 | return executor_.get(); 62 | } 63 | 64 | private: 65 | void InitLog(); 66 | bool IsWriteCacheFull(size_t len); 67 | uint32_t WriteCacheRatio(); 68 | bool IsWritePoolFull(size_t len); 69 | uint32_t WritePoolRatio(); 70 | void BackGroundFlush(); 71 | 72 | private: 73 | folly::ConcurrentHashMap fileLock_; // rwlock. write and flush are exclusive 74 | std::shared_ptr executor_; 75 | std::shared_ptr tokenBucket_; // upload flow limit 76 | std::atomic toStop_{false}; 77 | std::atomic backFlushRunning_{false}; 78 | std::thread bgFlushThread_; 79 | 80 | // added by tqy referring to xyq for Resizing 81 | std::shared_ptr ResizeWriteCache_; 82 | std::shared_ptr ResizeReadCache_; 83 | PoolId writePoolId_; 84 | PoolId readPoolId_; 85 | uint64_t writeCount_ = 0; 86 | uint64_t readCount_ = 0; 87 | uint64_t writeCacheSize_; 88 | uint64_t readCacheSize_; 89 | 90 | // added by tqy referring to xyq for LinUCB 91 | std::thread LinUCBThread_; 92 | std::atomic stopLinUCBThread_{false}; 93 | uint64_t writeByteAcc_ = 0; 94 | uint64_t readByteAcc_ = 0; 95 | uint64_t writeTimeAcc_ = 0; 96 | uint64_t readTimeAcc_ = 0; 97 | uint32_t resizeInterval_; 98 | }; 99 | 100 | #endif // HYBRIDCACHE_ACCESSOR_4_S3FS_H_ 101 | -------------------------------------------------------------------------------- /s3fs/hybridcache_disk_data_adaptor.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Project: HybridCache 3 | * Created Date: 24-6-7 4 | * Author: lshb 5 | */ 6 | 7 | #ifndef DISK_DATA_ADAPTOR_H_ 8 | #define DISK_DATA_ADAPTOR_H_ 9 | 10 | #include "data_adaptor.h" 11 | 12 | using HybridCache::ByteBuffer; 13 | 14 | class DiskDataAdaptor : public HybridCache::DataAdaptor { 15 | public: 16 | DiskDataAdaptor(std::shared_ptr dataAdaptor) : dataAdaptor_(dataAdaptor) {} 17 | DiskDataAdaptor() = default; 18 | ~DiskDataAdaptor() {} 19 | 20 | folly::Future DownLoad(const std::string &key, 21 | size_t start, 22 | size_t size, 23 | ByteBuffer &buffer); 24 | 25 | folly::Future UpLoad(const std::string &key, 26 | size_t size, 27 | const ByteBuffer &buffer, 28 | const std::map& headers); 29 | 30 | folly::Future Delete(const std::string &key); 31 | 32 | folly::Future Head(const std::string &key, 33 | size_t& size, 34 | std::map& headers); 35 | 36 | private: 37 | std::shared_ptr dataAdaptor_; 38 | }; 39 | 40 | #endif // DISK_DATA_ADAPTOR_H_ 41 | -------------------------------------------------------------------------------- /s3fs/hybridcache_s3_data_adaptor.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Project: HybridCache 3 | * Created Date: 24-3-11 4 | * Author: lshb 5 | */ 6 | 7 | #ifndef S3_DATA_ADAPTOR_H_ 8 | #define S3_DATA_ADAPTOR_H_ 9 | 10 | #include "data_adaptor.h" 11 | 12 | using HybridCache::ByteBuffer; 13 | 14 | class S3DataAdaptor : public HybridCache::DataAdaptor { 15 | public: 16 | folly::Future DownLoad(const std::string &key, 17 | size_t start, 18 | size_t size, 19 | ByteBuffer &buffer); 20 | 21 | folly::Future UpLoad(const std::string &key, 22 | size_t size, 23 | const ByteBuffer &buffer, 24 | const std::map& headers); 25 | 26 | folly::Future Delete(const std::string &key); 27 | 28 | folly::Future Head(const std::string &key, 29 | size_t& size, 30 | std::map& headers); 31 | }; 32 | 33 | #endif // S3_DATA_ADAPTOR_H_ 34 | -------------------------------------------------------------------------------- /s3fs/metaheader.h: -------------------------------------------------------------------------------- 1 | /* 2 | * s3fs - FUSE-based file system backed by Amazon S3 3 | * 4 | * Copyright(C) 2007 Randy Rizun 5 | * 6 | * This program is free software; you can redistribute it and/or 7 | * modify it under the terms of the GNU General Public License 8 | * as published by the Free Software Foundation; either version 2 9 | * of the License, or (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License 17 | * along with this program; if not, write to the Free Software 18 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | */ 20 | 21 | #ifndef S3FS_METAHEADER_H_ 22 | #define S3FS_METAHEADER_H_ 23 | 24 | #include 25 | #include 26 | #include 27 | 28 | //------------------------------------------------------------------- 29 | // headers_t 30 | //------------------------------------------------------------------- 31 | struct header_nocase_cmp 32 | { 33 | bool operator()(const std::string &strleft, const std::string &strright) const 34 | { 35 | return (strcasecmp(strleft.c_str(), strright.c_str()) < 0); 36 | } 37 | }; 38 | typedef std::map headers_t; 39 | 40 | //------------------------------------------------------------------- 41 | // Functions 42 | //------------------------------------------------------------------- 43 | struct timespec get_mtime(const headers_t& meta, bool overcheck = true); 44 | struct timespec get_ctime(const headers_t& meta, bool overcheck = true); 45 | struct timespec get_atime(const headers_t& meta, bool overcheck = true); 46 | off_t get_size(const char *s); 47 | off_t get_size(const headers_t& meta); 48 | mode_t get_mode(const char *s, int base = 0); 49 | mode_t get_mode(const headers_t& meta, const std::string& strpath, bool checkdir = false, bool forcedir = false); 50 | uid_t get_uid(const char *s); 51 | uid_t get_uid(const headers_t& meta); 52 | gid_t get_gid(const char *s); 53 | gid_t get_gid(const headers_t& meta); 54 | blkcnt_t get_blocks(off_t size); 55 | time_t cvtIAMExpireStringToTime(const char* s); 56 | time_t get_lastmodified(const char* s); 57 | time_t get_lastmodified(const headers_t& meta); 58 | bool is_need_check_obj_detail(const headers_t& meta); 59 | bool merge_headers(headers_t& base, const headers_t& additional, bool add_noexist); 60 | bool simple_parse_xml(const char* data, size_t len, const char* key, std::string& value); 61 | 62 | #endif // S3FS_METAHEADER_H_ 63 | 64 | /* 65 | * Local variables: 66 | * tab-width: 4 67 | * c-basic-offset: 4 68 | * End: 69 | * vim600: expandtab sw=4 ts=4 fdm=marker 70 | * vim<600: expandtab sw=4 ts=4 71 | */ 72 | -------------------------------------------------------------------------------- /s3fs/mpu_util.h: -------------------------------------------------------------------------------- 1 | /* 2 | * s3fs - FUSE-based file system backed by Amazon S3 3 | * 4 | * Copyright(C) 2007 Randy Rizun 5 | * 6 | * This program is free software; you can redistribute it and/or 7 | * modify it under the terms of the GNU General Public License 8 | * as published by the Free Software Foundation; either version 2 9 | * of the License, or (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License 17 | * along with this program; if not, write to the Free Software 18 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | */ 20 | 21 | #ifndef S3FS_MPU_UTIL_H_ 22 | #define S3FS_MPU_UTIL_H_ 23 | 24 | #include 25 | #include 26 | 27 | //------------------------------------------------------------------- 28 | // Structure / Typedef 29 | //------------------------------------------------------------------- 30 | typedef struct incomplete_multipart_upload_info 31 | { 32 | std::string key; 33 | std::string id; 34 | std::string date; 35 | }INCOMP_MPU_INFO; 36 | 37 | typedef std::vector incomp_mpu_list_t; 38 | 39 | //------------------------------------------------------------------- 40 | // enum for utility process mode 41 | //------------------------------------------------------------------- 42 | enum class utility_incomp_type{ 43 | NO_UTILITY_MODE = 0, // not utility mode 44 | INCOMP_TYPE_LIST, // list of incomplete mpu 45 | INCOMP_TYPE_ABORT // delete incomplete mpu 46 | }; 47 | 48 | extern utility_incomp_type utility_mode; 49 | 50 | //------------------------------------------------------------------- 51 | // Functions 52 | //------------------------------------------------------------------- 53 | int s3fs_utility_processing(time_t abort_time); 54 | 55 | #endif // S3FS_MPU_UTIL_H_ 56 | 57 | /* 58 | * Local variables: 59 | * tab-width: 4 60 | * c-basic-offset: 4 61 | * End: 62 | * vim600: expandtab sw=4 ts=4 fdm=marker 63 | * vim<600: expandtab sw=4 ts=4 64 | */ 65 | -------------------------------------------------------------------------------- /s3fs/s3fs.h: -------------------------------------------------------------------------------- 1 | /* 2 | * s3fs - FUSE-based file system backed by Amazon S3 3 | * 4 | * Copyright(C) 2007 Randy Rizun 5 | * 6 | * This program is free software; you can redistribute it and/or 7 | * modify it under the terms of the GNU General Public License 8 | * as published by the Free Software Foundation; either version 2 9 | * of the License, or (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License 17 | * along with this program; if not, write to the Free Software 18 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | */ 20 | 21 | #ifndef S3FS_S3FS_H_ 22 | #define S3FS_S3FS_H_ 23 | 24 | #define FUSE_USE_VERSION 26 25 | 26 | #include 27 | 28 | #define S3FS_FUSE_EXIT() \ 29 | do{ \ 30 | struct fuse_context* pcxt = fuse_get_context(); \ 31 | if(pcxt){ \ 32 | fuse_exit(pcxt->fuse); \ 33 | } \ 34 | }while(0) 35 | 36 | // [NOTE] 37 | // s3fs use many small allocated chunk in heap area for stats 38 | // cache and parsing xml, etc. The OS may decide that giving 39 | // this little memory back to the kernel will cause too much 40 | // overhead and delay the operation. 41 | // Address of gratitude, this workaround quotes a document of 42 | // libxml2.( http://xmlsoft.org/xmlmem.html ) 43 | // 44 | // When valgrind is used to test memory leak of s3fs, a large 45 | // amount of chunk may be reported. You can check the memory 46 | // release accurately by defining the S3FS_MALLOC_TRIM flag 47 | // and building it. Also, when executing s3fs, you can define 48 | // the MMAP_THRESHOLD environment variable and check more 49 | // accurate memory leak.( see, man 3 free ) 50 | // 51 | #ifdef S3FS_MALLOC_TRIM 52 | #ifdef HAVE_MALLOC_TRIM 53 | #include 54 | #define S3FS_MALLOCTRIM(pad) malloc_trim(pad) 55 | #else // HAVE_MALLOC_TRIM 56 | #define S3FS_MALLOCTRIM(pad) 57 | #endif // HAVE_MALLOC_TRIM 58 | #else // S3FS_MALLOC_TRIM 59 | #define S3FS_MALLOCTRIM(pad) 60 | #endif // S3FS_MALLOC_TRIM 61 | 62 | #define S3FS_XMLFREEDOC(doc) \ 63 | do{ \ 64 | xmlFreeDoc(doc); \ 65 | S3FS_MALLOCTRIM(0); \ 66 | }while(0) 67 | #define S3FS_XMLFREE(ptr) \ 68 | do{ \ 69 | xmlFree(ptr); \ 70 | S3FS_MALLOCTRIM(0); \ 71 | }while(0) 72 | #define S3FS_XMLXPATHFREECONTEXT(ctx) \ 73 | do{ \ 74 | xmlXPathFreeContext(ctx); \ 75 | S3FS_MALLOCTRIM(0); \ 76 | }while(0) 77 | #define S3FS_XMLXPATHFREEOBJECT(obj) \ 78 | do{ \ 79 | xmlXPathFreeObject(obj); \ 80 | S3FS_MALLOCTRIM(0); \ 81 | }while(0) 82 | 83 | #endif // S3FS_S3FS_H_ 84 | 85 | /* 86 | * Local variables: 87 | * tab-width: 4 88 | * c-basic-offset: 4 89 | * End: 90 | * vim600: expandtab sw=4 ts=4 fdm=marker 91 | * vim<600: expandtab sw=4 ts=4 92 | */ 93 | -------------------------------------------------------------------------------- /s3fs/s3fs_auth.h: -------------------------------------------------------------------------------- 1 | /* 2 | * s3fs - FUSE-based file system backed by Amazon S3 3 | * 4 | * Copyright(C) 2007 Randy Rizun 5 | * 6 | * This program is free software; you can redistribute it and/or 7 | * modify it under the terms of the GNU General Public License 8 | * as published by the Free Software Foundation; either version 2 9 | * of the License, or (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License 17 | * along with this program; if not, write to the Free Software 18 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | */ 20 | 21 | #ifndef S3FS_AUTH_H_ 22 | #define S3FS_AUTH_H_ 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | typedef std::array md5_t; 30 | typedef std::array sha256_t; 31 | 32 | //------------------------------------------------------------------- 33 | // Utility functions for Authentication 34 | //------------------------------------------------------------------- 35 | // 36 | // in common_auth.cpp 37 | // 38 | std::string s3fs_get_content_md5(int fd); 39 | std::string s3fs_sha256_hex_fd(int fd, off_t start, off_t size); 40 | std::string s3fs_get_content_md5(off_t fsize, char* buf); 41 | 42 | // 43 | // in xxxxxx_auth.cpp 44 | // 45 | const char* s3fs_crypt_lib_name(); 46 | bool s3fs_init_global_ssl(); 47 | bool s3fs_destroy_global_ssl(); 48 | bool s3fs_init_crypt_mutex(); 49 | bool s3fs_destroy_crypt_mutex(); 50 | std::unique_ptr s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned int* digestlen); 51 | std::unique_ptr s3fs_HMAC256(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned int* digestlen); 52 | bool s3fs_md5(const unsigned char* data, size_t datalen, md5_t* result); 53 | bool s3fs_md5_fd(int fd, off_t start, off_t size, md5_t* result); 54 | bool s3fs_sha256(const unsigned char* data, size_t datalen, sha256_t* digest); 55 | bool s3fs_sha256_fd(int fd, off_t start, off_t size, sha256_t* result); 56 | 57 | #endif // S3FS_AUTH_H_ 58 | 59 | /* 60 | * Local variables: 61 | * tab-width: 4 62 | * c-basic-offset: 4 63 | * End: 64 | * vim600: expandtab sw=4 ts=4 fdm=marker 65 | * vim<600: expandtab sw=4 ts=4 66 | */ 67 | -------------------------------------------------------------------------------- /s3fs/s3fs_global.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * s3fs - FUSE-based file system backed by Amazon S3 3 | * 4 | * Copyright(C) 2007 Takeshi Nakatani 5 | * 6 | * This program is free software; you can redistribute it and/or 7 | * modify it under the terms of the GNU General Public License 8 | * as published by the Free Software Foundation; either version 2 9 | * of the License, or (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License 17 | * along with this program; if not, write to the Free Software 18 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | */ 20 | 21 | #include 22 | #include 23 | 24 | #include "hybridcache_accessor_4_s3fs.h" 25 | 26 | //------------------------------------------------------------------- 27 | // Global variables 28 | //------------------------------------------------------------------- 29 | bool foreground = false; 30 | bool nomultipart = false; 31 | bool pathrequeststyle = false; 32 | bool complement_stat = false; 33 | bool noxmlns = false; 34 | bool use_newcache = false; 35 | std::string program_name; 36 | std::string service_path = "/"; 37 | std::string s3host = "https://s3.amazonaws.com"; 38 | std::string endpoint = "us-east-1"; 39 | std::string cipher_suites; 40 | std::string instance_name; 41 | std::shared_ptr accessor; 42 | 43 | /* 44 | * Local variables: 45 | * tab-width: 4 46 | * c-basic-offset: 4 47 | * End: 48 | * vim600: expandtab sw=4 ts=4 fdm=marker 49 | * vim<600: expandtab sw=4 ts=4 50 | */ 51 | -------------------------------------------------------------------------------- /s3fs/s3fs_help.h: -------------------------------------------------------------------------------- 1 | /* 2 | * s3fs - FUSE-based file system backed by Amazon S3 3 | * 4 | * Copyright(C) 2007 Randy Rizun 5 | * 6 | * This program is free software; you can redistribute it and/or 7 | * modify it under the terms of the GNU General Public License 8 | * as published by the Free Software Foundation; either version 2 9 | * of the License, or (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License 17 | * along with this program; if not, write to the Free Software 18 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | */ 20 | 21 | #ifndef S3FS_S3FS_HELP_H_ 22 | #define S3FS_S3FS_HELP_H_ 23 | 24 | //------------------------------------------------------------------- 25 | // Functions 26 | //------------------------------------------------------------------- 27 | void show_usage(); 28 | void show_help(); 29 | void show_version(); 30 | const char* short_version(); 31 | 32 | #endif // S3FS_S3FS_HELP_H_ 33 | 34 | /* 35 | * Local variables: 36 | * tab-width: 4 37 | * c-basic-offset: 4 38 | * End: 39 | * vim600: expandtab sw=4 ts=4 fdm=marker 40 | * vim<600: expandtab sw=4 ts=4 41 | */ 42 | -------------------------------------------------------------------------------- /s3fs/s3fs_lib.h: -------------------------------------------------------------------------------- 1 | #ifndef S3FS_S3FS_LIB_H_ 2 | #define S3FS_S3FS_LIB_H_ 3 | 4 | 5 | #include 6 | #include 7 | 8 | 9 | 10 | #ifdef S3FS_MALLOC_TRIM 11 | #ifdef HAVE_MALLOC_TRIM 12 | #include 13 | #define S3FS_MALLOCTRIM(pad) malloc_trim(pad) 14 | #else // HAVE_MALLOC_TRIM 15 | #define S3FS_MALLOCTRIM(pad) 16 | #endif // HAVE_MALLOC_TRIM 17 | #else // S3FS_MALLOC_TRIM 18 | #define S3FS_MALLOCTRIM(pad) 19 | #endif // S3FS_MALLOC_TRIM 20 | 21 | 22 | //------------------------------------------------------------------- 23 | // posix interface functions 24 | //------------------------------------------------------------------- 25 | #ifdef __cplusplus 26 | extern "C" { 27 | #endif 28 | 29 | struct S3DirStream; 30 | 31 | void s3fs_global_init(); 32 | 33 | void s3fs_global_uninit(); 34 | 35 | int posix_s3fs_create(const char* _path, int flags, mode_t mode); 36 | 37 | int posix_s3fs_open(const char* _path, int flags, mode_t mode); 38 | 39 | int posix_s3fs_multiread(int fd, void* buf, size_t size, off_t file_offset); 40 | 41 | int posix_s3fs_read(int fd, void* buf, size_t size); 42 | 43 | int posix_s3fs_multiwrite(int fd, const void* buf, size_t size, off_t file_offset); 44 | 45 | int posix_s3fs_write(int fd, const void* buf, size_t size); 46 | 47 | off_t posix_s3fs_lseek(int fd, off_t offset, int whence); 48 | 49 | int posix_s3fs_close(int fd); 50 | 51 | int posix_s3fs_stat(const char* _path, struct stat* stbuf); 52 | 53 | int posix_s3fs_fstat(int fd, struct stat* stbuf) ; 54 | 55 | int posix_s3fs_mkdir(const char* _path, mode_t mode); 56 | 57 | int posix_s3fs_opendir(const char* _path, S3DirStream* dirstream); 58 | 59 | int posix_s3fs_getdents(S3DirStream* dirstream, char* contents, size_t maxread, ssize_t* realbytes); 60 | 61 | int posix_s3fs_closedir(S3DirStream* dirstream); 62 | 63 | int posix_s3fs_unlink(const char* _path); 64 | 65 | #ifdef __cplusplus 66 | } 67 | #endif 68 | 69 | #endif // S3FS_S3FS_LIB_H_ 70 | -------------------------------------------------------------------------------- /s3fs/s3fs_version.md: -------------------------------------------------------------------------------- 1 | commit 70a30d6e26a5dfd07a00cf79ce1196079e5ab11a (tag: v1.94) 2 | Date: Fri Feb 23 12:56:01 2024 +0900 3 | -------------------------------------------------------------------------------- /s3fs/s3fs_xml.h: -------------------------------------------------------------------------------- 1 | /* 2 | * s3fs - FUSE-based file system backed by Amazon S3 3 | * 4 | * Copyright(C) 2007 Randy Rizun 5 | * 6 | * This program is free software; you can redistribute it and/or 7 | * modify it under the terms of the GNU General Public License 8 | * as published by the Free Software Foundation; either version 2 9 | * of the License, or (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License 17 | * along with this program; if not, write to the Free Software 18 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | */ 20 | 21 | #ifndef S3FS_S3FS_XML_H_ 22 | #define S3FS_S3FS_XML_H_ 23 | 24 | #include 25 | #include // [NOTE] nessetially include this header in some environments 26 | #include 27 | #include 28 | 29 | #include "mpu_util.h" 30 | 31 | class S3ObjList; 32 | 33 | typedef std::unique_ptr unique_ptr_xmlChar; 34 | typedef std::unique_ptr unique_ptr_xmlXPathObject; 35 | typedef std::unique_ptr unique_ptr_xmlXPathContext; 36 | typedef std::unique_ptr unique_ptr_xmlDoc; 37 | 38 | //------------------------------------------------------------------- 39 | // Functions 40 | //------------------------------------------------------------------- 41 | bool is_truncated(xmlDocPtr doc); 42 | int append_objects_from_xml_ex(const char* path, xmlDocPtr doc, xmlXPathContextPtr ctx, const char* ex_contents, const char* ex_key, const char* ex_etag, int isCPrefix, S3ObjList& head, bool prefix); 43 | int append_objects_from_xml(const char* path, xmlDocPtr doc, S3ObjList& head); 44 | unique_ptr_xmlChar get_next_continuation_token(xmlDocPtr doc); 45 | unique_ptr_xmlChar get_next_marker(xmlDocPtr doc); 46 | bool get_incomp_mpu_list(xmlDocPtr doc, incomp_mpu_list_t& list); 47 | 48 | bool simple_parse_xml(const char* data, size_t len, const char* key, std::string& value); 49 | 50 | bool init_parser_xml_lock(); 51 | bool destroy_parser_xml_lock(); 52 | 53 | #endif // S3FS_S3FS_XML_H_ 54 | 55 | /* 56 | * Local variables: 57 | * tab-width: 4 58 | * c-basic-offset: 4 59 | * End: 60 | * vim600: expandtab sw=4 ts=4 fdm=marker 61 | * vim<600: expandtab sw=4 ts=4 62 | */ 63 | -------------------------------------------------------------------------------- /s3fs/s3objlist.h: -------------------------------------------------------------------------------- 1 | /* 2 | * s3fs - FUSE-based file system backed by Amazon S3 3 | * 4 | * Copyright(C) 2007 Randy Rizun 5 | * 6 | * This program is free software; you can redistribute it and/or 7 | * modify it under the terms of the GNU General Public License 8 | * as published by the Free Software Foundation; either version 2 9 | * of the License, or (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License 17 | * along with this program; if not, write to the Free Software 18 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | */ 20 | 21 | #ifndef S3FS_S3OBJLIST_H_ 22 | #define S3FS_S3OBJLIST_H_ 23 | 24 | #include 25 | #include 26 | #include 27 | 28 | //------------------------------------------------------------------- 29 | // Structure / Typedef 30 | //------------------------------------------------------------------- 31 | struct s3obj_entry{ 32 | std::string normalname; // normalized name: if empty, object is normalized name. 33 | std::string orgname; // original name: if empty, object is original name. 34 | std::string etag; 35 | bool is_dir; 36 | 37 | s3obj_entry() : is_dir(false) {} 38 | }; 39 | 40 | typedef std::map s3obj_t; 41 | typedef std::vector s3obj_list_t; 42 | 43 | //------------------------------------------------------------------- 44 | // Class S3ObjList 45 | //------------------------------------------------------------------- 46 | class S3ObjList 47 | { 48 | private: 49 | s3obj_t objects; 50 | public: 51 | std::vector common_prefixes; 52 | 53 | private: 54 | bool insert_normalized(const char* name, const char* normalized, bool is_dir); 55 | const s3obj_entry* GetS3Obj(const char* name) const; 56 | 57 | s3obj_t::const_iterator begin() const { return objects.begin(); } 58 | s3obj_t::const_iterator end() const { return objects.end(); } 59 | 60 | public: 61 | S3ObjList() {} 62 | ~S3ObjList() {} 63 | 64 | bool IsEmpty() const { return objects.empty(); } 65 | bool insert(const char* name, const char* etag = nullptr, bool is_dir = false); 66 | std::string GetOrgName(const char* name) const; 67 | std::string GetNormalizedName(const char* name) const; 68 | std::string GetETag(const char* name) const; 69 | bool IsDir(const char* name) const; 70 | bool GetNameList(s3obj_list_t& list, bool OnlyNormalized = true, bool CutSlash = true) const; 71 | bool GetLastName(std::string& lastname) const; 72 | 73 | static bool MakeHierarchizedList(s3obj_list_t& list, bool haveSlash); 74 | }; 75 | 76 | #endif // S3FS_S3OBJLIST_H_ 77 | 78 | /* 79 | * Local variables: 80 | * tab-width: 4 81 | * c-basic-offset: 4 82 | * End: 83 | * vim600: expandtab sw=4 ts=4 fdm=marker 84 | * vim<600: expandtab sw=4 ts=4 85 | */ 86 | -------------------------------------------------------------------------------- /s3fs/sighandlers.h: -------------------------------------------------------------------------------- 1 | /* 2 | * s3fs - FUSE-based file system backed by Amazon S3 3 | * 4 | * Copyright(C) 2007 Randy Rizun 5 | * 6 | * This program is free software; you can redistribute it and/or 7 | * modify it under the terms of the GNU General Public License 8 | * as published by the Free Software Foundation; either version 2 9 | * of the License, or (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License 17 | * along with this program; if not, write to the Free Software 18 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | */ 20 | 21 | #ifndef S3FS_SIGHANDLERS_H_ 22 | #define S3FS_SIGHANDLERS_H_ 23 | 24 | #include 25 | 26 | class Semaphore; 27 | 28 | //---------------------------------------------- 29 | // class S3fsSignals 30 | //---------------------------------------------- 31 | class S3fsSignals 32 | { 33 | private: 34 | static std::unique_ptr pSingleton; 35 | static bool enableUsr1; 36 | 37 | std::unique_ptr pThreadUsr1; 38 | std::unique_ptr pSemUsr1; 39 | 40 | protected: 41 | static S3fsSignals* get() { return pSingleton.get(); } 42 | 43 | static void HandlerUSR1(int sig); 44 | static void* CheckCacheWorker(void* arg); 45 | 46 | static void HandlerUSR2(int sig); 47 | static bool InitUsr2Handler(); 48 | 49 | static void HandlerHUP(int sig); 50 | static bool InitHupHandler(); 51 | 52 | S3fsSignals(); 53 | S3fsSignals(const S3fsSignals&) = delete; 54 | S3fsSignals(S3fsSignals&&) = delete; 55 | S3fsSignals& operator=(const S3fsSignals&) = delete; 56 | S3fsSignals& operator=(S3fsSignals&&) = delete; 57 | 58 | bool InitUsr1Handler(); 59 | bool DestroyUsr1Handler(); 60 | bool WakeupUsr1Thread(); 61 | 62 | public: 63 | ~S3fsSignals(); 64 | static bool Initialize(); 65 | static bool Destroy(); 66 | 67 | static bool SetUsr1Handler(const char* path); 68 | }; 69 | 70 | #endif // S3FS_SIGHANDLERS_H_ 71 | 72 | /* 73 | * Local variables: 74 | * tab-width: 4 75 | * c-basic-offset: 4 76 | * End: 77 | * vim600: expandtab sw=4 ts=4 fdm=marker 78 | * vim<600: expandtab sw=4 ts=4 79 | */ 80 | -------------------------------------------------------------------------------- /test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/bin) 2 | 3 | add_executable(test_page_cache test_page_cache.cpp) 4 | target_link_libraries(test_page_cache PUBLIC hybridcache_local ${THIRD_PARTY_LIBRARIES}) 5 | 6 | add_executable(test_future test_future.cpp) 7 | target_link_libraries(test_future PUBLIC ${THIRD_PARTY_LIBRARIES}) 8 | 9 | add_executable(test_read_cache test_read_cache.cpp) 10 | target_link_libraries(test_read_cache PUBLIC hybridcache_local ${THIRD_PARTY_LIBRARIES}) 11 | 12 | add_executable(test_write_cache test_write_cache.cpp) 13 | target_link_libraries(test_write_cache PUBLIC hybridcache_local ${THIRD_PARTY_LIBRARIES}) 14 | 15 | add_executable(test_config test_config.cpp) 16 | target_link_libraries(test_config PUBLIC hybridcache_local ${THIRD_PARTY_LIBRARIES}) 17 | 18 | add_executable(test_global_read_cache test_global_read_cache.cpp) 19 | target_link_libraries(test_global_read_cache PUBLIC madfs_global) 20 | 21 | add_executable(test_global_read_cache_perf test_global_read_cache_perf.cpp) 22 | target_link_libraries(test_global_read_cache_perf PUBLIC madfs_global) 23 | 24 | add_executable(test_global_write_cache_perf test_global_write_cache_perf.cpp) 25 | target_link_libraries(test_global_write_cache_perf PUBLIC madfs_global) 26 | -------------------------------------------------------------------------------- /test/hybridcache.conf: -------------------------------------------------------------------------------- 1 | # ReadCache 2 | ReadCacheConfig.CacheConfig.CacheName=Read 3 | ReadCacheConfig.CacheConfig.MaxCacheSize=1073741824 4 | ReadCacheConfig.CacheConfig.PageBodySize=65536 5 | ReadCacheConfig.CacheConfig.PageMetaSize=1024 6 | ReadCacheConfig.CacheConfig.EnableCAS=1 7 | ReadCacheConfig.CacheConfig.SafeMode=1 8 | ReadCacheConfig.CacheConfig.CacheLibConfig.EnableNvmCache=0 9 | ReadCacheConfig.CacheConfig.CacheLibConfig.RaidPath= 10 | ReadCacheConfig.CacheConfig.CacheLibConfig.RaidFileNum= 11 | ReadCacheConfig.CacheConfig.CacheLibConfig.RaidFileSize= 12 | ReadCacheConfig.CacheConfig.CacheLibConfig.DataChecksum= 13 | ReadCacheConfig.DownloadNormalFlowLimit=1048576 14 | ReadCacheConfig.DownloadBurstFlowLimit=10485760 15 | 16 | # WriteCache 17 | WriteCacheConfig.CacheConfig.CacheName=Write 18 | WriteCacheConfig.CacheConfig.MaxCacheSize=104857600 19 | WriteCacheConfig.CacheConfig.PageBodySize=65536 20 | WriteCacheConfig.CacheConfig.PageMetaSize=1024 21 | WriteCacheConfig.CacheConfig.EnableCAS=1 22 | WriteCacheConfig.CacheConfig.SafeMode=1 23 | WriteCacheConfig.CacheSafeRatio=70 24 | WriteCacheConfig.EnableThrottle=0 25 | 26 | # GlobalCache 27 | UseGlobalCache=1 28 | GlobalCacheConfig.EnableWriteCache=1 29 | GlobalCacheConfig.EtcdAddress=http://192.168.1.87:2379 30 | GlobalCacheConfig.GlobalServers=optane07:8000,optane08:8000 31 | GlobalCacheConfig.GflagFile= 32 | 33 | ThreadNum=16 34 | BackFlushCacheRatio=40 35 | UploadNormalFlowLimit=1048576 36 | UploadBurstFlowLimit=10485760 37 | LogPath=. 38 | # LogLevel: GLOG_INFO=0, GLOG_WARNING=1, GLOG_ERROR=2, GLOG_FATAL=3 39 | LogLevel=1 40 | EnableLog=0 41 | FlushToRead=1 42 | CleanCacheByOpen=0 43 | EnableResize=0 44 | EnableLinUCB=0 45 | -------------------------------------------------------------------------------- /test/test_config.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "gtest/gtest.h" 4 | 5 | #include "config.h" 6 | 7 | using namespace std; 8 | using namespace HybridCache; 9 | 10 | TEST(ConfigRead, Read) { 11 | HybridCacheConfig cfg; 12 | EXPECT_EQ(true, GetHybridCacheConfig("../../test/hybridcache.conf", cfg)); 13 | EXPECT_EQ(1073741824, cfg.ReadCacheCfg.CacheCfg.MaxCacheSize); 14 | EXPECT_EQ(16, cfg.ThreadNum); 15 | 16 | EXPECT_EQ(true, cfg.UseGlobalCache); 17 | EXPECT_EQ("http://192.168.1.87:2379", cfg.GlobalCacheCfg.EtcdAddress); 18 | EXPECT_EQ(2, cfg.GlobalCacheCfg.GlobalServers.size()); 19 | EXPECT_EQ("optane08:8000", cfg.GlobalCacheCfg.GlobalServers[1]); 20 | } 21 | 22 | int main(int argc, char **argv) { 23 | printf("Running ConfigRead test from %s\n", __FILE__); 24 | testing::InitGoogleTest(&argc, argv); 25 | return RUN_ALL_TESTS(); 26 | } 27 | -------------------------------------------------------------------------------- /test/test_future.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "folly/futures/Future.h" 5 | #include "gtest/gtest.h" 6 | 7 | #include "common.h" 8 | 9 | using namespace folly; 10 | using namespace std; 11 | 12 | std::shared_ptr executor; 13 | 14 | folly::Future test(int i) { 15 | std::cout << i << " start" << endl; 16 | return folly::via(executor.get(), [i]() -> int { 17 | std::cout << i << " download ..." << endl; 18 | std::this_thread::sleep_for(2000ms); 19 | std::cout << i << " end" << endl; 20 | return 0; 21 | }); 22 | } 23 | 24 | folly::Future testCombine() { 25 | std::cout << "testCombine start" << endl; 26 | 27 | std::vector> fs; 28 | for (int i = 0; i < 3; i++) { 29 | fs.push_back(test(i)); 30 | } 31 | 32 | std::cout << "testCombine mid" << endl; 33 | 34 | auto f = collectAll(fs).via(executor.get()) 35 | .thenValue([](std::vector, std::allocator>>&& tups) { 36 | int res = 0; 37 | for (const auto& t : tups) { 38 | if (t.value() == 0) ++res; 39 | } 40 | std::cout << "testCombine end" << endl; 41 | return res; 42 | }); 43 | 44 | return f; 45 | } 46 | 47 | TEST(FollyFuture, combine) { 48 | auto f = testCombine(); 49 | std::cout << "testCombine running..." << endl; 50 | f.wait(); 51 | std::cout << "testCombine res:" << f.value() << endl; 52 | EXPECT_EQ(3, f.value()); 53 | } 54 | 55 | TEST(FollyFuture, chaining) { 56 | std::cout << "test chaining..." << endl; 57 | auto f = test(1); 58 | auto f2 = move(f).thenValue([](int i){ return i + 100; }); 59 | f2.wait(); 60 | std::cout << "chaining res:" << f2.value() << endl; 61 | } 62 | 63 | int main(int argc, char* argv[]) { 64 | executor = std::make_shared(16); 65 | 66 | printf("Running folly::future test from %s\n", __FILE__); 67 | testing::InitGoogleTest(&argc, argv); 68 | int res = RUN_ALL_TESTS(); 69 | 70 | executor->stop(); 71 | return res; 72 | } 73 | -------------------------------------------------------------------------------- /test/test_global_read_cache_perf.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include "S3DataAdaptor.h" 8 | #include "FileSystemDataAdaptor.h" 9 | #include "GlobalDataAdaptor.h" 10 | #include "ReadCacheClient.h" 11 | 12 | DEFINE_string(server, "0.0.0.0:8000", "IP Address of server"); 13 | DEFINE_int32(threads, 1, "Thread count in perf test"); 14 | DEFINE_int32(duration, 5, "Test duration in seconds"); 15 | DEFINE_int32(depth, 1, "IO depth"); 16 | DEFINE_bool(use_s3, false, "Use S3 storage"); 17 | DEFINE_string(filename, "sample.dat", "Test file name"); 18 | 19 | std::vector SplitString(const std::string &input) { 20 | std::vector result; 21 | std::stringstream ss(input); 22 | std::string item; 23 | while (std::getline(ss, item, ',')) { 24 | result.push_back(item); 25 | } 26 | return result; 27 | } 28 | 29 | TEST(global_cache_client, perf) 30 | { 31 | auto etcd_client = std::make_shared("http://127.0.0.1:2379"); 32 | 33 | std::shared_ptr base_adaptor = std::make_shared(); 34 | if (FLAGS_use_s3) { 35 | base_adaptor = std::make_shared(); 36 | } else { 37 | base_adaptor = std::make_shared(); 38 | } 39 | auto global_adaptor = std::make_shared(base_adaptor, SplitString(FLAGS_server), etcd_client); 40 | const size_t chunk_size = GetGlobalConfig().default_policy.read_chunk_size; 41 | 42 | struct stat st_buf; 43 | if (stat(FLAGS_filename.c_str(), &st_buf)) { 44 | PLOG(ERROR) << "Failed to stat file"; 45 | exit(EXIT_FAILURE); 46 | } 47 | auto chunk_count = std::min(1024, (int) (st_buf.st_size / chunk_size)); 48 | 49 | std::vector workers; 50 | std::atomic running(true); 51 | std::atomic operations_total(0); 52 | for (int i = 0; i < FLAGS_threads; ++i) { 53 | workers.emplace_back([&] { 54 | ByteBuffer buffer[FLAGS_depth]; 55 | for (int j = 0; j < FLAGS_depth; ++j) { 56 | buffer[j].data = new char[chunk_size]; 57 | buffer[j].len = chunk_size; 58 | } 59 | uint64_t operations = 0; 60 | std::vector > future_list; 61 | while(running) { 62 | future_list.clear(); 63 | for (int j = 0; j < FLAGS_depth; ++j) { 64 | future_list.emplace_back(global_adaptor->DownLoad(FLAGS_filename.c_str(), chunk_size * (lrand48() % chunk_count), chunk_size, buffer[j])); 65 | } 66 | folly::collectAll(future_list).wait(); 67 | operations += FLAGS_depth; 68 | } 69 | operations_total.fetch_add(operations); 70 | }); 71 | } 72 | sleep(FLAGS_duration); 73 | running = false; 74 | for (int i = 0; i < FLAGS_threads; ++i) { 75 | workers[i].join(); 76 | } 77 | LOG(INFO) << "operation per second: " << operations_total.load() / double(FLAGS_duration) 78 | << "data transfered (MB/s): " << chunk_size * operations_total.load() / double(FLAGS_duration) / 1024.0 / 1024.0; 79 | } 80 | 81 | int main(int argc, char **argv) 82 | { 83 | gflags::ParseCommandLineFlags(&argc, &argv, true); 84 | testing::InitGoogleTest(&argc, argv); 85 | return RUN_ALL_TESTS(); 86 | } 87 | --------------------------------------------------------------------------------