├── CMakeLists.txt ├── FindFUSE.cmake ├── FindThrift.cmake ├── LICENSE ├── README.md ├── VERSION ├── charybde_ops.cc ├── charybde_ops.h ├── charybdefs.c ├── cookbook ├── README.md ├── demo.py ├── recipes └── recipes.py ├── python_client.py ├── server.cc ├── server.hh ├── server.thrift └── tests ├── common.py ├── kill_in_flush_test.py ├── out_of_disk_space_test.py ├── python_client └── scylla_test_all_calls.py /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 2.8.11) 2 | project (CharybdeFS) 3 | 4 | set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}) 5 | 6 | set(CMAKE_CXX_FLAGS "-std=c++11 -D_FILE_OFFSET_BITS=64") 7 | set(CMAKE_C_FLAGS "-D_FILE_OFFSET_BITS=64") 8 | 9 | find_package(FUSE 2.9 REQUIRED) 10 | 11 | add_library(charybde_lib charybde_ops.cc server.cc) 12 | 13 | add_custom_target(thrift 14 | COMMAND thrift --gen cpp server.thrift) 15 | add_dependencies(charybde_lib thrift) 16 | 17 | add_library(server_lib charybde_ops.cc server.cc) 18 | 19 | add_library(server gen-cpp/server_constants.cpp gen-cpp/server.cpp gen-cpp/server_types.cpp) 20 | 21 | # find thrift headers and libs 22 | find_package(Thrift REQUIRED) 23 | include_directories(SYSTEM ${THRIFT_INCLUDE_DIR}) 24 | 25 | add_executable(charybdefs charybdefs.c) 26 | 27 | target_link_libraries(charybdefs charybde_lib) 28 | target_link_libraries(charybdefs server) 29 | target_link_libraries(charybdefs ${FUSE_LIBRARIES}) 30 | target_link_libraries(charybdefs pthread) 31 | target_link_libraries(charybdefs event) 32 | target_link_libraries(charybdefs ${THRIFT_LIBS}) 33 | -------------------------------------------------------------------------------- /FindFUSE.cmake: -------------------------------------------------------------------------------- 1 | # This module can find FUSE Library 2 | # 3 | # Requirements: 4 | # - CMake >= 2.8.3 5 | # 6 | # The following variables will be defined for your use: 7 | # - FUSE_FOUND : was FUSE found? 8 | # - FUSE_INCLUDE_DIRS : FUSE include directory 9 | # - FUSE_LIBRARIES : FUSE library 10 | # - FUSE_DEFINITIONS : FUSE cflags 11 | # - FUSE_VERSION : complete version of FUSE (major.minor) 12 | # - FUSE_MAJOR_VERSION : major version of FUSE 13 | # - FUSE_MINOR_VERSION : minor version of FUSE 14 | # 15 | # Example Usage: 16 | # 17 | # 1. Copy this file in the root of your project source directory 18 | # 2. Then, tell CMake to search this non-standard module in your project directory by adding to your CMakeLists.txt: 19 | # set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}) 20 | # 3. Finally call find_package() once, here are some examples to pick from 21 | # 22 | # Require FUSE 2.6 or later 23 | # find_package(FUSE 2.6 REQUIRED) 24 | # 25 | # if(FUSE_FOUND) 26 | # add_definitions(${FUSE_DEFINITIONS}) 27 | # include_directories(${FUSE_INCLUDE_DIRS}) 28 | # add_executable(myapp myapp.c) 29 | # target_link_libraries(myapp ${FUSE_LIBRARIES}) 30 | # endif() 31 | 32 | #============================================================================= 33 | # Copyright (c) 2012, julp 34 | # 35 | # Distributed under the OSI-approved BSD License 36 | # 37 | # This software is distributed WITHOUT ANY WARRANTY; without even the 38 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 39 | #============================================================================= 40 | 41 | cmake_minimum_required(VERSION 2.8.3) 42 | 43 | ########## Private ########## 44 | function(fusedebug _varname) 45 | if(FUSE_DEBUG) 46 | message("${_varname} = ${${_varname}}") 47 | endif(FUSE_DEBUG) 48 | endfunction(fusedebug) 49 | 50 | ########## Public ########## 51 | set(FUSE_FOUND TRUE) 52 | set(FUSE_LIBRARIES ) 53 | set(FUSE_DEFINITIONS ) 54 | set(FUSE_INCLUDE_DIRS ) 55 | 56 | find_package(PkgConfig) 57 | 58 | set(PC_FUSE_INCLUDE_DIRS ) 59 | set(PC_FUSE_LIBRARY_DIRS ) 60 | if(PKG_CONFIG_FOUND) 61 | pkg_check_modules(PC_FUSE "fuse" QUIET) 62 | if(PC_FUSE_FOUND) 63 | # fusedebug(PC_FUSE_LIBRARIES) 64 | # fusedebug(PC_FUSE_LIBRARY_DIRS) 65 | # fusedebug(PC_FUSE_LDFLAGS) 66 | # fusedebug(PC_FUSE_LDFLAGS_OTHER) 67 | # fusedebug(PC_FUSE_INCLUDE_DIRS) 68 | # fusedebug(PC_FUSE_CFLAGS) 69 | # fusedebug(PC_FUSE_CFLAGS_OTHER) 70 | set(FUSE_DEFINITIONS "${PC_FUSE_CFLAGS_OTHER}") 71 | endif(PC_FUSE_FOUND) 72 | endif(PKG_CONFIG_FOUND) 73 | 74 | find_path( 75 | FUSE_INCLUDE_DIRS 76 | NAMES fuse_common.h fuse_lowlevel.h fuse.h 77 | PATHS "${PC_FUSE_INCLUDE_DIRS}" 78 | DOC "Include directories for FUSE" 79 | ) 80 | 81 | if(NOT FUSE_INCLUDE_DIRS) 82 | set(FUSE_FOUND FALSE) 83 | endif(NOT FUSE_INCLUDE_DIRS) 84 | 85 | find_library( 86 | FUSE_LIBRARIES 87 | NAMES "fuse" 88 | PATHS "${PC_FUSE_LIBRARY_DIRS}" 89 | DOC "Libraries for FUSE" 90 | ) 91 | 92 | if(NOT FUSE_LIBRARIES) 93 | set(FUSE_FOUND FALSE) 94 | endif(NOT FUSE_LIBRARIES) 95 | 96 | if(FUSE_FOUND) 97 | if(EXISTS "${FUSE_INCLUDE_DIRS}/fuse_common.h") 98 | file(READ "${FUSE_INCLUDE_DIRS}/fuse_common.h" _contents) 99 | string(REGEX REPLACE ".*# *define *FUSE_MAJOR_VERSION *([0-9]+).*" "\\1" FUSE_MAJOR_VERSION "${_contents}") 100 | string(REGEX REPLACE ".*# *define *FUSE_MINOR_VERSION *([0-9]+).*" "\\1" FUSE_MINOR_VERSION "${_contents}") 101 | set(FUSE_VERSION "${FUSE_MAJOR_VERSION}.${FUSE_MINOR_VERSION}") 102 | endif() 103 | 104 | include(CheckCSourceCompiles) 105 | # Backup CMAKE_REQUIRED_* 106 | set(OLD_CMAKE_REQUIRED_INCLUDES "${CMAKE_REQUIRED_INCLUDES}") 107 | set(OLD_CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES}") 108 | set(OLD_CMAKE_REQUIRED_DEFINITIONS "${CMAKE_REQUIRED_DEFINITIONS}") 109 | # Add FUSE compilation flags 110 | set(CMAKE_REQUIRED_INCLUDES "${CMAKE_REQUIRED_INCLUDES}" "${FUSE_INCLUDE_DIRS}") 111 | set(CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES}" "${FUSE_LIBRARIES}") 112 | set(CMAKE_REQUIRED_DEFINITIONS "${CMAKE_REQUIRED_DEFINITIONS}" "${FUSE_DEFINITIONS}") 113 | check_c_source_compiles("#include 114 | #include 115 | #include 116 | #include 117 | #include 118 | #include 119 | 120 | int main(void) { 121 | return 0; 122 | }" FUSE_CFLAGS_CHECK) 123 | if(NOT FUSE_CFLAGS_CHECK) 124 | set(FUSE_DEFINITIONS "-D_FILE_OFFSET_BITS=64") 125 | # Should we run again previous test to assume the failure was due to missing definition -D_FILE_OFFSET_BITS=64? 126 | endif(NOT FUSE_CFLAGS_CHECK) 127 | # Restore CMAKE_REQUIRED_* 128 | set(CMAKE_REQUIRED_INCLUDES "${OLD_CMAKE_REQUIRED_INCLUDES}") 129 | set(CMAKE_REQUIRED_LIBRARIES "${OLD_CMAKE_REQUIRED_LIBRARIES}") 130 | set(CMAKE_REQUIRED_DEFINITIONS "${OLD_CMAKE_REQUIRED_DEFINITIONS}") 131 | endif(FUSE_FOUND) 132 | 133 | if(FUSE_INCLUDE_DIRS) 134 | include(FindPackageHandleStandardArgs) 135 | if(FUSE_FIND_REQUIRED AND NOT FUSE_FIND_QUIETLY) 136 | find_package_handle_standard_args(FUSE REQUIRED_VARS FUSE_LIBRARIES FUSE_INCLUDE_DIRS VERSION_VAR FUSE_VERSION) 137 | else() 138 | find_package_handle_standard_args(FUSE "FUSE not found" FUSE_LIBRARIES FUSE_INCLUDE_DIRS) 139 | endif() 140 | else(FUSE_INCLUDE_DIRS) 141 | if(FUSE_FIND_REQUIRED AND NOT FUSE_FIND_QUIETLY) 142 | message(FATAL_ERROR "Could not find FUSE include directory") 143 | endif() 144 | endif(FUSE_INCLUDE_DIRS) 145 | 146 | mark_as_advanced( 147 | FUSE_INCLUDE_DIRS 148 | FUSE_LIBRARIES 149 | ) 150 | 151 | # IN (args) 152 | fusedebug("FUSE_FIND_COMPONENTS") 153 | fusedebug("FUSE_FIND_REQUIRED") 154 | fusedebug("FUSE_FIND_QUIETLY") 155 | fusedebug("FUSE_FIND_VERSION") 156 | # OUT 157 | # Found 158 | fusedebug("FUSE_FOUND") 159 | # Definitions 160 | fusedebug("FUSE_DEFINITIONS") 161 | # Linking 162 | fusedebug("FUSE_INCLUDE_DIRS") 163 | fusedebug("FUSE_LIBRARIES") 164 | # Version 165 | fusedebug("FUSE_MAJOR_VERSION") 166 | fusedebug("FUSE_MINOR_VERSION") 167 | fusedebug("FUSE_VERSION") 168 | -------------------------------------------------------------------------------- /FindThrift.cmake: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Cloudera Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # - Find Thrift (a cross platform RPC lib/tool) 16 | # This module defines 17 | # THRIFT_VERSION, version string of ant if found 18 | # THRIFT_INCLUDE_DIR, where to find THRIFT headers 19 | # THRIFT_CONTRIB_DIR, where contrib thrift files (e.g. fb303.thrift) are installed 20 | # THRIFT_LIBS, THRIFT libraries 21 | # THRIFT_FOUND, If false, do not try to use ant 22 | 23 | # prefer the thrift version supplied in THRIFT_HOME 24 | message(STATUS "THRIFT_HOME: $ENV{THRIFT_HOME}") 25 | find_path(THRIFT_INCLUDE_DIR thrift/Thrift.h HINTS 26 | $ENV{THRIFT_HOME}/include/ 27 | /usr/local/include/ 28 | /opt/local/include/ 29 | ) 30 | 31 | find_path(THRIFT_CONTRIB_DIR share/fb303/if/fb303.thrift HINTS 32 | $ENV{THRIFT_HOME} 33 | /usr/local/ 34 | ) 35 | 36 | set(THRIFT_LIB_PATHS 37 | $ENV{THRIFT_HOME}/lib 38 | /usr/local/lib 39 | /opt/local/lib) 40 | 41 | find_path(THRIFT_STATIC_LIB_PATH libthrift.a PATHS ${THRIFT_LIB_PATHS}) 42 | 43 | # prefer the thrift version supplied in THRIFT_HOME 44 | find_library(THRIFT_LIB NAMES thrift HINTS ${THRIFT_LIB_PATHS}) 45 | find_library(THRIFT_NB_LIB NAMES thriftnb HINTS ${THRIFT_LIB_PATHS}) 46 | 47 | find_program(THRIFT_COMPILER thrift 48 | $ENV{THRIFT_HOME}/bin 49 | /usr/local/bin 50 | /usr/bin 51 | NO_DEFAULT_PATH 52 | ) 53 | 54 | if (THRIFT_LIB) 55 | set(THRIFT_FOUND TRUE) 56 | set(THRIFT_LIBS ${THRIFT_LIB} ${THRIFT_NB_LIB}) 57 | set(THRIFT_STATIC_LIB ${THRIFT_STATIC_LIB_PATH}/libthrift.a) 58 | exec_program(${THRIFT_COMPILER} 59 | ARGS -version OUTPUT_VARIABLE THRIFT_VERSION RETURN_VALUE THRIFT_RETURN) 60 | else () 61 | set(THRIFT_FOUND FALSE) 62 | endif () 63 | 64 | if (THRIFT_FOUND) 65 | if (NOT THRIFT_FIND_QUIETLY) 66 | message(STATUS "Thrift version: ${THRIFT_VERSION}") 67 | endif () 68 | else () 69 | message(STATUS "Thrift compiler/libraries NOT found. " 70 | "Thrift support will be disabled (${THRIFT_RETURN}, " 71 | "${THRIFT_INCLUDE_DIR}, ${THRIFT_LIB})") 72 | endif () 73 | 74 | 75 | mark_as_advanced( 76 | THRIFT_LIB 77 | THRIFT_COMPILER 78 | THRIFT_INCLUDE_DIR 79 | ) 80 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | 27-12-2015 3 | 4 | The author disclaims copyright to this source code. In place of 5 | a legal notice, here is a blessing: 6 | 7 | May you do good and not evil. 8 | May you find forgiveness for yourself and forgive others. 9 | May you share freely, never taking more than you give. 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | CharybdeFS 3 | ========== 4 | 5 | A fuse based fault injection filesystem 6 | with a Thrift RPC interface for instrumentation. 7 | 8 | Building 9 | ======== 10 | **CentOS 7.4** 11 | 12 | ```sh 13 | sudo yum install gcc-c++ cmake fuse fuse-devel thrift python-thrift thrift-devel 14 | thrift -r --gen cpp --gen py server.thrift 15 | cmake CMakeLists.txt 16 | make 17 | ``` 18 | **Ubuntu 16.04** 19 | 20 | ```sh 21 | sudo apt-get install software-properties-common 22 | sudo add-apt-repository ppa:scylladb/ppa 23 | sudo apt-get update 24 | sudo apt-get install g++ cmake fuse libfuse-dev python-thrift thrift-compiler scylla-libthrift010-dev libboost-dev 25 | thrift -r --gen cpp --gen py server.thrift 26 | cmake -DCMAKE_PREFIX_PATH=/opt/scylladb CMakeLists.txt 27 | make 28 | ``` 29 | 30 | Using 31 | ===== 32 | Load fuse module, if it is not loaded 33 | ```sh 34 | modprobe fuse 35 | ``` 36 | Create mount directory for Charybde. Tested application should use this directory. 37 | 38 | Note: Faults can be applied for files that are manipulated through this directory only! 39 | ```sh 40 | mkdir /mnt/charybdbde 41 | ``` 42 | `dest_dir` directory on the file system where actual files will be stored 43 | Running CharybdeFS 44 | ```sh 45 | ./charybdefs /mnt/charybdbde -omodules=subdir,subdir=/dest_dir 46 | ``` 47 | Running CharybdeFS and allowing other users to use the mount 48 | ```sh 49 | ./charybdefs /mnt/charybdbde -omodules=subdir,subdir=/dest_dir -oallow_other 50 | ``` 51 | 52 | Example tests 53 | ============= 54 | 55 | `tests/scylla_test_all_calls.py` is an example test running against the 56 | ScyllaDB database. It will insert data in the database, trigger some 57 | errors for a while, clear it and let the database recover. 58 | After this it will boot up scylla again and check that all commited 59 | queries are safe on disk. 60 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 0.1 2 | -------------------------------------------------------------------------------- /charybde_ops.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * ** 27-12-2015 3 | * ** 4 | * ** The author disclaims copyright to this source code. In place of 5 | * ** a legal notice, here is a blessing: 6 | * ** 7 | * ** May you do good and not evil. 8 | * ** May you find forgiveness for yourself and forgive others. 9 | * ** May you share freely, never taking more than you give. 10 | * ** 11 | */ 12 | 13 | #include "server.hh" 14 | 15 | #include 16 | 17 | extern "C" { 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | 32 | #include 33 | 34 | #include "charybde_ops.h" 35 | 36 | int charybde_getattr(const char *path, struct stat *buf) 37 | { 38 | static volatile int in_flight = 0; 39 | in_flight++; 40 | int ret = error_inject(in_flight, path, "getattr"); 41 | if (ret) { 42 | in_flight--; 43 | return ret; 44 | } 45 | 46 | ret = lstat(path, buf); 47 | if (ret < 0) { 48 | in_flight--; 49 | return -errno; 50 | } 51 | 52 | in_flight--; 53 | return 0; 54 | } 55 | 56 | int charybde_readlink(const char *path, char *buf, size_t bufsiz) 57 | { 58 | static volatile int in_flight = 0; 59 | in_flight++; 60 | int ret = error_inject(in_flight, path, "readlink"); 61 | if (ret) { 62 | in_flight--; 63 | return ret; 64 | } 65 | 66 | ret = readlink(path, buf, bufsiz); 67 | if (ret < 0) { 68 | in_flight--; 69 | return -errno; 70 | } 71 | buf[ret] = 0; // should be null terminated according to FUSE readlink definition. See #11 72 | in_flight--; 73 | return 0; 74 | } 75 | 76 | int charybde_mknod(const char *path, mode_t mode, dev_t dev) 77 | { 78 | static volatile int in_flight = 0; 79 | in_flight++; 80 | int ret = error_inject(in_flight, path, "mknod"); 81 | if (ret) { 82 | in_flight--; 83 | return ret; 84 | } 85 | 86 | ret = mknod(path, mode, dev); 87 | if (ret < 0) { 88 | in_flight--; 89 | return -errno; 90 | } 91 | 92 | in_flight--; 93 | return 0; 94 | } 95 | 96 | int charybde_mkdir(const char *path, mode_t mode) 97 | { 98 | static volatile int in_flight = 0; 99 | in_flight++; 100 | int ret = error_inject(in_flight, path, "mkdir"); 101 | if (ret) { 102 | in_flight--; 103 | return ret; 104 | } 105 | 106 | ret = mkdir(path, mode); 107 | if (ret < 0) { 108 | in_flight--; 109 | return -errno; 110 | } 111 | 112 | in_flight--; 113 | return 0; 114 | } 115 | 116 | int charybde_unlink(const char *path) 117 | { 118 | static volatile int in_flight = 0; 119 | in_flight++; 120 | int ret = error_inject(in_flight, path, "unlink"); 121 | if (ret) { 122 | in_flight--; 123 | return ret; 124 | } 125 | 126 | ret = unlink(path); 127 | if (ret < 0) { 128 | in_flight--; 129 | return -errno; 130 | } 131 | 132 | in_flight--; 133 | return 0; 134 | } 135 | 136 | int charybde_rmdir(const char *path) 137 | { 138 | static volatile int in_flight = 0; 139 | in_flight++; 140 | int ret = error_inject(in_flight, path, "rmdir"); 141 | if (ret) { 142 | in_flight--; 143 | return ret; 144 | } 145 | 146 | ret = rmdir(path); 147 | if (ret < 0) { 148 | in_flight--; 149 | return -errno; 150 | } 151 | 152 | in_flight--; 153 | return 0; 154 | } 155 | 156 | int charybde_symlink(const char *target, const char *linkpath) 157 | { 158 | static volatile int in_flight = 0; 159 | in_flight++; 160 | int ret = error_inject(in_flight, target, "symlink"); 161 | if (ret) { 162 | in_flight--; 163 | return ret; 164 | } 165 | 166 | ret = error_inject(in_flight, linkpath, "symlink"); 167 | if (ret) { 168 | in_flight--; 169 | return ret; 170 | } 171 | 172 | ret = symlink(target, linkpath); 173 | if (ret < 0) { 174 | in_flight--; 175 | return -errno; 176 | } 177 | 178 | in_flight--; 179 | return 0; 180 | } 181 | 182 | int charybde_rename(const char *oldpath, const char *newpath) 183 | { 184 | static volatile int in_flight = 0; 185 | in_flight++; 186 | int ret = error_inject(in_flight, oldpath, "rename"); 187 | if (ret) { 188 | in_flight--; 189 | return ret; 190 | } 191 | 192 | ret = error_inject(in_flight, newpath, "rename"); 193 | if (ret) { 194 | in_flight--; 195 | return ret; 196 | } 197 | 198 | ret = rename(oldpath, newpath); 199 | if (ret < 0) { 200 | in_flight--; 201 | return -errno; 202 | } 203 | 204 | in_flight--; 205 | return 0; 206 | } 207 | 208 | int charybde_link(const char *oldpath, const char *newpath) 209 | { 210 | static volatile int in_flight = 0; 211 | in_flight++; 212 | int ret = error_inject(in_flight, oldpath, "link"); 213 | if (ret) { 214 | in_flight--; 215 | return ret; 216 | } 217 | 218 | ret = error_inject(in_flight, newpath, "link"); 219 | if (ret) { 220 | in_flight--; 221 | return ret; 222 | } 223 | 224 | ret = link(oldpath, newpath); 225 | if (ret < 0) { 226 | in_flight--; 227 | return -errno; 228 | } 229 | 230 | in_flight--; 231 | return 0; 232 | } 233 | 234 | int charybde_chmod(const char *path, mode_t mode) 235 | { 236 | static volatile int in_flight = 0; 237 | in_flight++; 238 | int ret = error_inject(in_flight, path, "chmod"); 239 | if (ret) { 240 | in_flight--; 241 | return ret; 242 | } 243 | 244 | ret = chmod(path, mode); 245 | if (ret < 0) { 246 | in_flight--; 247 | return -errno; 248 | } 249 | 250 | in_flight--; 251 | return 0; 252 | } 253 | 254 | int charybde_chown(const char *path, uid_t owner, gid_t group) 255 | { 256 | static volatile int in_flight = 0; 257 | in_flight++; 258 | int ret = error_inject(in_flight, path, "chown"); 259 | if (ret) { 260 | in_flight--; 261 | return ret; 262 | } 263 | 264 | ret = chown(path, owner, group); 265 | if (ret < 0) { 266 | in_flight--; 267 | return -errno; 268 | } 269 | 270 | in_flight--; 271 | return 0; 272 | } 273 | 274 | int charybde_truncate(const char *path, off_t length) 275 | { 276 | static volatile int in_flight = 0; 277 | in_flight++; 278 | int ret = error_inject(in_flight, path, "truncate"); 279 | if (ret) { 280 | in_flight--; 281 | return ret; 282 | } 283 | 284 | ret = truncate(path, length); 285 | if (ret < 0) { 286 | in_flight--; 287 | return -errno; 288 | } 289 | 290 | in_flight--; 291 | return 0; 292 | } 293 | 294 | int charybde_open(const char *path, struct fuse_file_info *fi) 295 | { 296 | static volatile int in_flight = 0; 297 | in_flight++; 298 | int ret = error_inject(in_flight, path, "open"); 299 | if (ret) { 300 | in_flight--; 301 | return ret; 302 | } 303 | 304 | ret = open(path, fi->flags); 305 | if (ret < 0) { 306 | in_flight--; 307 | return -errno; 308 | } 309 | 310 | fi->fh = ret; 311 | in_flight--; 312 | return 0; 313 | } 314 | 315 | int charybde_read(const char *path, char *buf, size_t size, off_t offset, 316 | struct fuse_file_info *fi) 317 | { 318 | static volatile int in_flight = 0; 319 | in_flight++; 320 | int ret = error_inject(in_flight, path, "read"); 321 | if (ret) { 322 | in_flight--; 323 | return ret; 324 | } 325 | 326 | ret = pread(fi->fh, buf, size, offset); 327 | if (ret == -1) { 328 | ret = -errno; 329 | } 330 | 331 | in_flight--; 332 | return ret; 333 | } 334 | 335 | int charybde_write(const char *path, const char *buf, size_t size, off_t offset, 336 | struct fuse_file_info *fi) 337 | { 338 | static volatile int in_flight = 0; 339 | in_flight++; 340 | int ret = error_inject(in_flight, path, "write"); 341 | if (ret) { 342 | in_flight--; 343 | return ret; 344 | } 345 | 346 | std::cout << "WRITE" << std::endl; 347 | 348 | ret = pwrite(fi->fh, buf, size, offset); 349 | if (ret == -1) { 350 | ret = -errno; 351 | } 352 | 353 | in_flight--; 354 | return ret; 355 | } 356 | 357 | int charybde_statfs(const char *path, struct statvfs *buf) 358 | { 359 | static volatile int in_flight = 0; 360 | in_flight++; 361 | int ret = error_inject(in_flight, path, "statfs"); 362 | if (ret) { 363 | in_flight--; 364 | return ret; 365 | } 366 | 367 | ret = statvfs(path, buf); 368 | if (ret < 0) { 369 | in_flight--; 370 | return -errno; 371 | } 372 | 373 | in_flight--; 374 | return 0; 375 | } 376 | 377 | int charybde_flush(const char *path, struct fuse_file_info *fi) 378 | { 379 | static volatile int in_flight = 0; 380 | in_flight++; 381 | int ret = error_inject(in_flight, path, "flush"); 382 | if (ret) { 383 | in_flight--; 384 | return ret; 385 | } 386 | 387 | /* Took from fuse examples */ 388 | ret = close(dup(fi->fh)); 389 | if (ret < 0) { 390 | in_flight--; 391 | return -errno; 392 | } 393 | 394 | in_flight--; 395 | return 0; 396 | } 397 | 398 | int charybde_release(const char *path, struct fuse_file_info *fi) 399 | { 400 | static volatile int in_flight = 0; 401 | in_flight++; 402 | int ret = error_inject(in_flight, path, "release"); 403 | if (ret) { 404 | in_flight--; 405 | return ret; 406 | } 407 | 408 | close(fi->fh); 409 | 410 | in_flight--; 411 | return 0; 412 | } 413 | 414 | int charybde_fsync(const char *path, int datasync, struct fuse_file_info *fi) 415 | { 416 | static volatile int in_flight = 0; 417 | in_flight++; 418 | int ret = error_inject(in_flight, path, "fsync"); 419 | if (ret) { 420 | in_flight--; 421 | return ret; 422 | } 423 | 424 | if (datasync) { 425 | ret = fdatasync(fi->fh); 426 | if (ret < 0) { 427 | in_flight--; 428 | return -errno; 429 | } 430 | } else { 431 | ret = fsync(fi->fh); 432 | if (ret < 0) { 433 | in_flight--; 434 | return -errno; 435 | } 436 | } 437 | 438 | in_flight--; 439 | return 0; 440 | } 441 | 442 | int charybde_setxattr(const char *path, const char *name, 443 | const char *value, size_t size, int flags) 444 | { 445 | static volatile int in_flight = 0; 446 | in_flight++; 447 | int ret = error_inject(in_flight, path, "setxattr"); 448 | if (ret) { 449 | in_flight--; 450 | return ret; 451 | } 452 | 453 | ret = setxattr(path, name, value, size, flags); 454 | if (ret < 0) { 455 | in_flight--; 456 | return -errno; 457 | } 458 | 459 | in_flight--; 460 | return 0; 461 | } 462 | 463 | int charybde_getxattr(const char *path, const char *name, 464 | char *value, size_t size) 465 | { 466 | static volatile int in_flight = 0; 467 | in_flight++; 468 | int ret = error_inject(in_flight, path, "getxattr"); 469 | if (ret) { 470 | in_flight--; 471 | return ret; 472 | } 473 | 474 | ret = getxattr(path, name, value, size); 475 | if (ret < 0) { 476 | in_flight--; 477 | return -errno; 478 | } 479 | 480 | in_flight--; 481 | return ret; 482 | } 483 | 484 | int charybde_listxattr(const char *path, char *list, 485 | size_t size) 486 | { 487 | static volatile int in_flight = 0; 488 | in_flight++; 489 | int ret = error_inject(in_flight, path, "listxattr"); 490 | if (ret) { 491 | in_flight--; 492 | return ret; 493 | } 494 | 495 | ret = listxattr(path, list, size); 496 | if (ret < 0) { 497 | in_flight--; 498 | return -errno; 499 | } 500 | 501 | in_flight--; 502 | return ret; 503 | } 504 | 505 | int charybde_removexattr(const char *path, const char *name) 506 | { 507 | static volatile int in_flight = 0; 508 | in_flight++; 509 | int ret = error_inject(in_flight, path, "removexattr"); 510 | if (ret) { 511 | in_flight--; 512 | return ret; 513 | } 514 | 515 | 516 | ret = removexattr(path, name); 517 | if (ret < 0) { 518 | in_flight--; 519 | return -errno; 520 | } 521 | 522 | in_flight--; 523 | return 0; 524 | } 525 | 526 | int charybde_opendir(const char *path, struct fuse_file_info *fi) 527 | { 528 | static volatile int in_flight = 0; 529 | in_flight++; 530 | int ret = error_inject(in_flight, path, "opendir"); 531 | if (ret) { 532 | in_flight--; 533 | return ret; 534 | } 535 | 536 | auto dir = opendir(path); 537 | 538 | if (!dir) { 539 | in_flight--; 540 | return -errno; 541 | } 542 | 543 | fi->fh = (int64_t) dir; 544 | in_flight--; 545 | return 0; 546 | } 547 | 548 | int charybde_readdir(const char *path, void *buf, fuse_fill_dir_t filler, 549 | off_t offset, struct fuse_file_info *fi) 550 | { 551 | static volatile int in_flight = 0; 552 | in_flight++; 553 | int ret = error_inject(in_flight, path, "readdir"); 554 | if (ret) { 555 | in_flight--; 556 | return ret; 557 | } 558 | 559 | DIR *dp = (DIR *) fi->fh; 560 | struct dirent *de; 561 | 562 | while ((de = readdir(dp)) != NULL) { 563 | struct stat st; 564 | memset(&st, 0, sizeof(st)); 565 | st.st_ino = de->d_ino; 566 | st.st_mode = de->d_type << 12; 567 | if (filler(buf, de->d_name, &st, 0)) 568 | break; 569 | } 570 | 571 | in_flight--; 572 | return 0; 573 | } 574 | 575 | 576 | int charybde_releasedir(const char *path, struct fuse_file_info *fi) 577 | { 578 | static volatile int in_flight = 0; 579 | in_flight++; 580 | int ret = error_inject(in_flight, path, "releasedir"); 581 | if (ret) { 582 | in_flight--; 583 | return ret; 584 | } 585 | 586 | DIR *dir = (DIR *) fi->fh; 587 | 588 | ret = closedir(dir); 589 | if (ret < 0) { 590 | in_flight--; 591 | return -errno; 592 | } 593 | 594 | in_flight--; 595 | return 0; 596 | } 597 | 598 | int charybde_fsyncdir(const char *path, int datasync, struct fuse_file_info *fi) 599 | { 600 | static volatile int in_flight = 0; 601 | in_flight++; 602 | int ret = error_inject(in_flight, path, "fsyncdir"); 603 | if (ret) { 604 | in_flight--; 605 | return ret; 606 | } 607 | 608 | auto dir = opendir(path); 609 | if (!dir) { 610 | in_flight--; 611 | return -errno; 612 | } 613 | 614 | if (datasync) { 615 | ret = fdatasync(dirfd(dir)); 616 | if (ret < 0) { 617 | in_flight--; 618 | return -errno; 619 | } 620 | } else { 621 | ret = fsync(dirfd(dir)); 622 | if (ret < 0) { 623 | in_flight--; 624 | return -errno; 625 | } 626 | } 627 | 628 | closedir(dir); 629 | 630 | in_flight--; 631 | return 0; 632 | } 633 | 634 | void *charybde_init(struct fuse_conn_info *conn) 635 | { 636 | start_server_thread(); 637 | return NULL; 638 | } 639 | 640 | void charybde_destroy(void *) 641 | { 642 | 643 | } 644 | 645 | int charybde_access(const char *path, int mode) 646 | { 647 | static volatile int in_flight = 0; 648 | in_flight++; 649 | int ret = error_inject(in_flight, path, "access"); 650 | if (ret) { 651 | in_flight--; 652 | return ret; 653 | } 654 | 655 | ret = access(path, mode); 656 | if (ret < 0) { 657 | in_flight--; 658 | return -errno; 659 | } 660 | 661 | in_flight--; 662 | return 0; 663 | } 664 | 665 | int charybde_create(const char *path, mode_t mode, 666 | struct fuse_file_info *fi) 667 | { 668 | static volatile int in_flight = 0; 669 | in_flight++; 670 | int ret = error_inject(in_flight, path, "create"); 671 | if (ret) { 672 | in_flight--; 673 | return ret; 674 | } 675 | 676 | ret = creat(path, mode); 677 | if (ret < 0) { 678 | in_flight--; 679 | return -errno; 680 | } 681 | 682 | fi->fh = ret; 683 | 684 | in_flight--; 685 | return 0; 686 | } 687 | 688 | int charybde_ftruncate(const char *path, off_t length, 689 | struct fuse_file_info *fi) 690 | { 691 | static volatile int in_flight = 0; 692 | in_flight++; 693 | int ret = error_inject(in_flight, path, "ftruncate"); 694 | if (ret) { 695 | in_flight--; 696 | return ret; 697 | } 698 | 699 | ret = truncate(path, length); 700 | if (ret < 0) { 701 | in_flight--; 702 | return -errno; 703 | } 704 | 705 | in_flight--; 706 | return 0; 707 | } 708 | 709 | int charybde_fgetattr(const char *path, struct stat *buf, struct fuse_file_info *fi) 710 | { 711 | static volatile int in_flight = 0; 712 | in_flight++; 713 | int ret = error_inject(in_flight, path, "fgetattr"); 714 | if (ret) { 715 | in_flight--; 716 | return ret; 717 | } 718 | 719 | ret = fstat((int) fi->fh, buf); 720 | if (ret < 0) { 721 | in_flight--; 722 | return -errno; 723 | } 724 | 725 | in_flight--; 726 | return 0; 727 | } 728 | 729 | int charybde_lock(const char *path, struct fuse_file_info *fi, int cmd, 730 | struct flock *fl) 731 | { 732 | static volatile int in_flight = 0; 733 | in_flight++; 734 | int ret = error_inject(in_flight, path, "lock"); 735 | if (ret) { 736 | in_flight--; 737 | return ret; 738 | } 739 | 740 | ret = fcntl((int) fi->fh, cmd, fl); 741 | if (ret < 0) { 742 | in_flight--; 743 | return -errno; 744 | } 745 | 746 | in_flight--; 747 | return 0; 748 | } 749 | 750 | int charybde_utimens(const char *path, const struct timespec tv[2]) 751 | { 752 | static volatile int in_flight = 0; 753 | in_flight++; 754 | int ret = error_inject(in_flight, path, "bmap"); 755 | if (ret) { 756 | in_flight--; 757 | return ret; 758 | } 759 | 760 | std::cout << "charybde_utimens: unimplemented." << std::endl; 761 | 762 | in_flight--; 763 | return 0; 764 | } 765 | 766 | int charybde_bmap(const char *path, size_t blocksize, uint64_t *idx) 767 | { 768 | static volatile int in_flight = 0; 769 | in_flight++; 770 | int ret = error_inject(in_flight, path, "bmap"); 771 | if (ret) { 772 | in_flight--; 773 | return ret; 774 | } 775 | 776 | std::cout << "charybde_bmap: unimplemented." << std::endl; 777 | 778 | in_flight--; 779 | return 0; 780 | } 781 | 782 | int charybde_ioctl(const char *path, int cmd, void *arg, 783 | struct fuse_file_info *fi, 784 | unsigned int flags, void *data) 785 | { 786 | static volatile int in_flight = 0; 787 | in_flight++; 788 | int ret = error_inject(in_flight, path, "ioctl"); 789 | if (ret) { 790 | in_flight--; 791 | return ret; 792 | } 793 | 794 | ret = ioctl(fi->fh, cmd, arg); 795 | if (ret < 0) { 796 | in_flight--; 797 | return -errno; 798 | } 799 | 800 | in_flight--; 801 | return 0; 802 | } 803 | 804 | int charybde_poll(const char *path, struct fuse_file_info *fi, 805 | struct fuse_pollhandle *ph, unsigned *reventsp) 806 | { 807 | static volatile int in_flight = 0; 808 | in_flight++; 809 | int ret = error_inject(in_flight, path, "poll"); 810 | if (ret) { 811 | in_flight--; 812 | return ret; 813 | } 814 | 815 | std::cout << "charybde_poll: unimplemented." << std::endl; 816 | 817 | in_flight--; 818 | return 0; 819 | } 820 | 821 | int charybde_write_buf(const char *path, struct fuse_bufvec *buf, off_t off, 822 | struct fuse_file_info *fi) 823 | { 824 | static volatile int in_flight = 0; 825 | in_flight++; 826 | int ret = error_inject(in_flight, path, "write_buf"); 827 | if (ret) { 828 | in_flight--; 829 | return ret; 830 | } 831 | 832 | /* Took from fuse examples */ 833 | struct fuse_bufvec dst = FUSE_BUFVEC_INIT(fuse_buf_size(buf)); 834 | 835 | dst.buf[0].flags = (fuse_buf_flags) (FUSE_BUF_IS_FD | FUSE_BUF_FD_SEEK); 836 | dst.buf[0].fd = fi->fh; 837 | dst.buf[0].pos = off; 838 | 839 | ret = fuse_buf_copy(&dst, buf, FUSE_BUF_SPLICE_NONBLOCK); 840 | 841 | in_flight--; 842 | return ret; 843 | } 844 | 845 | int charybde_read_buf(const char *path, struct fuse_bufvec **bufp, 846 | size_t size, off_t off, 847 | struct fuse_file_info *fi) 848 | { 849 | static volatile int in_flight = 0; 850 | in_flight++; 851 | int ret = error_inject(in_flight, path, "read_buf"); 852 | if (ret) { 853 | in_flight--; 854 | return ret; 855 | } 856 | 857 | /* Took from fuse examples */ 858 | struct fuse_bufvec *src; 859 | 860 | src = (fuse_bufvec *) malloc(sizeof(struct fuse_bufvec)); 861 | if (!src) { 862 | in_flight--; 863 | return -ENOMEM; 864 | } 865 | 866 | *src = FUSE_BUFVEC_INIT(size); 867 | 868 | src->buf[0].flags = (fuse_buf_flags) (FUSE_BUF_IS_FD | FUSE_BUF_FD_SEEK); 869 | src->buf[0].fd = fi->fh; 870 | src->buf[0].pos = off; 871 | 872 | *bufp = src; 873 | 874 | in_flight--; 875 | return 0; 876 | } 877 | 878 | int charybde_flock(const char *path, struct fuse_file_info *fi, int op) 879 | { 880 | static volatile int in_flight = 0; 881 | in_flight++; 882 | int ret = error_inject(in_flight, path, "flock"); 883 | if (ret) { 884 | in_flight--; 885 | return ret; 886 | } 887 | 888 | ret = flock(((int) fi->fh), op); 889 | if (ret < 0) { 890 | in_flight--; 891 | return -errno; 892 | } 893 | 894 | in_flight--; 895 | return 0; 896 | } 897 | 898 | int charybde_fallocate(const char *path, int mode, 899 | off_t offset, off_t len, 900 | struct fuse_file_info *fi) 901 | { 902 | static volatile int in_flight = 0; 903 | in_flight++; 904 | int ret = error_inject(in_flight, path, "fallocate"); 905 | if (ret) { 906 | in_flight--; 907 | return ret; 908 | } 909 | 910 | ret = fallocate((int) fi->fh, mode, offset, len); 911 | if (ret < 0) { 912 | in_flight--; 913 | return -errno; 914 | } 915 | 916 | in_flight--; 917 | return 0; 918 | } 919 | 920 | }; 921 | -------------------------------------------------------------------------------- /charybde_ops.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ** 27-12-2015 3 | * ** 4 | * ** The author disclaims copyright to this source code. In place of 5 | * ** a legal notice, here is a blessing: 6 | * ** 7 | * ** May you do good and not evil. 8 | * ** May you find forgiveness for yourself and forgive others. 9 | * ** May you share freely, never taking more than you give. 10 | * ** 11 | */ 12 | 13 | #ifndef CHARYBDE_OPS_HH 14 | #define CHARYBDE_OPS_HH 15 | 16 | #include 17 | #include 18 | 19 | #include 20 | 21 | int charybde_getattr(const char *, struct stat *); 22 | int charybde_readlink(const char *, char *, size_t); 23 | int charybde_mknod(const char *, mode_t, dev_t); 24 | int charybde_mkdir(const char *, mode_t); 25 | int charybde_unlink(const char *); 26 | int charybde_rmdir(const char *); 27 | int charybde_symlink(const char *, const char *); 28 | int charybde_rename(const char *, const char *); 29 | int charybde_link(const char *, const char *); 30 | int charybde_chmod(const char *, mode_t); 31 | int charybde_chown(const char *, uid_t, gid_t); 32 | int charybde_truncate(const char *, off_t); 33 | int charybde_open(const char *, struct fuse_file_info *); 34 | int charybde_read(const char *, char *, size_t, off_t, 35 | struct fuse_file_info *); 36 | int charybde_write(const char *, const char *, size_t, off_t, 37 | struct fuse_file_info *); 38 | int charybde_statfs(const char *, struct statvfs *); 39 | int charybde_flush(const char *, struct fuse_file_info *); 40 | int charybde_release(const char *, struct fuse_file_info *); 41 | int charybde_fsync(const char *, int, struct fuse_file_info *); 42 | int charybde_setxattr(const char *, const char *, const char *, size_t, int); 43 | int charybde_getxattr(const char *, const char *, char *, size_t); 44 | int charybde_listxattr(const char *, char *, size_t); 45 | int charybde_removexattr(const char *, const char *); 46 | int charybde_opendir(const char *, struct fuse_file_info *); 47 | int charybde_readdir(const char *path, void *buf, fuse_fill_dir_t filler, 48 | off_t offset, struct fuse_file_info *fi); 49 | int charybde_releasedir(const char *, struct fuse_file_info *); 50 | int charybde_fsyncdir(const char *, int, struct fuse_file_info *); 51 | 52 | void *charybde_init(struct fuse_conn_info *conn); 53 | void charybde_destroy(void *); 54 | 55 | int charybde_access(const char *, int); 56 | int charybde_create(const char *, mode_t, struct fuse_file_info *); 57 | int charybde_ftruncate(const char *, off_t, struct fuse_file_info *); 58 | int charybde_fgetattr(const char *, struct stat *, struct fuse_file_info *); 59 | int charybde_lock(const char *, struct fuse_file_info *, int cmd, 60 | struct flock *); 61 | int charybde_utimens(const char *, const struct timespec tv[2]); 62 | int charybde_bmap(const char *, size_t blocksize, uint64_t *idx); 63 | int charybde_ioctl(const char *, int cmd, void *arg, 64 | struct fuse_file_info *, unsigned int flags, void *data); 65 | int charybde_poll(const char *path, struct fuse_file_info *fi, 66 | struct fuse_pollhandle *ph, unsigned *reventsp); 67 | int charybde_write_buf(const char *, struct fuse_bufvec *buf, off_t off, 68 | struct fuse_file_info *); 69 | int charybde_read_buf(const char *, struct fuse_bufvec **bufp, 70 | size_t size, off_t off, struct fuse_file_info *); 71 | int charybde_flock(const char *, struct fuse_file_info *, int op); 72 | int charybde_fallocate(const char *, int, off_t, off_t, 73 | struct fuse_file_info *); 74 | 75 | #endif 76 | -------------------------------------------------------------------------------- /charybdefs.c: -------------------------------------------------------------------------------- 1 | /* 2 | * ** 27-12-2015 3 | * ** 4 | * ** The author disclaims copyright to this source code. In place of 5 | * ** a legal notice, here is a blessing: 6 | * ** 7 | * ** May you do good and not evil. 8 | * ** May you find forgiveness for yourself and forgive others. 9 | * ** May you share freely, never taking more than you give. 10 | * ** 11 | */ 12 | 13 | #define FUSE_USE_VERSION 29 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include "charybde_ops.h" 21 | 22 | #include "server.hh" 23 | 24 | static struct fuse_operations charybde_oper = { 25 | .getattr = charybde_getattr, 26 | .readlink = charybde_readlink, 27 | .mknod = charybde_mknod, 28 | .mkdir = charybde_mkdir, 29 | .unlink = charybde_unlink, 30 | .rmdir = charybde_rmdir, 31 | .symlink = charybde_symlink, 32 | .rename = charybde_rename, 33 | .link = charybde_link, 34 | .chmod = charybde_chmod, 35 | .chown = charybde_chown, 36 | .truncate = charybde_truncate, 37 | .open = charybde_open, 38 | .read = charybde_read, 39 | .write = charybde_write, 40 | .statfs = charybde_statfs, 41 | .flush = charybde_flush, 42 | .release = charybde_release, 43 | .fsync = charybde_fsync, 44 | .setxattr = charybde_setxattr, 45 | .getxattr = charybde_getxattr, 46 | .listxattr = charybde_listxattr, 47 | .removexattr = charybde_removexattr, 48 | .opendir = charybde_opendir, 49 | .readdir = charybde_readdir, 50 | .releasedir = charybde_releasedir, 51 | .fsyncdir = charybde_fsyncdir, 52 | 53 | .init = charybde_init, 54 | .destroy = charybde_destroy, 55 | 56 | .access = charybde_access, 57 | .create = charybde_create, 58 | .ftruncate = charybde_ftruncate, 59 | .fgetattr = charybde_fgetattr, 60 | .lock = charybde_lock, 61 | .utimens = charybde_utimens, 62 | .bmap = charybde_bmap, 63 | .ioctl = charybde_ioctl, 64 | .poll = charybde_poll, 65 | .write_buf = charybde_write_buf, 66 | .read_buf = charybde_read_buf, 67 | .flock = charybde_flock, 68 | .fallocate = charybde_fallocate, 69 | }; 70 | 71 | int main(int argc, char *argv[]) 72 | { 73 | printf("starting fuse filesystem\n"); 74 | return fuse_main(argc, argv, &charybde_oper, NULL); 75 | } 76 | -------------------------------------------------------------------------------- /cookbook/README.md: -------------------------------------------------------------------------------- 1 | Usage: 2 | ====== 3 | 4 | python demo.py 5 | -------------------------------------------------------------------------------- /cookbook/demo.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import shutil 4 | import subprocess 5 | import time 6 | import unittest 7 | 8 | 9 | class CookbookUnitTests(unittest.TestCase): 10 | 11 | def build_charybdefs(self): 12 | if not os.path.exists("../charybdefs"): 13 | self._log.warning("Building CharybdeFS") 14 | os.chdir("..") 15 | subprocess.call(["cmake", "CMakeLists.txt"]) 16 | subprocess.call(["make"]) 17 | os.chdir("cookbook") 18 | 19 | def setUp(self): 20 | logging.basicConfig() 21 | self._log = logging.getLogger("demo") 22 | 23 | self._log.warning("Setting up demo") 24 | 25 | self.build_charybdefs() 26 | 27 | self._log.warning("Setting up mount points and target directory") 28 | 29 | # create the CharybdeFS target directory 30 | self._data_dir = os.path.expanduser("~/.CharybdeFSCookbookDir") 31 | os.mkdir(self._data_dir) 32 | # Create the CharybdeFS mount point 33 | self._mount_point = os.path.expanduser("~/.CharybdeFSCookbookDirData") 34 | os.mkdir(self._mount_point) 35 | 36 | self._log.warning("Starting CharybdeFS") 37 | # Killing previous CharybdeFS instances 38 | subprocess.call(["pkill", "-9", "charybdefs"]) 39 | # Spawning charybdeFS 40 | subprocess.call(["../charybdefs", self._mount_point, "-omodules=subdir,subdir=%s" % self._data_dir]) 41 | 42 | def victim(self): 43 | return os.path.join(self._mount_point, "blub") 44 | 45 | @staticmethod 46 | def run_recipe(name): 47 | subprocess.call(["./recipes", "--clear"]) 48 | subprocess.call(["./recipes", "--%s" % name]) 49 | time.sleep(0.5) 50 | 51 | def tearDown(self): 52 | self._log.warning("Tearing down demo and cleaning up everything") 53 | subprocess.call(["umount", self._mount_point]) 54 | subprocess.call(["pkill", "-9", "charybdefs"]) 55 | shutil.rmtree(self._mount_point) 56 | shutil.rmtree(self._data_dir) 57 | self._log.warning("\n") 58 | 59 | def test_disk_full(self): 60 | self.run_recipe("full") 61 | subprocess.call(["dd", "if=/dev/zero", 62 | "of=%s" % self.victim()]) 63 | 64 | def test_io_error(self): 65 | self.run_recipe("io-error") 66 | subprocess.call(["dd", "if=/dev/zero", 67 | "of=%s" % self.victim()]) 68 | 69 | def test_quota(self): 70 | self.run_recipe("quota") 71 | subprocess.call(["dd", "if=/dev/zero", 72 | "of=%s" % self.victim()]) 73 | 74 | def test_delay(self): 75 | self.run_recipe("delay") 76 | subprocess.call(["dd", "if=/dev/zero", 77 | "of=%s" % self.victim(), 78 | "bs=4k", "count=100"]) 79 | 80 | def test_random(self): 81 | self.run_recipe("random") 82 | subprocess.call(["dd", "if=/dev/zero", 83 | "of=%s" % self.victim(), 84 | "bs=4k", "count=1000"]) 85 | 86 | def test_specific_syscalls(self): 87 | self.run_recipe("specific-syscalls") 88 | 89 | self._log.warning("Doing ls") 90 | subprocess.call(["ls", self._mount_point]) 91 | 92 | self._log.warning("Doing write") 93 | subprocess.call(["dd", "if=/dev/zero", 94 | "of=%s" % self.victim(), 95 | "bs=4k", "count=1000"]) 96 | 97 | self._log.warning("Doing read") 98 | subprocess.call(["dd", "if=%s" % self.victim(), 99 | "of=/dev/null", "bs=4k", "count=1000"]) 100 | 101 | def test_probability(self): 102 | self.run_recipe("probability") 103 | 104 | subprocess.call(["dd", "if=/dev/zero", 105 | "of=%s" % self.victim(), 106 | "bs=4k", "count=1000"]) 107 | 108 | def test_file_pattern(self): 109 | self.run_recipe("file-pattern") 110 | 111 | self._log.warning("Writing postfix main.cf: good") 112 | subprocess.call(["dd", "if=/dev/zero", 113 | "of=%s" % os.path.join(self._mount_point, "main.cf"), 114 | "bs=4k", "count=1000"]) 115 | 116 | self._log.warning("Writing sendmail sendmail.cf: baaaad !") 117 | subprocess.call(["dd", "if=/dev/zero", 118 | "of=%s" % os.path.join(self._mount_point, "sendmail.cf"), 119 | "bs=4k", "count=1000"]) 120 | 121 | def test_broken_drive(self): 122 | self.run_recipe("broken-drive") 123 | 124 | subprocess.call(["dd", "if=/dev/zero", 125 | "of=%s" % self.victim(), 126 | "bs=4k", "count=100"]) 127 | 128 | if __name__ == '__main__': 129 | suite = unittest.TestLoader().loadTestsFromTestCase(CookbookUnitTests) 130 | unittest.TextTestRunner(verbosity=2).run(suite) 131 | -------------------------------------------------------------------------------- /cookbook/recipes: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ ! -d gen-py ]; then 4 | thrift -r --gen py ../server.thrift &> /dev/null 5 | fi 6 | 7 | python2 recipes.py $1 8 | -------------------------------------------------------------------------------- /cookbook/recipes.py: -------------------------------------------------------------------------------- 1 | import errno 2 | 3 | import sys 4 | 5 | sys.path.append('gen-py') 6 | from server import server 7 | from server.ttypes import * 8 | 9 | from thrift.transport import TSocket 10 | from thrift.transport import TTransport 11 | from thrift.protocol import TBinaryProtocol 12 | 13 | 14 | def usage(): 15 | print("Choose one of the following option:\n" 16 | " --clear\n" 17 | " --full\n" 18 | " --io-error\n" 19 | " --quota\n" 20 | " --delay\n" 21 | " --random\n" 22 | " --specific-syscalls\n" 23 | " --probability\n" 24 | " --file-pattern\n" 25 | " --broken-drive") 26 | sys.exit(1) 27 | 28 | 29 | def connect(): 30 | transport = TSocket.TSocket('127.0.0.1', 9090) 31 | transport = TTransport.TBufferedTransport(transport) 32 | protocol = TBinaryProtocol.TBinaryProtocol(transport) 33 | client = server.Client(protocol) 34 | transport.open() 35 | return client 36 | 37 | 38 | def main(): 39 | if len(sys.argv) != 2: 40 | usage() 41 | 42 | client = connect() 43 | 44 | client.clear_all_faults() 45 | 46 | if sys.argv[1] == "--clear": 47 | print("Clearing all faults conditions") 48 | sys.exit(0) 49 | elif sys.argv[1] == "--full": 50 | print("Simulating disk full") 51 | client.set_all_fault(False, errno.ENOSPC, 0, "", False, 0, False) 52 | elif sys.argv[1] == "--io-error": 53 | print("Simulating IO error") 54 | client.set_all_fault(False, errno.EIO, 0, "", False, 0, False) 55 | elif sys.argv[1] == "--quota": 56 | print("Simulating quota exceeded") 57 | client.set_all_fault(False, errno.EDQUOT, 0, "", False, 0, False) 58 | elif sys.argv[1] == "--delay": 59 | print("Simulating delayed IO") 60 | client.set_all_fault(False, 0, 0, "", False, 50000, False) 61 | elif sys.argv[1] == "--random": 62 | print("Simulating random errno") 63 | client.set_all_fault(True, 0, 0, "", False, 0, False) 64 | elif sys.argv[1] == "--specific-syscalls": 65 | print("Restricting random IO restricted to specific syscalls") 66 | client.set_fault(['read', 'read_buf', 'write', 'write_buf'], True, 0, 0, "", False, 0, False) 67 | elif sys.argv[1] == "--probability": 68 | print("Restricting random IO restricted to specific syscalls and 1% error probability") 69 | client.set_fault(['read', 'read_buf', 'write', 'write_buf'], True, 0, 1000, "", False, 0, False) 70 | elif sys.argv[1] == "--file-pattern": 71 | print("Restricting random IO restricted to specific syscalls while cursing *.sendmail.cf") 72 | client.set_fault(['read', 'read_buf', 'write', 'write_buf'], True, 0, 0, ".*sendmail.cf", False, 0, False) 73 | elif sys.argv[1] == "--broken-drive": 74 | print("The agonising drive simulator") 75 | client.set_all_fault(False, errno.EIO, 100, "", False, 100000, False) 76 | else: 77 | usage() 78 | 79 | 80 | if __name__ == "__main__": 81 | main() 82 | -------------------------------------------------------------------------------- /python_client.py: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path.append('gen-py') 3 | 4 | from gen.server import server 5 | from gen.server.ttypes import * 6 | 7 | from thrift import Thrift 8 | from thrift.transport import TSocket 9 | from thrift.transport import TTransport 10 | from thrift.protocol import TBinaryProtocol 11 | 12 | try: 13 | 14 | transport = TSocket.TSocket('127.0.0.1', 9090) 15 | transport = TTransport.TBufferedTransport(transport) 16 | protocol = TBinaryProtocol.TBinaryProtocol(transport) 17 | client = server.Client(protocol) 18 | transport.open() 19 | 20 | print(client.get_methods()) 21 | 22 | # client.set_fault(['flush', 'fsync', 'fsyncdir'], False, 0, 100000, "", True, 500000) 23 | client.set_fault(['flush', 'fsync', 'fsyncdir'], False, 0, 99000, "", True, 500000) 24 | # client.clear_all_faults() 25 | 26 | except Thrift.TException as tx: 27 | print('%s' % tx.message) 28 | -------------------------------------------------------------------------------- /server.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * ** 27-12-2015 3 | * ** 4 | * ** The author disclaims copyright to this source code. In place of 5 | * ** a legal notice, here is a blessing: 6 | * ** 7 | * ** May you do good and not evil. 8 | * ** May you find forgiveness for yourself and forgive others. 9 | * ** May you share freely, never taking more than you give. 10 | * ** 11 | */ 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include 25 | #include 26 | #include 27 | 28 | #include 29 | 30 | #include 31 | #include 32 | #include 33 | #include 34 | 35 | #include "gen-cpp/server.h" 36 | 37 | using namespace ::apache::thrift; 38 | using namespace ::apache::thrift::protocol; 39 | using namespace ::apache::thrift::transport; 40 | using namespace ::apache::thrift::server; 41 | 42 | using std::shared_ptr; 43 | 44 | struct fault_descriptor { 45 | bool random; // error code must be randomized 46 | int err_no; // error code to return 47 | int32_t probability; // 0 < probability < 100 48 | std::string regexp; // regular expression on filename 49 | bool kill_caller; // Must we kill the caller 50 | int32_t delay_us; // operation delay in us 51 | bool auto_delay; // must auto delay like an SSD 52 | }; 53 | 54 | std::set valid_methods; 55 | std::map fault_map; 56 | std::mutex mutex; 57 | 58 | void init_valid_methods() 59 | { 60 | valid_methods.insert("getattr"); 61 | valid_methods.insert("readlink"); 62 | valid_methods.insert("mknod"); 63 | valid_methods.insert("mkdir"); 64 | valid_methods.insert("unlink"); 65 | valid_methods.insert("rmdir"); 66 | valid_methods.insert("symlink"); 67 | valid_methods.insert("rename"); 68 | valid_methods.insert("link"); 69 | valid_methods.insert("chmod"); 70 | valid_methods.insert("chown"); 71 | valid_methods.insert("truncate"); 72 | valid_methods.insert("open"); 73 | valid_methods.insert("read"); 74 | valid_methods.insert("write"); 75 | valid_methods.insert("statfs"); 76 | valid_methods.insert("flush"); 77 | valid_methods.insert("release"); 78 | valid_methods.insert("fsync"); 79 | valid_methods.insert("setxattr"); 80 | valid_methods.insert("getxattr"); 81 | valid_methods.insert("listxattr"); 82 | valid_methods.insert("removexattr"); 83 | valid_methods.insert("opendir"); 84 | valid_methods.insert("readdir"); 85 | valid_methods.insert("releasedir"); 86 | valid_methods.insert("fsyncdir"); 87 | valid_methods.insert("access"); 88 | valid_methods.insert("create"); 89 | valid_methods.insert("ftruncate"); 90 | valid_methods.insert("fgetattr"); 91 | valid_methods.insert("lock"); 92 | valid_methods.insert("bmap"); 93 | valid_methods.insert("ioctl"); 94 | valid_methods.insert("poll"); 95 | valid_methods.insert("write_buf"); 96 | valid_methods.insert("read_buf"); 97 | valid_methods.insert("flock"); 98 | valid_methods.insert("fallocate"); 99 | } 100 | 101 | static bool is_valid_method(std::string method) 102 | { 103 | return valid_methods.count(method); 104 | } 105 | 106 | // return a random err_no 107 | static int random_err_no() 108 | { 109 | std::random_device rd; 110 | std::uniform_int_distribution dist(E2BIG, EXFULL); 111 | 112 | return dist(rd); 113 | } 114 | 115 | // return true if random number is not in the probability 116 | static bool get_lucky(int probability) 117 | { 118 | std::random_device rd; 119 | std::uniform_int_distribution dist(1, 100000); 120 | 121 | if (!probability) { 122 | return false; 123 | } 124 | 125 | if (dist(rd) > probability) { 126 | return true; 127 | } 128 | 129 | return false; 130 | } 131 | 132 | // return an err_no if we must proceed to error injection 133 | int error_inject(volatile int in_flight, std::string path, std::string method) 134 | { 135 | std::lock_guard lk(mutex); 136 | 137 | // no fault injection for this method 138 | if (!fault_map.count(method)) { 139 | return 0; 140 | } 141 | 142 | // get the fault injection descritor 143 | auto descr = fault_map[method]; 144 | 145 | int err_no = 0; 146 | 147 | // get the err_no to inject 148 | if (descr.err_no) { 149 | err_no = descr.err_no; 150 | } else if (descr.random) { 151 | err_no = random_err_no(); 152 | } 153 | 154 | if (descr.regexp.size()) { 155 | std::regex r(descr.regexp); 156 | if (!std::regex_match(path, r)) { 157 | return 0; 158 | } 159 | } 160 | 161 | // Are we out of error probability 162 | if (get_lucky(descr.probability)) { 163 | return 0; 164 | } 165 | 166 | uint32_t delay = 0; 167 | if (descr.delay_us) { 168 | delay = descr.delay_us; 169 | } 170 | 171 | if (descr.auto_delay) { 172 | delay = 0; 173 | } 174 | 175 | if (delay) { 176 | std::this_thread::sleep_for( 177 | std::chrono::microseconds(delay)); 178 | } 179 | 180 | if (descr.kill_caller) { 181 | static struct fuse_context *context; 182 | context = fuse_get_context(); 183 | kill(context->pid, SIGKILL); 184 | return 0; 185 | } 186 | 187 | return -err_no; 188 | } 189 | 190 | class server_handler: public serverIf { 191 | void get_methods(std::vector & _return) 192 | { 193 | for (auto method: valid_methods) { 194 | _return.push_back(method); 195 | } 196 | } 197 | 198 | void clear_all_faults() 199 | { 200 | std::lock_guard lk(mutex); 201 | fault_map.clear(); 202 | } 203 | 204 | void clear_fault(const std::string& method) 205 | { 206 | std::lock_guard lk(mutex); 207 | fault_map.erase(method); 208 | } 209 | 210 | void set_fault(const std::vector& methods, const bool random, 211 | const int32_t err_no, const int32_t probability, 212 | const std::string& regexp, const bool kill_caller, 213 | int32_t delay_us, const bool auto_delay) 214 | { 215 | struct fault_descriptor descr; 216 | 217 | descr.random = random; 218 | descr.err_no = err_no; 219 | descr.probability = probability; 220 | descr.regexp = regexp; 221 | descr.kill_caller = kill_caller; 222 | descr.delay_us = delay_us; 223 | descr.auto_delay = auto_delay; 224 | 225 | std::lock_guard lk(mutex); 226 | for (auto method: methods) { 227 | if (is_valid_method(method)) { 228 | fault_map[method] = descr; 229 | } 230 | } 231 | } 232 | 233 | void set_all_fault(const bool random, const int32_t err_no, 234 | const int32_t probability, const std::string& regexp, 235 | const bool kill_caller, const int32_t delay_us, 236 | const bool auto_delay) 237 | { 238 | std::vector methods; 239 | 240 | for (auto method: valid_methods) { 241 | methods.push_back(method); 242 | } 243 | 244 | set_fault(methods, random, err_no, probability, 245 | regexp, kill_caller, delay_us, 246 | auto_delay); 247 | } 248 | 249 | }; 250 | 251 | void server_thread() 252 | { 253 | int port = 9090; 254 | 255 | init_valid_methods(); 256 | 257 | std::cout << "Server Thread started" << std::endl; 258 | try { 259 | shared_ptr handler(new server_handler()); 260 | shared_ptr processor(new serverProcessor(handler)); 261 | shared_ptr serverTransport(new TServerSocket(port)); 262 | shared_ptr transportFactory(new TBufferedTransportFactory()); 263 | shared_ptr protocolFactory(new TBinaryProtocolFactory()); 264 | 265 | TSimpleServer server(processor, serverTransport, transportFactory, protocolFactory); 266 | std::cout << "Starting to serve" << std::endl; 267 | server.serve(); 268 | } catch(...) { 269 | std::cout << "Caught exception" << std::endl; 270 | } 271 | std::cout << "Finished serving" << std::endl; 272 | } 273 | 274 | std::thread *thread; 275 | 276 | void start_server_thread() { 277 | thread = new std::thread(server_thread); 278 | } 279 | -------------------------------------------------------------------------------- /server.hh: -------------------------------------------------------------------------------- 1 | /* 2 | * ** 27-12-2015 3 | * ** 4 | * ** The author disclaims copyright to this source code. In place of 5 | * ** a legal notice, here is a blessing: 6 | * ** 7 | * ** May you do good and not evil. 8 | * ** May you find forgiveness for yourself and forgive others. 9 | * ** May you share freely, never taking more than you give. 10 | * ** 11 | */ 12 | 13 | #ifndef SERVER_HH 14 | #define SERVER_HH 15 | 16 | #ifdef __cplusplus 17 | 18 | #include 19 | 20 | int error_inject(volatile int in_flight, std::string path, std::string method); 21 | 22 | #endif 23 | 24 | void start_server_thread(); 25 | 26 | #endif 27 | -------------------------------------------------------------------------------- /server.thrift: -------------------------------------------------------------------------------- 1 | // These are the method of the CharybdeFS thrift client object 2 | // 3 | // For more info on how to use thrift with python 4 | // look at: https://thrift.apache.org/tutorial/py 5 | // 6 | 7 | service server { 8 | 9 | // Used to get the list of availables systems calls 10 | list get_methods(), 11 | 12 | // Used to clear all faults sources 13 | void clear_all_faults(), 14 | 15 | // Used to clear a specific method fault 16 | void clear_fault(string method), 17 | 18 | // Set fault on a specific list of methods 19 | void set_fault(list methods, // the list of methods to operate on 20 | bool random, // Must we return random errno 21 | i32 err_no, // A specific errno to return 22 | i32 probability, // Fault probability over 100 000 23 | string regexp, // A regexp matching a victim file 24 | bool kill_caller, // Kill -9 the caller process 25 | i32 delay_us, // Delay to inject in the fs calls 26 | bool auto_delay), // Not implemented yet: Will be used to simulate SSDs latencies 27 | 28 | // Works like set_fault but applies the fault to all methods 29 | void set_all_fault(bool random, 30 | i32 err_no, 31 | i32 probability, 32 | string regexp, 33 | bool kill_caller, 34 | i32 delay_us, 35 | bool auto_delay), 36 | } 37 | -------------------------------------------------------------------------------- /tests/common.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import shutil 4 | import time 5 | import mmap 6 | import signal 7 | import subprocess 8 | 9 | from thrift.transport import TSocket 10 | from thrift.transport import TTransport 11 | from thrift.protocol import TBinaryProtocol 12 | 13 | sys.path.append('gen-py') 14 | from server import server 15 | from server.ttypes import * 16 | 17 | 18 | def clear_scylla_dir(): 19 | shutil.rmtree("/var/lib/scylla/commitlog", ignore_errors=True) 20 | shutil.rmtree("/var/lib/scylla/data", ignore_errors=True) 21 | 22 | 23 | def build_log_filename(method, write): 24 | op = "read" 25 | if write: 26 | op = "write" 27 | return "scylla-%s-log-for-%s.txt" % (op, method) 28 | 29 | 30 | def start_scylla(method, write): 31 | print("Starting scylla") 32 | log = open(build_log_filename(method, write), 'w') 33 | proc = subprocess.Popen(['/usr/bin/scylla', 34 | '-m', '512M', 35 | '--developer-mode=1'], 36 | preexec_fn=os.setsid, 37 | stdout=log, stderr=log) 38 | time.sleep(60) 39 | return proc, log 40 | 41 | 42 | def stop_scylla(proc, log): 43 | os.killpg(os.getpgid(proc.pid), signal.SIGTERM) 44 | proc.wait() 45 | log.close() 46 | 47 | 48 | def has_message(method, message): 49 | result = False 50 | f = open(build_log_filename(method, True)) 51 | s = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) 52 | if s.find(message) != -1: 53 | result = True 54 | s.close() 55 | f.close() 56 | return result 57 | 58 | 59 | def connect(): 60 | transport = TSocket.TSocket('127.0.0.1', 9090) 61 | transport = TTransport.TBufferedTransport(transport) 62 | protocol = TBinaryProtocol.TBinaryProtocol(transport) 63 | client = server.Client(protocol) 64 | transport.open() 65 | return client 66 | -------------------------------------------------------------------------------- /tests/kill_in_flush_test.py: -------------------------------------------------------------------------------- 1 | from subprocess import check_output 2 | import random 3 | 4 | from common import * 5 | 6 | 7 | def stress(): 8 | FNULL = open(os.devnull, 'w') 9 | proc = subprocess.Popen(['/usr/bin/cassandra-stress', 10 | 'write', 11 | 'duration=15m', 12 | '-mode', 13 | 'native', 14 | 'cql3', 15 | '-rate', 16 | 'threads=700', 17 | '-node', 18 | 'localhost'], 19 | preexec_fn=os.setsid, 20 | stdout=FNULL, 21 | stderr=subprocess.STDOUT) 22 | return proc 23 | 24 | 25 | def is_running(name): 26 | try: 27 | check_output(["pidof", name]) == 0 28 | except subprocess.CalledProcessError: 29 | return False 30 | return True 31 | 32 | 33 | def loop(): 34 | client = connect() 35 | client.clear_all_faults() 36 | 37 | clear_scylla_dir() 38 | scylla_proc, scylla_log = start_scylla("fsync", True) 39 | time.sleep(5) 40 | 41 | # start the stress and wait 42 | print("Starting cassandra stress") 43 | stress_proc = stress() 44 | # setup a random time to hit every write path at random 45 | # commitlog / compaction / SSTable creation ... 46 | time.sleep(random.randint(15, 50)) 47 | 48 | # trigger kill on flush/sync and wait it work 49 | print("Setting flush/sync to kill scylla") 50 | client.set_fault(['flush', 'fsync', 'fsyncdir'], False, 0, 100000, "", True, 0, False) 51 | print("Waiting for scylla to die") 52 | while is_running("scylla"): 53 | time.sleep(1) 54 | print("Scylla died") 55 | print("Clearing flush/sync kill") 56 | client.clear_all_faults() 57 | 58 | print("Stopping cassandra stress") 59 | os.killpg(os.getpgid(stress_proc.pid), signal.SIGTERM) 60 | stress_proc.wait() 61 | 62 | scylla_proc, scylla_log = start_scylla("fsync-recover", True) 63 | 64 | time.sleep(60) 65 | 66 | if not is_running("scylla"): 67 | print("Scylla failed to boot after fsync kill. Something bad happened.") 68 | sys.exit(1) 69 | 70 | stop_scylla(scylla_proc, scylla_log) 71 | 72 | if not has_message("fsync-recover", b'Starting listening for CQL clients'): 73 | print("Scylla did not boot completely after fsync kill. Something bad happened") 74 | sys.exit(2) 75 | 76 | print("Success") 77 | print("") 78 | 79 | 80 | def main(): 81 | while True: 82 | loop() 83 | 84 | if __name__ == "__main__": 85 | main() 86 | -------------------------------------------------------------------------------- /tests/out_of_disk_space_test.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | from common import * 4 | 5 | 6 | def stress(): 7 | FNULL = open(os.devnull, 'w') 8 | proc = subprocess.Popen(['/usr/bin/cassandra-stress', 9 | 'write', 10 | 'duration=15m', 11 | '-mode', 12 | 'native', 13 | 'cql3', 14 | '-rate', 15 | 'threads=700', 16 | '-node', 17 | 'localhost'], 18 | preexec_fn=os.setsid, 19 | stdout=FNULL, 20 | stderr=subprocess.STDOUT) 21 | return proc 22 | 23 | 24 | def cleanup(scylla_proc, scylla_log, stress_proc): 25 | print("Stopping cassandra stress") 26 | os.killpg(os.getpgid(stress_proc.pid), signal.SIGTERM) 27 | stress_proc.wait() 28 | 29 | stop_scylla(scylla_proc, scylla_log) 30 | 31 | 32 | def main(): 33 | client = connect() 34 | client.clear_all_faults() 35 | 36 | scylla_proc, scylla_log = start_scylla("out-of-disk-space", True) 37 | 38 | print("Starting cassandra stress") 39 | stress_proc = stress() 40 | # Sufficient time for activity such as memtable write, compaction, etc. 41 | time.sleep(random.randint(20, 50)) 42 | 43 | methods = client.get_methods() 44 | ENOSPC = 28 # no space left on device error 45 | print("Creating out of disk space scenario") 46 | client.set_fault(methods, False, ENOSPC, 100000, "", False, 0, False) 47 | 48 | # Leave system with no space left for a few seconds. 49 | time.sleep(20) 50 | 51 | # Check that the system faced the scenario 52 | if not has_message("out-of-disk-space", b'No space left on device'): 53 | print("Failed to make system run out of disk space") 54 | cleanup(scylla_proc, scylla_log, stress_proc) 55 | sys.exit(1) 56 | print("System is out of disk space as expected") 57 | 58 | # Clean faults and let the system recover from the out of disk space scenario. 59 | print("Destroying out of disk space scenario") 60 | client.clear_all_faults() 61 | scylla_log.seek(0) 62 | scylla_log.truncate() 63 | time.sleep(20) 64 | 65 | # Checking that the system was actually able to recover 66 | if has_message("out-of-disk-space", b'No space left on device'): 67 | print("Failed to make system recover from out of disk space scenario") 68 | cleanup(scylla_proc, scylla_log, stress_proc) 69 | sys.exit(1) 70 | print("System is no longer out of disk space as expected") 71 | 72 | cleanup(scylla_proc, scylla_log, stress_proc) 73 | 74 | print("Success") 75 | print("") 76 | 77 | if __name__ == "__main__": 78 | main() 79 | -------------------------------------------------------------------------------- /tests/python_client: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import sys 3 | sys.path.append('gen-py') 4 | 5 | from gen.server import server 6 | from gen.server.ttypes import * 7 | 8 | from thrift import Thrift 9 | from thrift.transport import TSocket 10 | from thrift.transport import TTransport 11 | from thrift.protocol import TBinaryProtocol 12 | 13 | try: 14 | 15 | transport = TSocket.TSocket('127.0.0.1', 9090) 16 | transport = TTransport.TBufferedTransport(transport) 17 | protocol = TBinaryProtocol.TBinaryProtocol(transport) 18 | client = server.Client(protocol) 19 | transport.open() 20 | 21 | print(client.get_methods()) 22 | 23 | # client.set_fault(['flush', 'fsync', 'fsyncdir'], False, 0, 100000, "", True, 500000) 24 | client.set_all_fault(True, 0, 100000, "", False, 0) 25 | # client.clear_all_faults() 26 | 27 | except Thrift.TException as tx: 28 | print('%s' % tx.message) 29 | -------------------------------------------------------------------------------- /tests/scylla_test_all_calls.py: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path.append('gen-py') 3 | 4 | import random 5 | import threading 6 | 7 | from gen.server.ttypes import * 8 | 9 | from common import * 10 | 11 | 12 | def create_database(session): 13 | session.execute("""CREATE KEYSPACE sha WITH 14 | replication = {'class': 'SimpleStrategy', 'replication_factor': 1}""") 15 | 16 | session.execute("""CREATE TABLE sha.test ( 17 | data ascii PRIMARY KEY, 18 | );""") 19 | session.execute("""Use sha""") 20 | 21 | counter = 0 22 | keep_going = False 23 | lock = threading.Lock() 24 | 25 | 26 | def mkdata(gen): 27 | res = "".join([str(x) for x in (gen + x for x in range(1000))]) 28 | gen += 1000 29 | return res, gen 30 | 31 | 32 | def writer(): 33 | """thread writer function""" 34 | from cassandra.cluster import Cluster 35 | cluster = Cluster() 36 | session = cluster.connect() 37 | 38 | create_database(session) 39 | 40 | global keep_going 41 | global counter 42 | global lock 43 | counter = 0 44 | gen = 0 45 | keep_going = True 46 | random.seed(0) 47 | lock.acquire() 48 | while keep_going: 49 | data, gen = mkdata(gen) 50 | try: 51 | query = "INSERT INTO sha.test (data) VALUES('%s');" % (data) 52 | session.execute(query) 53 | except Exception as e: 54 | print(str(e)) 55 | keep_going = False 56 | continue 57 | counter += 1 58 | # let a change to the other thread to acquire the lock 59 | lock.release() 60 | lock.acquire() 61 | lock.release() 62 | 63 | 64 | def read_check(count): 65 | from cassandra.cluster import Cluster 66 | cluster = Cluster() 67 | session = cluster.connect() 68 | 69 | gen = 0 70 | counter = 0 71 | random.seed(0) 72 | 73 | while counter <= count: 74 | data, gen = mkdata(gen) 75 | 76 | try: 77 | query = "SELECT data FROM sha.test WHERE data='%s';" % data 78 | rows = session.execute(query) 79 | count = 0 80 | for row in rows: 81 | count += 1 82 | if count != 1: 83 | return False 84 | except Exception as e: 85 | print(str(e)) 86 | keep_going = False 87 | continue 88 | 89 | counter += 1 90 | 91 | return True 92 | 93 | 94 | def start_writing(): 95 | t = threading.Thread(target=writer) 96 | t.start() 97 | return t 98 | 99 | 100 | def stop_writing(t): 101 | global keep_going 102 | global lock 103 | 104 | lock.acquire() 105 | keep_going = False 106 | res = counter 107 | lock.release() 108 | 109 | t.join() 110 | 111 | return res 112 | 113 | 114 | def test_method(method, client): 115 | print("Testing %s" % method) 116 | clear_scylla_dir() 117 | scylla_proc, scylla_log = start_scylla(method, True) 118 | 119 | write_thread = start_writing() 120 | import random 121 | time.sleep(random.randint(300, 1000)) 122 | 123 | client.set_fault([method], True, 0, 100000, "", False, 0, False) 124 | time.sleep(5) 125 | client.clear_all_faults() 126 | 127 | if has_message(method, b'ERROR') and\ 128 | not has_message(method, b'Shutdown communications until operator examinate the situation.'): 129 | print("Error handling method=%s" % method) 130 | sys.exit(1) 131 | 132 | time.sleep(30) 133 | 134 | count = stop_writing(write_thread) 135 | stop_scylla(scylla_proc, scylla_log) 136 | 137 | result = False 138 | scylla_proc, scylla_log = start_scylla(method, False) 139 | try: 140 | result = read_check(count) 141 | except Exception as e: 142 | print("Read check error for method=%s: %s" % (method, str(e))) 143 | sys.exit(2) 144 | if not result: 145 | print("Error in %s" % method) 146 | sys.exit(3) 147 | stop_scylla(scylla_proc, scylla_log) 148 | 149 | 150 | def main(): 151 | client = connect() 152 | 153 | methods = client.get_methods() 154 | 155 | while True: 156 | for method in methods: 157 | test_method(method, client) 158 | 159 | if __name__ == "__main__": 160 | main() 161 | --------------------------------------------------------------------------------