├── ReadMe.markdown ├── TFS_RESTful_API.markdown ├── conf └── example.conf ├── config └── src ├── ngx_http_connection_pool.c ├── ngx_http_connection_pool.h ├── ngx_http_tfs.c ├── ngx_http_tfs.h ├── ngx_http_tfs_block_cache.c ├── ngx_http_tfs_block_cache.h ├── ngx_http_tfs_data_server_message.c ├── ngx_http_tfs_data_server_message.h ├── ngx_http_tfs_duplicate.c ├── ngx_http_tfs_duplicate.h ├── ngx_http_tfs_errno.h ├── ngx_http_tfs_json.c ├── ngx_http_tfs_json.h ├── ngx_http_tfs_local_block_cache.c ├── ngx_http_tfs_local_block_cache.h ├── ngx_http_tfs_meta_server_message.c ├── ngx_http_tfs_meta_server_message.h ├── ngx_http_tfs_module.c ├── ngx_http_tfs_name_server_message.c ├── ngx_http_tfs_name_server_message.h ├── ngx_http_tfs_peer_connection.c ├── ngx_http_tfs_peer_connection.h ├── ngx_http_tfs_protocol.h ├── ngx_http_tfs_raw_fsname.c ├── ngx_http_tfs_raw_fsname.h ├── ngx_http_tfs_rc_server_info.c ├── ngx_http_tfs_rc_server_info.h ├── ngx_http_tfs_rc_server_message.c ├── ngx_http_tfs_rc_server_message.h ├── ngx_http_tfs_remote_block_cache.c ├── ngx_http_tfs_remote_block_cache.h ├── ngx_http_tfs_restful.c ├── ngx_http_tfs_restful.h ├── ngx_http_tfs_root_server_message.c ├── ngx_http_tfs_root_server_message.h ├── ngx_http_tfs_server_handler.c ├── ngx_http_tfs_server_handler.h ├── ngx_http_tfs_tair_helper.c ├── ngx_http_tfs_tair_helper.h ├── ngx_http_tfs_timers.c ├── ngx_http_tfs_timers.h ├── ngx_tfs_common.c └── ngx_tfs_common.h /ReadMe.markdown: -------------------------------------------------------------------------------- 1 | 名称 2 | ==== 3 | 4 | * nginx-tfs 5 | 6 | 描述 7 | ==== 8 | 9 | * 这个模块实现了TFS的客户端，为TFS提供了RESTful API。TFS的全称是Taobao File System，是淘宝开源的一个分布式文件系统。 10 | 11 | 编译安装 12 | ======= 13 | 14 | 1. TFS模块使用了一个开源的JSON库来支持JSON，请先安装[yajl](http://lloyd.github.com/yajl/)-2.0.1。 15 | 16 | 2. 下载[nginx](http://www.nginx.org/)或[tengine](http://tengine.taobao.org/)。 17 | 18 | 3. ./configure --add-module=/path/to/nginx-tfs 19 | 20 | 4. make && make install 21 | 22 | 配置 23 | ==== 24 | 25 | http { 26 | tfs_upstream tfs_rc { 27 | server 127.0.0.1:6100; 28 | type rcs; 29 | rcs_zone name=tfs1 size=128M; 30 | rcs_interface eth0; 31 | rcs_heartbeat lock_file=/logs/lk.file interval=10s; 32 | } 33 | 34 | server { 35 | listen 7500; 36 | server_name localhost; 37 | 38 | tfs_keepalive max_cached=100 bucket_count=10; 39 | tfs_log "pipe:/usr/sbin/cronolog -p 30min /path/to/nginx/logs/cronolog/%Y/%m/%Y-%m-%d-%H-%M-tfs_access.log"; 40 | 41 | location / { 42 | tfs_pass tfs://tfs_rc; 43 | } 44 | } 45 | } 46 | 47 | 指令 48 | ==== 49 | 50 | server 51 | ------------ 52 | 53 | **Syntax**： *server address* 54 | 55 | **Default**： *none* 56 | 57 | **Context**： *tfs_upstream* 58 | 59 | 指定后端TFS服务器的地址，当指令type为rcs时为RcServer的地址，如果为为ns时为NameServer的地址。此指令必须配置。例如: 60 | 61 | server 10.0.0.1:8108; 62 | 63 | type 64 | ---------------- 65 | 66 | **Syntax**： *type [ns | rcs]* 67 | 68 | **Default**： *ns* 69 | 70 | **Context**： *tfs_upstream* 71 | 72 | 设置server类型，类型只能为ns或者rcs，如果为ns,则指令server指定的地址为NameServer的地址，如果为rcs,则为RcServer的地址。如需使用自定义文件名功能请设置类型为rcs，使用自定义文件名功能需额外配置MetaServer和RootServer。 73 | 74 | rcs\_zone 75 | -------------- 76 | 77 | **Syntax**： *rcs_zone name=n size=num* 78 | 79 | **Default**： *none* 80 | 81 | **Context**： *tfs_upstream* 82 | 83 | 配置TFS应用在RcServer的配置信息。若开启RcServer（配置了type rcs），则必须配置此指令。配置此指令会在共享内存中缓存TFS应用在RcServer的配置信息，并可以通过指令rcs_heartbeat来和RcServer进行keepalive以保证应用的配置信息的及时更新。例如： 84 | 85 | rcs_zone name=tfs1 size=128M; 86 | 87 | rcs\_heartbeat 88 | -------------- 89 | 90 | **Syntax**： *rcs_heartbeat lock_file=/path/to/file interval=time* 91 | 92 | **Default**： *none* 93 | 94 | **Context**： *tfs_upstream* 95 | 96 | 配置TFS应用和RcServer的keepalive，应用可通过此功能来和RcServer定期交互，以及时更新其配置信息。若开启RcServer功能（配置了type rcs），则必须配置此指令。例如： 97 | 98 | rcs_heartbeat lock_file=/path/to/nginx/logs/lk.file interval=10s; 99 | 100 | rcs\_interface 101 | ---------------- 102 | 103 | **Syntax**： *rcs\_interface interface* 104 | 105 | **Default**： *none* 106 | 107 | **Context**： *tfs_upstream* 108 | 109 | 配置TFS模块使用的网卡。若开启RcServer功能（配置了type rcs），则必须配置此指令。例如： 110 | 111 | rcs_interface eth0; 112 | 113 | tfs\_upstream 114 | ---------------- 115 | 116 | **Syntax**： *tfs\_upstream name {...}* 117 | 118 | **Default**： *none* 119 | 120 | **Context**： *http* 121 | 122 | 配置TFS模块的server信息,这个块包括上面几个命令。例如： 123 | 124 | tfs_upstream tfs_rc { 125 | server 127.0.0.1:6100; 126 | type rcs; 127 | rcs_zone name=tfs1 size=128M; 128 | rcs_interface eth0; 129 | rcs_heartbeat lock_file=/logs/lk.file interval=10s; 130 | } 131 | 132 | 133 | tfs_pass 134 | -------- 135 | 136 | **Syntax**： *tfs_pass name* 137 | 138 | **Default**： *none* 139 | 140 | **Context**： *location* 141 | 142 | 是否打开TFS模块功能，此指令为关键指令，决定请求是否由TFS模块处理，必须配置。需要注意，这里不支持直接写ip地址或者域名，这里只支持指令tfs_upstream name {...} 配置的upstream,并且必须以 tfs:// 开头。例如： 143 | 144 | 145 | tfs_upstream tfs_rc { 146 | }; 147 | 148 | location / { 149 | tfs_pass tfs://tfs_rc; 150 | } 151 | 152 | tfs_keepalive 153 | ------------- 154 | 155 | **Syntax**： *tfs_keepalive max_cached=num bucket_count=num* 156 | 157 | **Default**： *none* 158 | 159 | **Context**： *http, server* 160 | 161 | 配置TFS模块使用的连接池的大小，TFS模块的连接池会缓存TFS模块和后端TFS服务器的连接。可以把这个连接池看作由多个hash桶构成的hash表，其中bucket\_count是桶的个数，max\_cached是桶的容量。此指令必须配置。注意，应该根据机器的内存情况来合理配置连接池的大小。例如： 162 | 163 | tfs_keepalive max_cached=100 bucket_count=15; 164 | 165 | tfs\_block\_cache\_zone 166 | ----------------------- 167 | 168 | **Syntax**： *tfs_block_cache_zone size=num* 169 | 170 | **Default**： *none* 171 | 172 | **Context**： *http* 173 | 174 | 配置TFS模块的本地BlockCache。配置此指令会在共享内存中缓存TFS中的Block和DataServer的映射关系。注意，应根据机器的内存情况来合理配置BlockCache大小。例如： 175 | 176 | tfs_block_cache_zone size=256M; 177 | 178 | tfs\_log 179 | ---------------- 180 | 181 | **Syntax**： *tfs_log path* 182 | 183 | **Default**： *none* 184 | 185 | **Context**： *http, server* 186 | 187 | 是否进行TFS访问记录。配置此指令会以固定格式将访问TFS的请求记录到指定log中，以便进行分析。具体格式参见代码。例如： 188 | 189 | tfs_log "pipe:/usr/sbin/cronolog -p 30min /path/to/nginx/logs/cronolog/%Y/%m/%Y-%m-%d-%H-%M-tfs_access.log"; 190 | 191 | 注：cronolog支持依赖于tengine提供的扩展的日志模块。 192 | 193 | tfs\_body\_buffer\_size 194 | ----------------------- 195 | 196 | **Syntax**： *tfs_body_buffer_size size* 197 | 198 | **Default**： *8k|16k* 199 | 200 | **Context**： *http, server, location* 201 | 202 | 配置用于和后端TFS服务器交互时使用的的body_buffer的大小。默认为页大小的2倍。建议设为2m。例如： 203 | 204 | tfs_body_buffer_size 2m; 205 | 206 | tfs\_connect\_timeout 207 | --------------------- 208 | 209 | **Syntax**： *tfs_connect_timeout time* 210 | 211 | **Default**： *3s* 212 | 213 | **Context**： *http, server, location* 214 | 215 | 配置连接后端TFS服务器的超时时间。 216 | 217 | tfs\_send\_timeout 218 | ------------------ 219 | 220 | **Syntax**： *tfs_send_timeout time* 221 | 222 | **Default**： *3s* 223 | 224 | **Context**： *http, server, location* 225 | 226 | 配置向后端TFS服务器发送数据的超时时间。 227 | 228 | tfs\_read\_timeout 229 | ------------------ 230 | 231 | **Syntax**： *tfs_read_timeout time* 232 | 233 | **Default**： *3s* 234 | 235 | **Context**： *http, server, location* 236 | 237 | 配置从后端TFS服务器接收数据的超时时间。 238 | 239 | 其他 240 | ---- 241 | 能支持上传文件大小决定于client_max_body_size指令配置的大小。 242 | 243 | 244 | 245 | -------------------------------------------------------------------------------- /conf/example.conf: -------------------------------------------------------------------------------- 1 | 2 | #user nobody; 3 | worker_processes 8; 4 | 5 | error_log logs/error.log warn; 6 | #error_log "pipe:/usr/sbin/cronolog /path/to/nginx/logs/cronolog/%Y/%m/%Y-%m-%d-error.log" warn; 7 | 8 | #pid logs/nginx.pid; 9 | 10 | #daemon off; 11 | 12 | events { 13 | worker_connections 10240; 14 | } 15 | 16 | 17 | http { 18 | include mime.types; 19 | default_type application/octet-stream; 20 | 21 | #log_format main '$remote_addr - $remote_user [$time_local] "$request" ' 22 | # '$status $body_bytes_sent "$http_referer" ' 23 | # '"$http_user_agent" "$http_x_forwarded_for"'; 24 | 25 | #access_log logs/access.log main; 26 | 27 | sendfile on; 28 | #tcp_nopush on; 29 | 30 | #keepalive_timeout 0; 31 | keepalive_timeout 65; 32 | 33 | gzip off; 34 | 35 | tfs_upstream tfs_ups { 36 | # ns_addr 37 | server 10.0.0.1:8108; 38 | # rcs_addr 39 | #server 10.0.0.1:7200; 40 | type ns; 41 | #rcs_interface eth0; 42 | #rcs_zone name=tfs1 size=128M; 43 | #rcs_heartbeat lock_file=/logs/lk.file interval=10s; 44 | } 45 | 46 | tfs_block_cache_zone size=256M; 47 | 48 | tfs_send_timeout 3s; 49 | tfs_connect_timeout 3s; 50 | tfs_read_timeout 3s; 51 | 52 | tfs_body_buffer_size 2m; 53 | 54 | server { 55 | listen 7500; 56 | server_name localhost; 57 | client_max_body_size 40960m; 58 | 59 | #charset koi8-r; 60 | 61 | #access_log "pipe:/usr/sbin/cronolog /path/to/nginx/logs/cronolog/%Y/%m/%Y-%m-%d-access.log"; 62 | 63 | tfs_keepalive max_cached=100 bucket_count=10; 64 | #tfs_log "pipe:/usr/sbin/cronolog -p 30min /path/to/nginx/logs/cronolog/%Y/%m/%Y-%m-%d-%H-%M-tfs_access.log"; 65 | 66 | location / { 67 | tfs_pass tfs://tfs_ups; 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /config: -------------------------------------------------------------------------------- 1 | 2 | # Copyright (C) 2010-2013 Alibaba Group Holding Limited 3 | 4 | ngx_feature_name= 5 | ngx_feature_run=no 6 | ngx_feature_incs="#include \ 7 | #include " 8 | ngx_feature_test="yajl_version();" 9 | 10 | if [ -n "$LIBYAJL_INC" -o -n "$LIBYAJL_LIB" ]; then 11 | # explicit set libdyajl lib path 12 | ngx_feature="libdyajl library in directories specified by LIBYAJL_INC ($LIBYAJL_INC) and LIBYAJL_LIB ($LIBYAJL_LIB)" 13 | ngx_feature_path="$LIBYAJL_INC" 14 | if [ $NGX_RPATH = YES ]; then 15 | ngx_feature_libs="-R$LIBYAJL_LIB -L$LIBYAJL_LIB -lyajl" 16 | else 17 | ngx_feature_libs="-L$LIBYAJL_LIB -lyajl" 18 | fi 19 | . auto/feature 20 | else 21 | # auto-discovery 22 | ngx_feature="libyajl library" 23 | ngx_feature_path= 24 | ngx_feature_libs="-lyajl" 25 | . auto/feature 26 | 27 | if [ $ngx_found = no ]; then 28 | # FreeBSD, OpenBSD, linux 29 | ngx_feature="libyajl library in /usr/local/" 30 | ngx_feature_path="/usr/local/include/yajl /usr/local/include" 31 | if [ $NGX_RPATH = YES ]; then 32 | ngx_feature_libs="-R/usr/local/lib -L/usr/local/lib -lyajl" 33 | else 34 | ngx_feature_libs="-L/usr/local/lib -lyajl" 35 | fi 36 | . auto/feature 37 | fi 38 | 39 | if [ $ngx_found = no ]; then 40 | # NetBSD 41 | ngx_feature="libyajl library in /usr/pkg/" 42 | ngx_feature_path="/usr/pkg/include/yajl /usr/pkg/include" 43 | if [ $NGX_RPATH = YES ]; then 44 | ngx_feature_libs="-R/usr/pkg/lib -L/usr/pkg/lib -lyajl" 45 | else 46 | ngx_feature_libs="-L/usr/pkg/lib -lyajl" 47 | fi 48 | . auto/feature 49 | fi 50 | 51 | if [ $ngx_found = no ]; then 52 | # MacPorts 53 | ngx_feature="libyajl library in /opt/local/" 54 | ngx_feature_path="/opt/local/include/yajl /opt/local/include" 55 | if [ $NGX_RPATH = YES ]; then 56 | ngx_feature_libs="-R/opt/local/lib -L/opt/local/lib -lyajl" 57 | else 58 | ngx_feature_libs="-L/opt/local/lib -lyajl" 59 | fi 60 | . auto/feature 61 | fi 62 | fi 63 | 64 | if [ $ngx_found = yes ]; then 65 | CORE_INCS="$CORE_INCS $ngx_feature_path" 66 | CORE_LIBS="$CORE_LIBS $ngx_feature_libs" 67 | else 68 | cat << END 69 | $0: error: the ngx_tfs addon requires the libyajl library. 70 | END 71 | exit 1 72 | fi 73 | 74 | 75 | ngx_addon_name=ngx_http_tfs_module 76 | HTTP_MODULES="$HTTP_MODULES ngx_http_tfs_module" 77 | 78 | 79 | NGX_ADDON_DEPS="$NGX_ADDON_DEPS \ 80 | $ngx_addon_dir/src/ngx_http_tfs.h \ 81 | $ngx_addon_dir/src/ngx_http_tfs_errno.h \ 82 | $ngx_addon_dir/src/ngx_tfs_common.h \ 83 | $ngx_addon_dir/src/ngx_http_tfs_restful.h \ 84 | $ngx_addon_dir/src/ngx_http_tfs_restful.h \ 85 | $ngx_addon_dir/src/ngx_http_tfs_protocol.h \ 86 | $ngx_addon_dir/src/ngx_http_connection_pool.h \ 87 | $ngx_addon_dir/src/ngx_http_tfs_json.h \ 88 | $ngx_addon_dir/src/ngx_http_tfs_rc_server_info.h \ 89 | $ngx_addon_dir/src/ngx_http_tfs_raw_fsname.h \ 90 | $ngx_addon_dir/src/ngx_http_tfs_tair_helper.h \ 91 | $ngx_addon_dir/src/ngx_http_tfs_duplicate.h \ 92 | $ngx_addon_dir/src/ngx_http_tfs_block_cache.h \ 93 | $ngx_addon_dir/src/ngx_http_tfs_local_block_cache.h \ 94 | $ngx_addon_dir/src/ngx_http_tfs_remote_block_cache.h \ 95 | $ngx_addon_dir/src/ngx_http_tfs_timers.h \ 96 | $ngx_addon_dir/src/ngx_http_tfs_rc_server_message.h \ 97 | $ngx_addon_dir/src/ngx_http_tfs_name_server_message.h \ 98 | $ngx_addon_dir/src/ngx_http_tfs_data_server_message.h \ 99 | $ngx_addon_dir/src/ngx_http_tfs_root_server_message.h \ 100 | $ngx_addon_dir/src/ngx_http_tfs_meta_server_message.h \ 101 | $ngx_addon_dir/src/ngx_http_tfs_peer_connection.h \ 102 | $ngx_addon_dir/src/ngx_http_tfs_server_handler.h " 103 | 104 | 105 | NGX_ADDON_SRCS="$NGX_ADDON_SRCS \ 106 | $ngx_addon_dir/src/ngx_tfs_common.c \ 107 | $ngx_addon_dir/src/ngx_http_tfs.c \ 108 | $ngx_addon_dir/src/ngx_http_tfs_module.c \ 109 | $ngx_addon_dir/src/ngx_http_tfs_restful.c \ 110 | $ngx_addon_dir/src/ngx_http_connection_pool.c \ 111 | $ngx_addon_dir/src/ngx_http_tfs_json.c \ 112 | $ngx_addon_dir/src/ngx_http_tfs_rc_server_info.c \ 113 | $ngx_addon_dir/src/ngx_http_tfs_raw_fsname.c \ 114 | $ngx_addon_dir/src/ngx_http_tfs_tair_helper.c \ 115 | $ngx_addon_dir/src/ngx_http_tfs_duplicate.c \ 116 | $ngx_addon_dir/src/ngx_http_tfs_block_cache.c \ 117 | $ngx_addon_dir/src/ngx_http_tfs_local_block_cache.c \ 118 | $ngx_addon_dir/src/ngx_http_tfs_remote_block_cache.c \ 119 | $ngx_addon_dir/src/ngx_http_tfs_timers.c \ 120 | $ngx_addon_dir/src/ngx_http_tfs_rc_server_message.c \ 121 | $ngx_addon_dir/src/ngx_http_tfs_name_server_message.c \ 122 | $ngx_addon_dir/src/ngx_http_tfs_data_server_message.c \ 123 | $ngx_addon_dir/src/ngx_http_tfs_root_server_message.c \ 124 | $ngx_addon_dir/src/ngx_http_tfs_meta_server_message.c \ 125 | $ngx_addon_dir/src/ngx_http_tfs_peer_connection.c \ 126 | $ngx_addon_dir/src/ngx_http_tfs_server_handler.c " 127 | 128 | 129 | CORE_LIBS="$CORE_LIBS" 130 | CORE_INCS="$CORE_INCS $ngx_addon_dir/src" 131 | -------------------------------------------------------------------------------- /src/ngx_http_connection_pool.c: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #include 8 | 9 | 10 | static void ngx_http_connection_pool_close(ngx_connection_t *c); 11 | static void ngx_http_connection_pool_close_handler(ngx_event_t *ev); 12 | static void ngx_http_connection_pool_dummy_handler(ngx_event_t *ev); 13 | 14 | static ngx_int_t ngx_http_connection_pool_get(ngx_peer_connection_t *pc, 15 | void *data); 16 | static void ngx_http_connection_pool_free(ngx_peer_connection_t *pc, 17 | void *data, ngx_uint_t state); 18 | 19 | 20 | ngx_http_connection_pool_t * 21 | ngx_http_connection_pool_init(ngx_pool_t *pool, ngx_uint_t max_cached, 22 | ngx_uint_t bucket_count) 23 | { 24 | ngx_uint_t j, k; 25 | ngx_http_connection_pool_t *conn_pool; 26 | ngx_http_connection_pool_elt_t *cached; 27 | 28 | conn_pool = ngx_pcalloc(pool, sizeof(ngx_http_connection_pool_t)); 29 | if (conn_pool == NULL) { 30 | return NULL; 31 | } 32 | 33 | conn_pool->bucket_count = bucket_count; 34 | conn_pool->max_cached = max_cached; 35 | 36 | conn_pool->cache = ngx_pcalloc(pool, sizeof(ngx_queue_t) * bucket_count); 37 | if (conn_pool->cache == NULL) { 38 | return NULL; 39 | } 40 | 41 | conn_pool->free = ngx_pcalloc(pool, sizeof(ngx_queue_t) * bucket_count); 42 | if (conn_pool->free == NULL) { 43 | return NULL; 44 | } 45 | 46 | for (j = 0; j < bucket_count; j++) { 47 | ngx_queue_init(&conn_pool->cache[j]); 48 | ngx_queue_init(&conn_pool->free[j]); 49 | cached = ngx_pcalloc(pool, 50 | sizeof(ngx_http_connection_pool_elt_t) * max_cached); 51 | if (cached == NULL) { 52 | return NULL; 53 | } 54 | 55 | for (k = 0; k < max_cached; k++) { 56 | ngx_queue_insert_head(&conn_pool->free[j], &cached[k].queue); 57 | } 58 | } 59 | 60 | conn_pool->get_peer = ngx_http_connection_pool_get; 61 | conn_pool->free_peer = ngx_http_connection_pool_free; 62 | return conn_pool; 63 | } 64 | 65 | 66 | ngx_int_t 67 | ngx_http_connection_pool_get(ngx_peer_connection_t *pc, void *data) 68 | { 69 | u_char pc_addr[32] = {'\0'}; 70 | ngx_uint_t bucket_id, hash; 71 | ngx_queue_t *q, *cache, *free; 72 | ngx_connection_t *c; 73 | ngx_http_connection_pool_t *p; 74 | ngx_http_connection_pool_elt_t *item; 75 | 76 | p = data; 77 | 78 | #if (NGX_DEBUG) 79 | p->count--; 80 | #endif 81 | 82 | ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0, 83 | "get keepalive peer"); 84 | 85 | p->failed = 0; 86 | 87 | hash = ngx_murmur_hash2((u_char *) pc->sockaddr, pc->socklen); 88 | bucket_id = hash % p->bucket_count; 89 | 90 | cache = &p->cache[bucket_id]; 91 | free = &p->free[bucket_id]; 92 | 93 | ngx_sprintf(pc_addr, "%s:%d", 94 | inet_ntoa(((struct sockaddr_in*)(pc->sockaddr))->sin_addr), 95 | ntohs(((struct sockaddr_in*)(pc->sockaddr))->sin_port)); 96 | 97 | for (q = ngx_queue_head(cache); 98 | q != ngx_queue_sentinel(cache); 99 | q = ngx_queue_next(q)) 100 | { 101 | item = ngx_queue_data(q, ngx_http_connection_pool_elt_t, queue); 102 | c = item->connection; 103 | 104 | if (ngx_memn2cmp((u_char *) &item->sockaddr, (u_char *) pc->sockaddr, 105 | item->socklen, pc->socklen) 106 | == 0) 107 | { 108 | ngx_queue_remove(q); 109 | ngx_queue_insert_head(free, q); 110 | 111 | ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, 112 | "get keepalive peer: using connection %p", c); 113 | 114 | c->idle = 0; 115 | c->log = pc->log; 116 | c->read->log = pc->log; 117 | c->write->log = pc->log; 118 | c->pool->log = pc->log; 119 | 120 | pc->connection = c; 121 | pc->cached = 1; 122 | 123 | item->free = free; 124 | return NGX_DONE; 125 | } 126 | } 127 | 128 | return NGX_OK; 129 | } 130 | 131 | 132 | void 133 | ngx_http_connection_pool_free(ngx_peer_connection_t *pc, 134 | void *data, ngx_uint_t state) 135 | { 136 | ngx_http_connection_pool_t *p = data; 137 | ngx_http_connection_pool_elt_t *item; 138 | 139 | ngx_uint_t hash, bucket_id; 140 | ngx_queue_t *q, *cache, *free; 141 | ngx_connection_t *c; 142 | 143 | ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0, 144 | "free keepalive peer"); 145 | 146 | /* remember failed state - peer.free() may be called more than once */ 147 | 148 | if (state & NGX_PEER_FAILED) { 149 | p->failed = 1; 150 | } 151 | 152 | /* cache valid connections */ 153 | 154 | c = pc->connection; 155 | 156 | if (p->failed 157 | || c == NULL 158 | || c->read->eof 159 | || c->read->error 160 | || c->read->timedout 161 | || c->write->error 162 | || c->write->timedout) 163 | { 164 | return; 165 | } 166 | 167 | if (ngx_handle_read_event(c->read, 0) != NGX_OK) { 168 | return; 169 | } 170 | 171 | #if (NGX_DEBUG) 172 | p->count++; 173 | #endif 174 | ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, 175 | "free keepalive peer: saving connection %p", c); 176 | 177 | hash = ngx_murmur_hash2((u_char *) pc->sockaddr, pc->socklen); 178 | bucket_id = hash % p->bucket_count; 179 | 180 | cache = &p->cache[bucket_id]; 181 | free = &p->free[bucket_id]; 182 | 183 | if (ngx_queue_empty(free)) { 184 | q = ngx_queue_last(cache); 185 | ngx_queue_remove(q); 186 | 187 | item = ngx_queue_data(q, ngx_http_connection_pool_elt_t, queue); 188 | 189 | ngx_http_connection_pool_close(item->connection); 190 | 191 | } else { 192 | q = ngx_queue_head(free); 193 | ngx_queue_remove(q); 194 | 195 | item = ngx_queue_data(q, ngx_http_connection_pool_elt_t, queue); 196 | } 197 | 198 | item->connection = c; 199 | item->free = free; 200 | ngx_queue_insert_head(cache, q); 201 | 202 | pc->connection = NULL; 203 | 204 | if (c->read->timer_set) { 205 | ngx_del_timer(c->read); 206 | } 207 | 208 | if (c->write->timer_set) { 209 | ngx_del_timer(c->write); 210 | } 211 | 212 | c->write->handler = ngx_http_connection_pool_dummy_handler; 213 | c->read->handler = ngx_http_connection_pool_close_handler; 214 | 215 | c->data = item; 216 | c->idle = 1; 217 | c->log = ngx_cycle->log; 218 | c->read->log = ngx_cycle->log; 219 | c->write->log = ngx_cycle->log; 220 | c->pool->log = ngx_cycle->log; 221 | 222 | item->socklen = pc->socklen; 223 | ngx_memcpy(&item->sockaddr, pc->sockaddr, pc->socklen); 224 | 225 | if (c->read->ready) { 226 | ngx_http_connection_pool_close_handler(c->read); 227 | } 228 | } 229 | 230 | 231 | static void 232 | ngx_http_connection_pool_close_handler(ngx_event_t *ev) 233 | { 234 | ngx_http_connection_pool_elt_t *item; 235 | 236 | int n; 237 | char buf[1]; 238 | ngx_connection_t *c; 239 | 240 | ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ev->log, 0, 241 | "keepalive close handler"); 242 | 243 | c = ev->data; 244 | 245 | if (c->close) { 246 | goto close; 247 | } 248 | 249 | n = recv(c->fd, buf, 1, MSG_PEEK); 250 | 251 | if (n == -1 && ngx_socket_errno == NGX_EAGAIN) { 252 | /* stale event */ 253 | 254 | if (ngx_handle_read_event(c->read, 0) != NGX_OK) { 255 | goto close; 256 | } 257 | 258 | return; 259 | } 260 | 261 | close: 262 | 263 | item = c->data; 264 | 265 | 266 | ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ev->log, 0, 267 | "connection pool close connection"); 268 | 269 | ngx_http_connection_pool_close(c); 270 | 271 | ngx_queue_remove(&item->queue); 272 | ngx_queue_insert_head(item->free, &item->queue); 273 | } 274 | 275 | 276 | static void 277 | ngx_http_connection_pool_close(ngx_connection_t *c) 278 | { 279 | 280 | #if (NGX_HTTP_SSL) 281 | 282 | if (c->ssl) { 283 | c->ssl->no_wait_shutdown = 1; 284 | c->ssl->no_send_shutdown = 1; 285 | 286 | if (ngx_ssl_shutdown(c) == NGX_AGAIN) { 287 | c->ssl->handler = ngx_http_connection_pool_close; 288 | return; 289 | } 290 | } 291 | 292 | #endif 293 | 294 | ngx_destroy_pool(c->pool); 295 | ngx_close_connection(c); 296 | } 297 | 298 | 299 | static void 300 | ngx_http_connection_pool_dummy_handler(ngx_event_t *ev) 301 | { 302 | ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ev->log, 0, 303 | "keepalive dummy handler"); 304 | } 305 | 306 | #if (NGX_DEBUG) 307 | void 308 | ngx_http_connection_pool_check(ngx_http_connection_pool_t *conn_pool, 309 | ngx_log_t *log) 310 | { 311 | if (conn_pool->count != 0) { 312 | ngx_log_error(NGX_LOG_ERR, log, 0, 313 | "<== conn pool check ==> " 314 | "some keepalive peer do not free!, conn_pool count: %i", 315 | conn_pool->count); 316 | 317 | } else { 318 | ngx_log_debug0(NGX_LOG_DEBUG_HTTP, log, 0, 319 | "<== conn pool check ==> all keepalive peers are free"); 320 | } 321 | } 322 | #endif 323 | -------------------------------------------------------------------------------- /src/ngx_http_connection_pool.h: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #ifndef _NGX_HTTP_CONNECTION_POOL_H_INCLUDED_ 8 | #define _NGX_HTTP_CONNECTION_POOL_H_INCLUDED_ 9 | 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | 16 | typedef struct ngx_http_connection_pool_s ngx_http_connection_pool_t; 17 | 18 | 19 | typedef struct { 20 | ngx_queue_t queue; 21 | ngx_connection_t *connection; 22 | socklen_t socklen; 23 | u_char sockaddr[NGX_SOCKADDRLEN]; 24 | ngx_queue_t *free; 25 | } ngx_http_connection_pool_elt_t; 26 | 27 | 28 | struct ngx_http_connection_pool_s { 29 | ngx_queue_t *cache; 30 | ngx_queue_t *free; 31 | ngx_uint_t max_cached; 32 | ngx_uint_t bucket_count; 33 | 34 | ngx_uint_t failed; /* unsigned:1 */ 35 | ngx_pool_t *pool; 36 | #if (NGX_DEBUG) 37 | ngx_int_t count; /* check get&free op pairs */ 38 | #endif 39 | 40 | ngx_event_get_peer_pt get_peer; 41 | ngx_event_free_peer_pt free_peer; 42 | }; 43 | 44 | 45 | ngx_http_connection_pool_t *ngx_http_connection_pool_init(ngx_pool_t *pool, 46 | ngx_uint_t max_count, ngx_uint_t bucket_count); 47 | 48 | #if (NGX_DEBUG) 49 | void ngx_http_connection_pool_check(ngx_http_connection_pool_t *coon_pool, 50 | ngx_log_t *log); 51 | #endif 52 | 53 | #endif /* _NGX_HTTP_CONNECTION_POOL_H_INCLUDED_ */ 54 | -------------------------------------------------------------------------------- /src/ngx_http_tfs.h: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #ifndef _NGX_HTTP_TFS_H_INCLUDED_ 8 | #define _NGX_HTTP_TFS_H_INCLUDED_ 9 | 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | 27 | typedef ngx_table_elt_t *(*ngx_http_tfs_create_header_pt)(ngx_http_request_t *r); 28 | 29 | 30 | typedef struct { 31 | ngx_str_t state_msg; 32 | ngx_int_t state; 33 | } ngx_http_tfs_state_t; 34 | 35 | 36 | typedef struct { 37 | uint32_t ds_count; 38 | ngx_http_tfs_inet_t *ds_addrs; 39 | int32_t version; 40 | int32_t lease_id; 41 | } ngx_http_tfs_block_info_t; 42 | 43 | 44 | typedef enum { 45 | NGX_HTTP_TFS_FROM_NONE = 0, 46 | NGX_HTTP_TFS_FROM_CACHE, 47 | NGX_HTTP_TFS_FROM_NS, 48 | } ngx_http_tfs_block_info_src_e; 49 | 50 | 51 | typedef struct { 52 | uint32_t block_id; 53 | uint64_t file_id; 54 | int64_t offset; /* offset in the file */ 55 | uint32_t size; 56 | uint32_t crc; 57 | } NGX_PACKED ngx_http_tfs_segment_info_t; 58 | 59 | 60 | typedef struct { 61 | u_char file_name[NGX_HTTP_TFS_FILE_NAME_LEN]; 62 | int64_t offset; 63 | uint32_t size; 64 | uint32_t crc; 65 | } NGX_PACKED ngx_http_tfs_tmp_segment_info_t; 66 | 67 | 68 | struct ngx_http_tfs_segment_data_s { 69 | uint8_t cache_hit; 70 | ngx_http_tfs_block_info_src_e block_info_src; 71 | ngx_http_tfs_segment_info_t segment_info; 72 | /* read/write offset inside this segment */ 73 | uint32_t oper_offset; 74 | /* read/write size inside this segment */ 75 | uint32_t oper_size; 76 | union { 77 | uint64_t write_file_number; 78 | }; 79 | ngx_http_tfs_block_info_t block_info; 80 | ngx_uint_t ds_retry; 81 | ngx_uint_t ds_index; 82 | ngx_chain_t *data; 83 | } NGX_PACKED; 84 | 85 | 86 | typedef struct { 87 | 88 | uint8_t still_have; /* for custom file */ 89 | uint32_t cluster_id; 90 | uint32_t open_mode; 91 | /* not for large_file's data */ 92 | int64_t file_offset; 93 | uint64_t left_length; 94 | uint64_t file_hole_size; 95 | uint32_t last_write_segment_index; 96 | uint32_t segment_index; 97 | uint32_t segment_count; 98 | ngx_http_tfs_segment_data_t *segment_data; 99 | } NGX_PACKED ngx_http_tfs_file_t; 100 | 101 | 102 | struct ngx_http_tfs_upstream_s { 103 | ngx_str_t lock_file; 104 | ngx_msec_t rcs_interval; 105 | 106 | ngx_str_t rcs_zone_name; 107 | ngx_shm_zone_t *rcs_shm_zone; 108 | ngx_http_tfs_rc_ctx_t *rc_ctx; 109 | 110 | /* upstream name and port */ 111 | in_port_t port; 112 | ngx_str_t host; 113 | ngx_addr_t *ups_addr; 114 | 115 | struct sockaddr_in local_addr; 116 | u_char local_addr_text[NGX_INET_ADDRSTRLEN]; 117 | 118 | ngx_flag_t enable_rcs; 119 | 120 | ngx_http_tfs_timers_data_t *timer_data; 121 | 122 | unsigned used:1; 123 | }; 124 | 125 | 126 | struct ngx_http_tfs_loc_conf_s { 127 | ngx_msec_t timeout; 128 | 129 | size_t max_temp_file_size; 130 | size_t temp_file_write_size; 131 | 132 | size_t busy_buffers_size_conf; 133 | 134 | uint64_t meta_root_server; 135 | ngx_http_tfs_meta_table_t meta_server_table; 136 | 137 | ngx_http_tfs_upstream_t *upstream; 138 | }; 139 | 140 | 141 | typedef struct { 142 | 143 | ngx_log_t *log; 144 | } ngx_http_tfs_srv_conf_t; 145 | 146 | 147 | struct ngx_http_tfs_main_conf_s { 148 | ngx_msec_t tfs_connect_timeout; 149 | ngx_msec_t tfs_send_timeout; 150 | ngx_msec_t tfs_read_timeout; 151 | 152 | ngx_msec_t tair_timeout; 153 | ngx_http_tfs_tair_instance_t dup_instances[NGX_HTTP_TFS_MAX_CLUSTER_COUNT]; 154 | 155 | size_t send_lowat; 156 | size_t buffer_size; 157 | size_t body_buffer_size; 158 | size_t busy_buffers_size; 159 | 160 | ngx_shm_zone_t *block_cache_shm_zone; 161 | 162 | ngx_flag_t enable_remote_block_cache; 163 | ngx_http_tfs_tair_instance_t remote_block_cache_instance; 164 | ngx_http_tfs_local_block_cache_ctx_t *local_block_cache_ctx; 165 | 166 | ngx_http_connection_pool_t *conn_pool; 167 | 168 | uint32_t cluster_id; 169 | 170 | ngx_array_t upstreams; 171 | }; 172 | 173 | 174 | typedef ngx_int_t (*tfs_peer_handler_pt)(ngx_http_tfs_t *t); 175 | typedef void (*ngx_http_tfs_handler_pt)(ngx_http_request_t *r, 176 | ngx_http_tfs_t *t); 177 | typedef ngx_int_t (*ngx_http_tfs_sub_process_pt)(ngx_http_tfs_t *t); 178 | 179 | 180 | typedef struct { 181 | ngx_list_t headers; 182 | 183 | ngx_uint_t status_n; 184 | ngx_str_t status_line; 185 | 186 | ngx_table_elt_t *status; 187 | ngx_table_elt_t *date; 188 | ngx_table_elt_t *server; 189 | ngx_table_elt_t *connection; 190 | 191 | ngx_table_elt_t *expires; 192 | ngx_table_elt_t *etag; 193 | ngx_table_elt_t *x_accel_expires; 194 | ngx_table_elt_t *x_accel_redirect; 195 | ngx_table_elt_t *x_accel_limit_rate; 196 | 197 | ngx_table_elt_t *content_type; 198 | ngx_table_elt_t *content_length; 199 | 200 | ngx_table_elt_t *last_modified; 201 | ngx_table_elt_t *location; 202 | ngx_table_elt_t *accept_ranges; 203 | ngx_table_elt_t *www_authenticate; 204 | 205 | #if (NGX_HTTP_GZIP) 206 | ngx_table_elt_t *content_encoding; 207 | #endif 208 | 209 | off_t content_length_n; 210 | 211 | ngx_array_t cache_control; 212 | } ngx_http_tfs_headers_in_t; 213 | 214 | 215 | struct ngx_http_tfs_s { 216 | ngx_http_tfs_handler_pt read_event_handler; 217 | ngx_http_tfs_handler_pt write_event_handler; 218 | 219 | ngx_http_tfs_peer_connection_t *tfs_peer; 220 | ngx_http_tfs_peer_connection_t *tfs_peer_servers; 221 | uint8_t tfs_peer_count; 222 | 223 | ngx_http_tfs_loc_conf_t *loc_conf; 224 | ngx_http_tfs_srv_conf_t *srv_conf; 225 | ngx_http_tfs_main_conf_t *main_conf; 226 | 227 | ngx_http_tfs_restful_ctx_t r_ctx; 228 | 229 | ngx_output_chain_ctx_t output; 230 | ngx_chain_writer_ctx_t writer; 231 | 232 | ngx_chain_t *request_bufs; 233 | ngx_chain_t *send_body; 234 | ngx_pool_t *pool; 235 | 236 | ngx_buf_t header_buffer; 237 | 238 | ngx_chain_t *recv_chain; 239 | 240 | ngx_chain_t *out_bufs; 241 | ngx_chain_t *busy_bufs; 242 | ngx_chain_t *free_bufs; 243 | 244 | ngx_http_tfs_rcs_info_t *rc_info_node; 245 | ngx_rbtree_node_t *node; 246 | 247 | ngx_uint_t logical_cluster_index; 248 | ngx_uint_t rw_cluster_index; 249 | 250 | /* keep alive */ 251 | ngx_queue_t *curr_ka_queue; 252 | 253 | /* header pointer */ 254 | void *header; 255 | ngx_int_t header_size; 256 | 257 | tfs_peer_handler_pt create_request; 258 | tfs_peer_handler_pt input_filter; 259 | tfs_peer_handler_pt retry_handler; 260 | tfs_peer_handler_pt process_request_body; 261 | tfs_peer_handler_pt finalize_request; 262 | 263 | void *finalize_data; 264 | void *data; 265 | 266 | ngx_int_t request_sent; 267 | ngx_uint_t sent_size; 268 | off_t length; 269 | 270 | ngx_log_t *log; 271 | 272 | ngx_int_t parse_state; 273 | 274 | /* final file name */ 275 | ngx_str_t file_name; 276 | 277 | ngx_int_t state; 278 | 279 | /* custom file */ 280 | ngx_http_tfs_custom_meta_info_t meta_info; 281 | ngx_str_t last_file_path; 282 | int64_t last_file_pid; 283 | uint8_t last_file_type; 284 | ngx_int_t *dir_lens; 285 | ngx_int_t last_dir_level; 286 | uint16_t orig_action; 287 | ngx_array_t file_holes; 288 | 289 | ngx_http_tfs_headers_in_t headers_in; 290 | 291 | /* delete */ 292 | ngx_int_t group_count; 293 | ngx_int_t group_seq; 294 | 295 | /* name ip */ 296 | ngx_http_tfs_inet_t name_server_addr; 297 | ngx_str_t name_server_addr_text; 298 | 299 | ngx_http_tfs_json_gen_t *json_output; 300 | 301 | ngx_uint_t status; 302 | ngx_str_t status_line; 303 | ngx_int_t tfs_status; 304 | 305 | uint64_t output_size; 306 | 307 | /* de-duplicate info */ 308 | ngx_http_tfs_dedup_ctx_t dedup_ctx; 309 | 310 | /* stat info */ 311 | ngx_http_tfs_stat_info_t stat_info; 312 | 313 | /* file info */ 314 | ngx_chain_t *meta_segment_data; 315 | ngx_http_tfs_file_t file; 316 | ngx_http_tfs_segment_head_t *seg_head; 317 | ngx_http_tfs_raw_file_info_t file_info; 318 | ngx_buf_t *readv2_rsp_tail_buf; 319 | uint8_t read_ver; 320 | 321 | /* block cache */ 322 | ngx_http_tfs_block_cache_ctx_t block_cache_ctx; 323 | 324 | /* for parallel write segments */ 325 | ngx_http_tfs_t *parent; 326 | ngx_http_tfs_t *next; 327 | ngx_http_tfs_t *free_sts; 328 | ngx_http_tfs_sub_process_pt sp_callback; 329 | uint32_t sp_count; 330 | uint32_t sp_done_count; 331 | uint32_t sp_fail_count; 332 | uint32_t sp_succ_count; 333 | uint32_t sp_curr; 334 | unsigned sp_ready:1; 335 | 336 | unsigned header_only:1; 337 | unsigned header_sent:1; 338 | unsigned has_split_frag:1; 339 | /* for custrom file read */ 340 | unsigned is_first_segment:1; 341 | 342 | unsigned use_dedup:1; 343 | unsigned is_stat_dup_file:1; 344 | unsigned is_large_file:1; 345 | unsigned is_process_meta_seg:1; 346 | unsigned retry_curr_ns:1; 347 | }; 348 | 349 | 350 | ngx_int_t ngx_http_tfs_init(ngx_http_tfs_t *t); 351 | void ngx_http_tfs_finalize_request(ngx_http_request_t *r, 352 | ngx_http_tfs_t *t, ngx_int_t rc); 353 | void ngx_http_tfs_finalize_state(ngx_http_tfs_t *t, ngx_int_t rc); 354 | ngx_int_t ngx_http_tfs_reinit(ngx_http_request_t *r, ngx_http_tfs_t *t); 355 | ngx_int_t ngx_http_tfs_connect(ngx_http_tfs_t *t); 356 | ngx_int_t ngx_http_tfs_lookup_block_cache(ngx_http_tfs_t *t, 357 | ngx_http_tfs_segment_data_t *segment_data); 358 | ngx_int_t ngx_http_tfs_batch_lookup_block_cache(ngx_http_tfs_t *t); 359 | void ngx_http_tfs_remove_block_cache(ngx_http_tfs_t *t, 360 | ngx_http_tfs_segment_data_t *segment_data); 361 | ngx_int_t ngx_http_tfs_set_output_appid(ngx_http_tfs_t *t, uint64_t app_id); 362 | void ngx_http_tfs_set_custom_initial_parameters(ngx_http_tfs_t *t); 363 | ngx_int_t ngx_http_tfs_misc_ctx_init(ngx_http_tfs_t *t, 364 | ngx_http_tfs_rcs_info_t *rc_node); 365 | ngx_int_t ngx_http_tfs_batch_process_start(ngx_http_tfs_t *t); 366 | ngx_int_t ngx_http_tfs_batch_process_end(ngx_http_tfs_t *t); 367 | ngx_int_t ngx_http_tfs_batch_process_next(ngx_http_tfs_t *t); 368 | 369 | 370 | #endif /* _NGX_TFS_H_INCLUDED_ */ 371 | 372 | -------------------------------------------------------------------------------- /src/ngx_http_tfs_block_cache.c: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | 13 | ngx_int_t 14 | ngx_http_tfs_block_cache_lookup(ngx_http_tfs_block_cache_ctx_t *ctx, 15 | ngx_pool_t *pool, ngx_log_t *log, ngx_http_tfs_block_cache_key_t *key, 16 | ngx_http_tfs_block_cache_value_t *value) 17 | { 18 | ngx_int_t rc = NGX_DECLINED; 19 | 20 | if (ctx->curr_lookup_cache == NGX_HTTP_TFS_LOCAL_BLOCK_CACHE) { 21 | 22 | ctx->curr_lookup_cache = NGX_HTTP_TFS_REMOTE_BLOCK_CACHE; 23 | 24 | if (ctx->use_cache & NGX_HTTP_TFS_LOCAL_BLOCK_CACHE) { 25 | rc = ngx_http_tfs_local_block_cache_lookup(ctx->local_ctx, 26 | pool, log, key, value); 27 | 28 | if (rc == NGX_OK) { 29 | return rc; 30 | } 31 | } 32 | } 33 | 34 | if (ctx->curr_lookup_cache == NGX_HTTP_TFS_REMOTE_BLOCK_CACHE) { 35 | 36 | ctx->curr_lookup_cache = NGX_HTTP_TFS_NO_BLOCK_CACHE; 37 | 38 | if (ctx->use_cache & NGX_HTTP_TFS_REMOTE_BLOCK_CACHE) { 39 | rc = ngx_http_tfs_remote_block_cache_lookup(&ctx->remote_ctx, 40 | pool, log, key); 41 | } 42 | } 43 | 44 | return rc; 45 | } 46 | 47 | 48 | void 49 | ngx_http_tfs_block_cache_insert(ngx_http_tfs_block_cache_ctx_t *ctx, 50 | ngx_pool_t *pool, ngx_log_t *log, ngx_http_tfs_block_cache_key_t *key, 51 | ngx_http_tfs_block_cache_value_t *value) 52 | { 53 | if (ctx->use_cache & NGX_HTTP_TFS_REMOTE_BLOCK_CACHE) { 54 | ngx_http_tfs_remote_block_cache_insert(&ctx->remote_ctx, 55 | pool, log, key, value); 56 | } 57 | 58 | if (ctx->use_cache & NGX_HTTP_TFS_LOCAL_BLOCK_CACHE) { 59 | ngx_http_tfs_local_block_cache_insert(ctx->local_ctx, log, key, value); 60 | } 61 | } 62 | 63 | 64 | void 65 | ngx_http_tfs_block_cache_remove(ngx_http_tfs_block_cache_ctx_t *ctx, 66 | ngx_pool_t *pool, ngx_log_t *log, ngx_http_tfs_block_cache_key_t* key, 67 | uint8_t hit_status) 68 | { 69 | if (hit_status != NGX_HTTP_TFS_NO_BLOCK_CACHE 70 | && (ctx->use_cache & NGX_HTTP_TFS_LOCAL_BLOCK_CACHE)) 71 | { 72 | ngx_http_tfs_local_block_cache_remove(ctx->local_ctx, log, key); 73 | } 74 | 75 | if (hit_status == NGX_HTTP_TFS_REMOTE_BLOCK_CACHE) { 76 | ngx_http_tfs_remote_block_cache_remove(&ctx->remote_ctx, 77 | pool, log, key); 78 | } 79 | } 80 | 81 | 82 | ngx_int_t 83 | ngx_http_tfs_block_cache_batch_lookup(ngx_http_tfs_block_cache_ctx_t *ctx, 84 | ngx_pool_t *pool, ngx_log_t *log, ngx_array_t *keys, 85 | ngx_array_t *kvs) 86 | { 87 | uint32_t i; 88 | ngx_int_t rc; 89 | ngx_uint_t local_miss_count; 90 | ngx_array_t local_miss_keys; 91 | ngx_http_tfs_t *t; 92 | ngx_http_tfs_segment_data_t *segment_data; 93 | ngx_http_tfs_block_cache_key_t *key; 94 | 95 | rc = NGX_DECLINED; 96 | 97 | if (ctx->curr_lookup_cache == NGX_HTTP_TFS_LOCAL_BLOCK_CACHE) { 98 | 99 | ctx->curr_lookup_cache = NGX_HTTP_TFS_REMOTE_BLOCK_CACHE; 100 | 101 | if (ctx->use_cache & NGX_HTTP_TFS_LOCAL_BLOCK_CACHE) { 102 | rc = ngx_http_tfs_local_block_cache_batch_lookup(ctx->local_ctx, 103 | pool, log, keys, 104 | kvs); 105 | 106 | if (rc == NGX_OK || rc == NGX_ERROR) { 107 | return rc; 108 | } 109 | } 110 | } 111 | 112 | /* rc == NGX_DECLIEND */ 113 | if (ctx->curr_lookup_cache == NGX_HTTP_TFS_REMOTE_BLOCK_CACHE) { 114 | 115 | ctx->curr_lookup_cache = NGX_HTTP_TFS_NO_BLOCK_CACHE; 116 | 117 | if (ctx->use_cache & NGX_HTTP_TFS_REMOTE_BLOCK_CACHE) { 118 | t = ctx->remote_ctx.data; 119 | local_miss_count = keys->nelts - kvs->nelts; 120 | 121 | rc = ngx_array_init(&local_miss_keys, t->pool, local_miss_count, 122 | sizeof(ngx_http_tfs_block_cache_key_t)); 123 | if (rc == NGX_ERROR) { 124 | return rc; 125 | } 126 | 127 | segment_data = &t->file.segment_data[t->file.segment_index]; 128 | for (i = 0; i < keys->nelts; i++, segment_data++) { 129 | if (segment_data->cache_hit == NGX_HTTP_TFS_NO_BLOCK_CACHE) { 130 | key = (ngx_http_tfs_block_cache_key_t *) 131 | ngx_array_push(&local_miss_keys); 132 | key->ns_addr = *((uint64_t*)(&t->name_server_addr)); 133 | key->block_id = segment_data->segment_info.block_id; 134 | } 135 | } 136 | 137 | rc = ngx_http_tfs_remote_block_cache_batch_lookup(&ctx->remote_ctx, 138 | pool, log, 139 | &local_miss_keys); 140 | } 141 | } 142 | return rc; 143 | } 144 | 145 | 146 | ngx_int_t 147 | ngx_http_tfs_block_cache_cmp(ngx_http_tfs_block_cache_key_t *left, 148 | ngx_http_tfs_block_cache_key_t *right) 149 | { 150 | if (left->ns_addr == right->ns_addr) { 151 | if (left->block_id == right->block_id) { 152 | return 0; 153 | } 154 | if (left->block_id < right->block_id) { 155 | return -1; 156 | } 157 | return 1; 158 | } 159 | if (left->ns_addr < right->ns_addr) { 160 | return -1; 161 | } 162 | return 1; 163 | } 164 | -------------------------------------------------------------------------------- /src/ngx_http_tfs_block_cache.h: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #ifndef _NGX_HTTP_TFS_BLOCK_CACHE_H_INCLUDED_ 8 | #define _NGX_HTTP_TFS_BLOCK_CACHE_H_INCLUDED_ 9 | 10 | 11 | #include 12 | #include 13 | 14 | 15 | #define NGX_HTTP_TFS_BLOCK_CACHE_KEY_SIZE \ 16 | sizeof(ngx_http_tfs_block_cache_key_t) 17 | 18 | #define NGX_HTTP_TFS_REMOTE_BLOCK_CACHE_VALUE_BASE_SIZE sizeof(uint32_t) 19 | 20 | #define NGX_HTTP_TFS_BLOCK_CACHE_DISCARD_ITEM_COUNT 10000 21 | 22 | #define NGX_HTTP_TFS_BLOCK_CACHE_STAT_COUNT (3000 * 60 * 60) 23 | 24 | #define NGX_HTTP_TFS_NO_BLOCK_CACHE 0x0 25 | #define NGX_HTTP_TFS_LOCAL_BLOCK_CACHE 0x1 26 | #define NGX_HTTP_TFS_REMOTE_BLOCK_CACHE 0x2 27 | 28 | 29 | typedef struct { 30 | uint64_t ns_addr; 31 | uint32_t block_id; 32 | } __attribute__ ((__packed__)) ngx_http_tfs_block_cache_key_t; 33 | 34 | 35 | typedef struct { 36 | uint32_t ds_count; 37 | uint64_t *ds_addrs; 38 | } ngx_http_tfs_block_cache_value_t; 39 | 40 | 41 | typedef struct { 42 | ngx_http_tfs_block_cache_key_t *key; 43 | ngx_http_tfs_block_cache_value_t *value; 44 | } ngx_http_tfs_block_cache_kv_t; 45 | 46 | 47 | typedef struct { 48 | ngx_rbtree_t rbtree; 49 | ngx_rbtree_node_t sentinel; 50 | ngx_queue_t queue; 51 | uint64_t discard_item_count; 52 | uint64_t hit_count; 53 | uint64_t miss_count; 54 | } ngx_http_tfs_block_cache_shctx_t; 55 | 56 | 57 | typedef struct { 58 | ngx_http_tfs_block_cache_shctx_t *sh; 59 | ngx_slab_pool_t *shpool; 60 | } ngx_http_tfs_local_block_cache_ctx_t; 61 | 62 | 63 | typedef struct { 64 | void *data; 65 | ngx_http_tfs_tair_instance_t *tair_instance; 66 | } ngx_http_tfs_remote_block_cache_ctx_t; 67 | 68 | 69 | typedef struct { 70 | ngx_http_tfs_local_block_cache_ctx_t *local_ctx; 71 | ngx_http_tfs_remote_block_cache_ctx_t remote_ctx; 72 | uint8_t use_cache; 73 | uint8_t curr_lookup_cache; 74 | } ngx_http_tfs_block_cache_ctx_t; 75 | 76 | 77 | ngx_int_t ngx_http_tfs_block_cache_lookup(ngx_http_tfs_block_cache_ctx_t *ctx, 78 | ngx_pool_t *pool, ngx_log_t *log, ngx_http_tfs_block_cache_key_t* key, 79 | ngx_http_tfs_block_cache_value_t *value); 80 | void ngx_http_tfs_block_cache_insert(ngx_http_tfs_block_cache_ctx_t *ctx, 81 | ngx_pool_t *pool, ngx_log_t *log, ngx_http_tfs_block_cache_key_t *key, 82 | ngx_http_tfs_block_cache_value_t *value); 83 | void ngx_http_tfs_block_cache_remove(ngx_http_tfs_block_cache_ctx_t *ctx, 84 | ngx_pool_t *pool, ngx_log_t *log, ngx_http_tfs_block_cache_key_t *key, 85 | uint8_t hit_status); 86 | ngx_int_t ngx_http_tfs_block_cache_batch_lookup( 87 | ngx_http_tfs_block_cache_ctx_t *ctx, 88 | ngx_pool_t *pool, ngx_log_t *log, ngx_array_t* keys, ngx_array_t *kvs); 89 | void ngx_http_tfs_block_cache_batch_insert(ngx_http_tfs_block_cache_ctx_t *ctx, 90 | ngx_pool_t *pool, ngx_log_t *log, ngx_array_t* kvs); 91 | ngx_int_t ngx_http_tfs_block_cache_cmp(ngx_http_tfs_block_cache_key_t *left, 92 | ngx_http_tfs_block_cache_key_t *right); 93 | 94 | 95 | #endif /* _NGX_HTTP_TFS_BLOCK_CACHE_H_INCLUDED_ */ 96 | -------------------------------------------------------------------------------- /src/ngx_http_tfs_data_server_message.h: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #ifndef _NGX_HTTP_TFS_DATA_SERVER_MESSAGE_H_INCLUDED_ 8 | #define _NGX_HTTP_TFS_DATA_SERVER_MESSAGE_H_INCLUDED_ 9 | 10 | 11 | #include 12 | 13 | 14 | ngx_chain_t *ngx_http_tfs_data_server_create_message(ngx_http_tfs_t *t); 15 | ngx_int_t ngx_http_tfs_data_server_parse_message(ngx_http_tfs_t *t); 16 | ngx_http_tfs_inet_t *ngx_http_tfs_select_data_server(ngx_http_tfs_t *t, 17 | ngx_http_tfs_segment_data_t *segment_data); 18 | 19 | ngx_int_t ngx_http_tfs_get_meta_segment(ngx_http_tfs_t *t); 20 | ngx_int_t ngx_http_tfs_set_meta_segment_data(ngx_http_tfs_t *t); 21 | ngx_int_t ngx_http_tfs_parse_meta_segment(ngx_http_tfs_t *t, ngx_chain_t *data); 22 | ngx_int_t ngx_http_tfs_get_segment_for_write(ngx_http_tfs_t *t); 23 | 24 | ngx_int_t ngx_http_tfs_get_segment_for_read(ngx_http_tfs_t *t); 25 | ngx_int_t ngx_http_tfs_get_segment_for_delete(ngx_http_tfs_t *t); 26 | 27 | ngx_int_t ngx_http_tfs_fill_file_hole(ngx_http_tfs_t *t, size_t file_hole_size); 28 | 29 | 30 | #endif /* _NGX_HTTP_TFS_DATA_SERVER_MESSAGE_H_INCLUDED_ */ 31 | -------------------------------------------------------------------------------- /src/ngx_http_tfs_duplicate.c: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | static void ngx_http_tfs_duplicate_get_handler(ngx_http_tair_key_value_t *kv, 13 | ngx_int_t rc, void *data); 14 | static void ngx_http_tfs_duplicate_put_handler(ngx_int_t rc, void *data); 15 | static void ngx_http_tfs_duplicate_delete_handler(ngx_int_t rc, void *data); 16 | static void ngx_http_tfs_duplicate_callback(ngx_http_tfs_dedup_ctx_t *ctx, 17 | ngx_int_t rc); 18 | 19 | ngx_int_t ngx_http_tfs_duplicate_check_suffix(ngx_str_t *tfs_name, 20 | ngx_str_t *suffix); 21 | ngx_int_t ngx_http_tfs_duplicate_check_filename(ngx_str_t *dup_name, 22 | ngx_http_tfs_raw_fsname_t* fsname); 23 | 24 | 25 | ngx_int_t 26 | ngx_http_tfs_get_duplicate_info(ngx_http_tfs_dedup_ctx_t *ctx, 27 | ngx_pool_t *pool, ngx_log_t * log, ngx_chain_t *data) 28 | { 29 | u_char *p; 30 | ssize_t data_len; 31 | ngx_int_t rc; 32 | ngx_http_tair_data_t tair_key; 33 | 34 | data_len = 0; 35 | 36 | rc = ngx_http_tfs_sum_md5(data, ctx->tair_key, &data_len, log); 37 | if (rc == NGX_ERROR) { 38 | return NGX_ERROR; 39 | } 40 | 41 | p = ctx->tair_key; 42 | p += NGX_HTTP_TFS_MD5_RESULT_LEN; 43 | 44 | *(uint32_t *) p = htonl(data_len); 45 | 46 | tair_key.type = NGX_HTTP_TAIR_BYTEARRAY; 47 | tair_key.data = ctx->tair_key; 48 | tair_key.len = NGX_HTTP_TFS_DUPLICATE_KEY_SIZE; 49 | 50 | ctx->md5_sumed = 1; 51 | 52 | rc = ngx_http_tfs_tair_get_helper(ctx->tair_instance, pool, log, 53 | &tair_key, 54 | ngx_http_tfs_duplicate_get_handler, 55 | ctx); 56 | 57 | return rc; 58 | } 59 | 60 | 61 | static void 62 | ngx_http_tfs_duplicate_get_handler(ngx_http_tair_key_value_t *kv, ngx_int_t rc, 63 | void *data) 64 | { 65 | u_char *p; 66 | ngx_http_tfs_t *t; 67 | ngx_http_tfs_dedup_ctx_t *ctx; 68 | 69 | ctx = data; 70 | t = ctx->data; 71 | 72 | if (rc == NGX_HTTP_ETAIR_SUCCESS) { 73 | p = kv->value->data; 74 | if (p != NULL 75 | && (kv->value->len > NGX_HTTP_TFS_DUPLICATE_VALUE_BASE_SIZE)) 76 | { 77 | ctx->file_ref_count = *(int32_t *)p; 78 | p += sizeof(int32_t); 79 | ctx->dup_file_name.len = kv->value->len - sizeof(int32_t); 80 | ctx->dup_file_name.data = ngx_pcalloc(t->pool, 81 | ctx->dup_file_name.len); 82 | if (ctx->dup_file_name.data == NULL) { 83 | rc = NGX_ERROR; 84 | 85 | } else { 86 | ngx_memcpy(ctx->dup_file_name.data, p, ctx->dup_file_name.len); 87 | rc = NGX_OK; 88 | } 89 | ctx->dup_version = kv->version; 90 | 91 | } else { 92 | rc = NGX_ERROR; 93 | } 94 | ngx_log_debug3(NGX_LOG_DEBUG_HTTP, t->log, 0, 95 | "get duplicate info: " 96 | "file name: %V, file ref count: %d, dup_version: %d", 97 | &ctx->dup_file_name, 98 | ctx->file_ref_count, 99 | ctx->dup_version); 100 | 101 | } else { 102 | rc = NGX_ERROR; 103 | ctx->dup_version = NGX_HTTP_TFS_DUPLICATE_INITIAL_MAGIC_VERSION; 104 | } 105 | ngx_http_tfs_duplicate_callback(ctx, rc); 106 | } 107 | 108 | 109 | ngx_int_t 110 | ngx_http_tfs_set_duplicate_info(ngx_http_tfs_dedup_ctx_t *ctx, 111 | ngx_pool_t *pool, ngx_log_t * log, ngx_chain_t *data) 112 | { 113 | u_char *p; 114 | ssize_t data_len; 115 | ngx_int_t rc; 116 | ngx_http_tair_data_t tair_key; 117 | ngx_http_tair_data_t tair_value; 118 | 119 | data_len = 0; 120 | 121 | if (!ctx->md5_sumed) { 122 | rc = ngx_http_tfs_sum_md5(data, ctx->tair_key, &data_len, log); 123 | if (rc == NGX_ERROR) { 124 | return NGX_ERROR; 125 | } 126 | 127 | p = ctx->tair_key; 128 | p += NGX_HTTP_TFS_MD5_RESULT_LEN; 129 | 130 | *(uint32_t *) p = htonl(data_len); 131 | ctx->md5_sumed = 1; 132 | } 133 | 134 | tair_key.len = NGX_HTTP_TFS_DUPLICATE_KEY_SIZE; 135 | tair_key.data = ctx->tair_key; 136 | tair_key.type = NGX_HTTP_TAIR_BYTEARRAY; 137 | 138 | tair_value.len = 139 | NGX_HTTP_TFS_DUPLICATE_VALUE_BASE_SIZE + ctx->dup_file_name.len; 140 | tair_value.data = ngx_pcalloc(pool, tair_value.len); 141 | if (tair_value.data == NULL) { 142 | return NGX_ERROR; 143 | } 144 | ngx_memcpy(tair_value.data, &ctx->file_ref_count, 145 | NGX_HTTP_TFS_DUPLICATE_VALUE_BASE_SIZE); 146 | ngx_memcpy(tair_value.data + NGX_HTTP_TFS_DUPLICATE_VALUE_BASE_SIZE, 147 | ctx->dup_file_name.data, ctx->dup_file_name.len); 148 | tair_value.type = NGX_HTTP_TAIR_BYTEARRAY; 149 | ngx_log_debug3(NGX_LOG_DEBUG_HTTP, log, 0, 150 | "set duplicate info: " 151 | "file name: %V, file ref count: %d, dup_version: %d", 152 | &ctx->dup_file_name, 153 | ctx->file_ref_count, 154 | ctx->dup_version); 155 | 156 | rc = ngx_http_tfs_tair_put_helper(ctx->tair_instance, pool, log, 157 | &tair_key, &tair_value, 0/*expire*/, 158 | ctx->dup_version, 159 | ngx_http_tfs_duplicate_put_handler, ctx); 160 | 161 | return rc; 162 | } 163 | 164 | 165 | static void 166 | ngx_http_tfs_duplicate_put_handler(ngx_int_t rc, void *data) 167 | { 168 | ngx_http_tfs_dedup_ctx_t *ctx; 169 | 170 | ctx = data; 171 | 172 | if (rc == NGX_HTTP_ETAIR_SUCCESS) { 173 | rc = NGX_OK; 174 | 175 | } else { 176 | rc = NGX_ERROR; 177 | } 178 | ngx_http_tfs_duplicate_callback(ctx, rc); 179 | } 180 | 181 | 182 | ngx_int_t 183 | ngx_http_tfs_delete_duplicate_info(ngx_http_tfs_dedup_ctx_t *ctx, 184 | ngx_pool_t *pool, ngx_log_t * log, ngx_chain_t *data) 185 | { 186 | u_char *p; 187 | ssize_t data_len; 188 | ngx_int_t rc; 189 | ngx_array_t tair_keys; 190 | ngx_http_tair_data_t *tair_key; 191 | 192 | data_len = 0; 193 | 194 | if (!ctx->md5_sumed) { 195 | rc = ngx_http_tfs_sum_md5(data, ctx->tair_key, &data_len, log); 196 | if (rc == NGX_ERROR) { 197 | return NGX_ERROR; 198 | } 199 | 200 | p = ctx->tair_key; 201 | p += NGX_HTTP_TFS_MD5_RESULT_LEN; 202 | 203 | *(uint32_t *) p = htonl(data_len); 204 | ctx->md5_sumed = 1; 205 | } 206 | 207 | rc = ngx_array_init(&tair_keys, pool, 1, sizeof(ngx_http_tair_data_t)); 208 | if (rc == NGX_ERROR) { 209 | return rc; 210 | } 211 | tair_key = (ngx_http_tair_data_t*) ngx_array_push(&tair_keys); 212 | 213 | tair_key->type = NGX_HTTP_TAIR_BYTEARRAY; 214 | tair_key->data = ctx->tair_key; 215 | tair_key->len = NGX_HTTP_TFS_DUPLICATE_KEY_SIZE; 216 | 217 | rc = ngx_http_tfs_tair_delete_helper(ctx->tair_instance, pool, log, 218 | &tair_keys, 219 | ngx_http_tfs_duplicate_delete_handler, 220 | ctx); 221 | 222 | return rc; 223 | } 224 | 225 | 226 | static void 227 | ngx_http_tfs_duplicate_delete_handler(ngx_int_t rc, void *data) 228 | { 229 | ngx_http_tfs_dedup_ctx_t *ctx; 230 | 231 | ctx = data; 232 | 233 | if (rc == NGX_HTTP_ETAIR_SUCCESS) { 234 | rc = NGX_OK; 235 | 236 | } else { 237 | rc = NGX_ERROR; 238 | } 239 | ngx_http_tfs_duplicate_callback(ctx, rc); 240 | } 241 | 242 | 243 | ngx_int_t 244 | ngx_http_tfs_duplicate_check_suffix(ngx_str_t *tfs_name, ngx_str_t *suffix) 245 | { 246 | ngx_int_t rc; 247 | 248 | rc = NGX_ERROR; 249 | if ((tfs_name->len == NGX_HTTP_TFS_FILE_NAME_LEN && suffix->len == 0) 250 | || (tfs_name->len > NGX_HTTP_TFS_FILE_NAME_LEN && suffix->len > 0 251 | && ((tfs_name->len - NGX_HTTP_TFS_FILE_NAME_LEN) == suffix->len) 252 | && (ngx_strncmp(suffix->data, 253 | tfs_name->data + NGX_HTTP_TFS_FILE_NAME_LEN, 254 | suffix->len) == 0))) 255 | { 256 | rc = NGX_OK; 257 | } 258 | 259 | return rc; 260 | } 261 | 262 | 263 | ngx_int_t 264 | ngx_http_tfs_duplicate_check_filename(ngx_str_t *dup_file_name, 265 | ngx_http_tfs_raw_fsname_t* fsname) 266 | { 267 | ngx_int_t rc; 268 | ngx_str_t dup_file_suffix = ngx_null_string; 269 | ngx_http_tfs_raw_fsname_t dup_fsname; 270 | 271 | rc = ngx_http_tfs_raw_fsname_parse(dup_file_name, &dup_file_suffix, 272 | &dup_fsname); 273 | if (rc == NGX_OK) { 274 | if (fsname->cluster_id == dup_fsname.cluster_id 275 | && fsname->file.block_id == dup_fsname.file.block_id 276 | && fsname->file.seq_id == dup_fsname.file.seq_id 277 | && fsname->file.suffix == dup_fsname.file.suffix) 278 | { 279 | return NGX_OK; 280 | } 281 | } 282 | 283 | return NGX_ERROR; 284 | } 285 | 286 | 287 | static void 288 | ngx_http_tfs_duplicate_callback(ngx_http_tfs_dedup_ctx_t *ctx, ngx_int_t rc) 289 | { 290 | ngx_http_tfs_t *t; 291 | ngx_http_tfs_rcs_info_t *rc_info; 292 | 293 | t = ctx->data; 294 | rc_info = t->rc_info_node; 295 | 296 | switch (t->r_ctx.action.code) { 297 | case NGX_HTTP_TFS_ACTION_REMOVE_FILE: 298 | switch(t->state) { 299 | case NGX_HTTP_TFS_STATE_REMOVE_READ_META_SEGMENT: 300 | /* exist in tair */ 301 | if (rc == NGX_OK) { 302 | /* check file name */ 303 | rc = ngx_http_tfs_duplicate_check_filename(&ctx->dup_file_name, 304 | &t->r_ctx.fsname); 305 | if (rc == NGX_OK) { 306 | /* file name match, modify ref count and save tair */ 307 | if (t->r_ctx.unlink_type == NGX_HTTP_TFS_UNLINK_DELETE) { 308 | if (--ctx->file_ref_count <= 0) { 309 | /* if ref count is 0, 310 | * remove key from tair then unlink file */ 311 | t->state = NGX_HTTP_TFS_STATE_REMOVE_GET_BLK_INFO; 312 | t->is_stat_dup_file = NGX_HTTP_TFS_NO; 313 | t->tfs_peer->body_buffer = ctx->save_body_buffer; 314 | rc = ngx_http_tfs_delete_duplicate_info(ctx, 315 | t->pool, 316 | t->log, 317 | t->meta_segment_data); 318 | /* do not care delete tair fail, 319 | * go on unlinking file */ 320 | if (rc == NGX_ERROR) { 321 | ngx_http_tfs_finalize_state(t, NGX_OK); 322 | } 323 | 324 | return; 325 | } 326 | 327 | /* file_ref_count > 0, just save tair */ 328 | t->state = NGX_HTTP_TFS_STATE_REMOVE_DONE; 329 | rc = ngx_http_tfs_set_duplicate_info(ctx, t->pool, 330 | t->log, 331 | t->meta_segment_data); 332 | /* do not care save tair fail, return success */ 333 | if (rc == NGX_ERROR) { 334 | ngx_http_tfs_finalize_state(t, NGX_DONE); 335 | } 336 | 337 | return; 338 | } 339 | } 340 | 341 | /* file name not match, unlink file */ 342 | t->tfs_peer->body_buffer = ctx->save_body_buffer; 343 | t->state = NGX_HTTP_TFS_STATE_REMOVE_GET_BLK_INFO; 344 | t->is_stat_dup_file = NGX_HTTP_TFS_NO; 345 | ngx_http_tfs_finalize_state(t, NGX_OK); 346 | 347 | return; 348 | } 349 | 350 | /* not exist in tair, unlink file */ 351 | t->tfs_peer->body_buffer = ctx->save_body_buffer; 352 | t->state = NGX_HTTP_TFS_STATE_REMOVE_GET_BLK_INFO; 353 | t->is_stat_dup_file = NGX_HTTP_TFS_NO; 354 | ngx_http_tfs_finalize_state(t, NGX_OK); 355 | return; 356 | case NGX_HTTP_TFS_STATE_REMOVE_GET_BLK_INFO: 357 | case NGX_HTTP_TFS_STATE_REMOVE_DELETE_DATA: 358 | ngx_http_tfs_finalize_state(t, NGX_OK); 359 | return; 360 | case NGX_HTTP_TFS_STATE_REMOVE_DONE: 361 | ngx_http_tfs_finalize_state(t, NGX_DONE); 362 | return; 363 | } 364 | break; 365 | case NGX_HTTP_TFS_ACTION_WRITE_FILE: 366 | switch(t->state) { 367 | case NGX_HTTP_TFS_STATE_WRITE_CLUSTER_ID_NS: 368 | case NGX_HTTP_TFS_STATE_WRITE_GET_BLK_INFO: 369 | /* exist in tair */ 370 | if (rc == NGX_OK) { 371 | /* check suffix */ 372 | rc = ngx_http_tfs_duplicate_check_suffix(&ctx->dup_file_name, 373 | &t->r_ctx.file_suffix); 374 | if (rc == NGX_OK) { 375 | /* suffix match, need to stat file */ 376 | rc = ngx_http_tfs_raw_fsname_parse(&ctx->dup_file_name, 377 | &ctx->dup_file_suffix, 378 | &t->r_ctx.fsname); 379 | if (rc == NGX_OK) { 380 | t->file.cluster_id = t->r_ctx.fsname.cluster_id; 381 | t->is_stat_dup_file = NGX_HTTP_TFS_YES; 382 | t->state = NGX_HTTP_TFS_STATE_WRITE_GET_BLK_INFO; 383 | } 384 | 385 | } else { 386 | /* suffix not match, need save new tfs file, 387 | * do not save tair */ 388 | t->use_dedup = NGX_HTTP_TFS_NO; 389 | } 390 | } /* not exist in tair need save new tfs file and tair */ 391 | 392 | /* need reset meta segment */ 393 | rc = ngx_http_tfs_get_meta_segment(t); 394 | if (rc != NGX_OK) { 395 | ngx_log_error(NGX_LOG_ERR, t->log, 0, 396 | "tfs get meta segment failed"); 397 | ngx_http_tfs_finalize_request(t->data, t, 398 | NGX_HTTP_INTERNAL_SERVER_ERROR); 399 | return; 400 | } 401 | 402 | /* lookup block cache */ 403 | if (t->is_stat_dup_file) { 404 | /* dedup write may need to stat file */ 405 | if (rc_info->use_remote_block_cache) { 406 | rc = ngx_http_tfs_get_remote_block_cache_instance( 407 | &t->block_cache_ctx.remote_ctx, 408 | &rc_info->remote_block_cache_info); 409 | if (rc == NGX_ERROR) { 410 | ngx_log_error(NGX_LOG_ERR, t->log, 0, 411 | "get remote block cache instance failed."); 412 | 413 | } else { 414 | t->block_cache_ctx.use_cache |= 415 | NGX_HTTP_TFS_REMOTE_BLOCK_CACHE; 416 | } 417 | } 418 | 419 | rc = ngx_http_tfs_lookup_block_cache(t, 420 | &t->file.segment_data[t->file.segment_index]); 421 | if (rc == NGX_ERROR) { 422 | ngx_log_error(NGX_LOG_ERR, t->log, 0, 423 | "lookup block cache failed"); 424 | } 425 | 426 | if (rc == NGX_DECLINED 427 | && (t->block_cache_ctx.use_cache 428 | & NGX_HTTP_TFS_REMOTE_BLOCK_CACHE)) 429 | { 430 | return; 431 | } 432 | } 433 | 434 | ngx_http_tfs_finalize_state(t, NGX_OK); 435 | break; 436 | case NGX_HTTP_TFS_STATE_WRITE_STAT_DUP_FILE: 437 | if (rc == NGX_OK) { 438 | t->state = NGX_HTTP_TFS_STATE_WRITE_DONE; 439 | ngx_http_tfs_finalize_state(t, NGX_DONE); 440 | 441 | } else { 442 | /* save tair(add ref count) failed, 443 | * need save new tfs file, do not save tair */ 444 | t->state = NGX_HTTP_TFS_STATE_WRITE_CLUSTER_ID_NS; 445 | t->is_stat_dup_file = NGX_HTTP_TFS_NO; 446 | t->use_dedup = NGX_HTTP_TFS_NO; 447 | /* need reset output buf */ 448 | t->out_bufs = NULL; 449 | /* need reset block id and file id */ 450 | t->file.segment_data[0].segment_info.block_id = 0; 451 | t->file.segment_data[0].segment_info.file_id = 0; 452 | ngx_http_tfs_finalize_state(t, NGX_OK); 453 | } 454 | break; 455 | case NGX_HTTP_TFS_STATE_WRITE_DONE: 456 | ngx_http_tfs_finalize_state(t, NGX_DONE); 457 | break; 458 | } 459 | } 460 | return; 461 | } 462 | 463 | 464 | #ifdef NGX_HTTP_TFS_USE_TAIR 465 | 466 | ngx_int_t 467 | ngx_http_tfs_get_dedup_instance(ngx_http_tfs_dedup_ctx_t *ctx, 468 | ngx_http_tfs_tair_server_addr_info_t *server_addr_info, 469 | uint32_t server_addr_hash) 470 | { 471 | ngx_str_t *st; 472 | ngx_int_t rc, i; 473 | ngx_array_t config_server; 474 | ngx_http_tfs_t *t; 475 | ngx_http_etair_server_conf_t *server; 476 | ngx_http_tfs_tair_instance_t *instance; 477 | 478 | t = ctx->data; 479 | 480 | for (i = 0; i < NGX_HTTP_TFS_MAX_CLUSTER_COUNT; i++) { 481 | instance = &t->main_conf->dup_instances[i]; 482 | 483 | if (instance->server == NULL) { 484 | break; 485 | } 486 | 487 | if (instance->server_addr_hash == server_addr_hash) { 488 | ctx->tair_instance = instance; 489 | return NGX_OK; 490 | } 491 | } 492 | 493 | /* not found && full, clear */ 494 | if (i > NGX_HTTP_TFS_MAX_CLUSTER_COUNT) { 495 | for (i = 0; i < NGX_HTTP_TFS_MAX_CLUSTER_COUNT; i++) { 496 | instance = &t->main_conf->dup_instances[i]; 497 | if (instance->server != NULL) { 498 | ngx_http_etair_destory_server(instance->server, 499 | (ngx_cycle_t *) ngx_cycle); 500 | instance->server = NULL; 501 | } 502 | } 503 | instance = &t->main_conf->dup_instances[0]; 504 | } 505 | 506 | rc = ngx_array_init(&config_server, t->pool, 507 | NGX_HTTP_TFS_TAIR_CONFIG_SERVER_COUNT, 508 | sizeof(ngx_str_t)); 509 | if (rc == NGX_ERROR) { 510 | return NGX_ERROR; 511 | } 512 | 513 | for (i = 0; i < NGX_HTTP_TFS_TAIR_CONFIG_SERVER_COUNT; i++) { 514 | if (server_addr_info->server[i].len > 0 ) { 515 | st = (ngx_str_t *) ngx_array_push(&config_server); 516 | *st = server_addr_info->server[i]; 517 | } 518 | } 519 | 520 | server = &server_addr_info->server[NGX_HTTP_TFS_TAIR_CONFIG_SERVER_COUNT]; 521 | server = ngx_http_etair_create_server(server, 522 | &config_server, 523 | t->main_conf->tair_timeout, 524 | (ngx_cycle_t *) ngx_cycle); 525 | if (server == NULL) { 526 | return NGX_ERROR; 527 | } 528 | instance->server = server; 529 | instance->server_addr_hash = server_addr_hash; 530 | instance->area = server_addr_info->area; 531 | ctx->tair_instance = instance; 532 | 533 | return NGX_OK; 534 | } 535 | 536 | #else 537 | 538 | ngx_int_t 539 | ngx_http_tfs_get_dedup_instance(ngx_http_tfs_dedup_ctx_t *ctx, 540 | ngx_http_tfs_tair_server_addr_info_t *server_addr_info, 541 | uint32_t server_addr_hash) 542 | { 543 | return NGX_ERROR; 544 | } 545 | 546 | #endif 547 | -------------------------------------------------------------------------------- /src/ngx_http_tfs_duplicate.h: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #ifndef _NGX_HTTP_TFS_DUPLICATE_H_INCLUDED_ 8 | #define _NGX_HTTP_TFS_DUPLICATE_H_INCLUDED_ 9 | 10 | 11 | #include 12 | #include 13 | 14 | 15 | typedef struct { 16 | u_char tair_key[NGX_HTTP_TFS_DUPLICATE_KEY_SIZE]; 17 | int32_t dup_version; 18 | int32_t file_ref_count; 19 | ngx_str_t dup_file_name; 20 | ngx_str_t dup_file_suffix; 21 | ngx_buf_t save_body_buffer; 22 | ngx_http_tfs_t *data; 23 | ngx_http_tfs_tair_instance_t *tair_instance; 24 | unsigned md5_sumed:1; 25 | } ngx_http_tfs_dedup_ctx_t; 26 | 27 | 28 | ngx_int_t ngx_http_tfs_get_duplicate_info(ngx_http_tfs_dedup_ctx_t *ctx, 29 | ngx_pool_t *pool, ngx_log_t *log, ngx_chain_t *data); 30 | ngx_int_t ngx_http_tfs_set_duplicate_info(ngx_http_tfs_dedup_ctx_t *ctx, 31 | ngx_pool_t *pool, ngx_log_t *log, ngx_chain_t *data); 32 | ngx_int_t ngx_http_tfs_delete_duplicate_info(ngx_http_tfs_dedup_ctx_t *ctx, 33 | ngx_pool_t *pool, ngx_log_t *log, ngx_chain_t *data); 34 | 35 | ngx_int_t ngx_http_tfs_get_dedup_instance(ngx_http_tfs_dedup_ctx_t *ctx, 36 | ngx_http_tfs_tair_server_addr_info_t *server_addr_info, 37 | uint32_t server_addr_hash); 38 | 39 | 40 | #endif /* _NGX_HTTP_TFS_DUPLICATE_H_INCLUDED_ */ 41 | -------------------------------------------------------------------------------- /src/ngx_http_tfs_errno.h: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #ifndef _NGX_HTTP_TFS_ERRNO_H_INCLUDED_ 8 | #define _NGX_HTTP_TFS_ERRNO_H_INCLUDED_ 9 | 10 | 11 | #include 12 | 13 | 14 | #define NGX_HTTP_TFS_EXIT_GENERAL_ERROR -1000 15 | #define NGX_HTTP_TFS_EXIT_CONFIG_ERROR -1001 16 | #define NGX_HTTP_TFS_EXIT_UNKNOWN_MSGTYPE -1002 17 | #define NGX_HTTP_TFS_EXIT_INVALID_ARGU -1003 18 | #define NGX_HTTP_TFS_EXIT_ALL_SEGMENT_ -1004 19 | #define NGX_HTTP_TFS_EXIT_INVALIDFD_ERROR -1005 20 | #define NGX_HTTP_TFS_EXIT_NOT_INIT_ERROR -1006 21 | #define NGX_HTTP_TFS_EXIT_INVALID_ARGU_ERROR -1007 22 | #define NGX_HTTP_TFS_EXIT_NOT_PERM_OPER -1008 23 | #define NGX_HTTP_TFS_EXIT_NOT_OPEN_ERROR -1009 24 | #define NGX_HTTP_TFS_EXIT_CHECK_CRC_ERROR -1010 25 | #define NGX_HTTP_TFS_EXIT_SERIALIZE_ERROR -1011 26 | #define NGX_HTTP_TFS_EXIT_DESERIALIZE_ERROR -1012 27 | /* access permission error */ 28 | #define NGX_HTTP_TFS_EXIT_ACCESS_PERMISSION_ERROR -1013 29 | /* system parameter error */ 30 | #define NGX_HTTP_TFS_EXIT_SYSTEM_PARAMETER_ERROR -1014 31 | #define NGX_HTTP_TFS_EXIT_UNIQUE_META_NOT_EXIST -1015 32 | /* fuction parameter error */ 33 | #define NGX_HTTP_TFS_EXIT_PARAMETER_ERROR -1016 34 | /* mmap file failed */ 35 | #define NGX_HTTP_TFS_EXIT_MMAP_FILE_ERROR -1017 36 | /* lru value not found by key */ 37 | #define NGX_HTTP_TFS_EXIT_LRU_VALUE_NOT_EXIST -1018 38 | /* lru value existed */ 39 | #define NGX_HTTP_TFS_EXIT_LRU_VALUE_EXIST -1019 40 | /* channel id invalid */ 41 | #define NGX_HTTP_TFS_EXIT_CHANNEL_ID_INVALID -1020 42 | /* data packet timeout */ 43 | #define NGX_HTTP_TFS_EXIT_DATA_PACKET_TIMEOUT -1021 44 | 45 | #define NGX_HTTP_TFS_EXIT_FILE_OP_ERROR -2000 46 | #define NGX_HTTP_TFS_EXIT_OPEN_FILE_ERROR -2001 47 | #define NGX_HTTP_TFS_EXIT_INVALID_FD -2002 48 | #define NGX_HTTP_TFS_EXIT_RECORD_SIZE_ERROR -2003 49 | #define NGX_HTTP_TFS_EXIT_READ_FILE_ERROR -2004 50 | #define NGX_HTTP_TFS_EXIT_WRITE_FILE_ERROR -2005 51 | #define NGX_HTTP_TFS_EXIT_FILESYSTEM_ERROR -2006 52 | #define NGX_HTTP_TFS_EXIT_FILE_FORMAT_ERROR -2007 53 | #define NGX_HTTP_TFS_EXIT_SLOTS_OFFSET_SIZE_ERROR -2008 54 | #define NGX_HTTP_TFS_EXIT_FILE_BUSY_ERROR -2009 55 | 56 | #define NGX_HTTP_TFS_EXIT_NETWORK_ERROR -3000 57 | #define NGX_HTTP_TFS_EXIT_IOCTL_ERROR -3001 58 | #define NGX_HTTP_TFS_EXIT_CONNECT_ERROR -3002 59 | #define NGX_HTTP_TFS_EXIT_SENDMSG_ERROR -3003 60 | #define NGX_HTTP_TFS_EXIT_RECVMSG_ERROR -3004 61 | #define NGX_HTTP_TFS_EXIT_TIMEOUT_ERROR -3005 62 | /* waitid exist error */ 63 | #define NGX_HTTP_TFS_EXIT_WAITID_EXIST_ERROR -3006 64 | /* waitid not found in waitid set */ 65 | #define NGX_HTTP_TFS_EXIT_WAITID_NOT_FOUND_ERROR -3007 66 | /* socket nof found in socket map */ 67 | #define NGX_HTTP_TFS_EXIT_SOCKET_NOT_FOUND_ERROR -3008 68 | 69 | #define NGX_HTTP_TFS_EXIT_TFS_ERROR -5000 70 | #define NGX_HTTP_TFS_EXIT_NO_BLOCK -5001 71 | #define NGX_HTTP_TFS_EXIT_NO_DATASERVER -5002 72 | #define NGX_HTTP_TFS_EXIT_BLOCK_NOT_FOUND -5003 73 | #define NGX_HTTP_TFS_EXIT_DATASERVER_NOT_FOUND -5004 74 | /* lease not found */ 75 | #define NGX_HTTP_TFS_EXIT_CANNOT_GET_LEASE -5005 76 | #define NGX_HTTP_TFS_EXIT_COMMIT_ERROR -5006 77 | #define NGX_HTTP_TFS_EXIT_LEASE_EXPIRED -5007 78 | #define NGX_HTTP_TFS_EXIT_BINLOG_ERROR -5008 79 | #define NGX_HTTP_TFS_EXIT_NO_REPLICATE -5009 80 | #define NGX_HTTP_TFS_EXIT_BLOCK_BUSY -5010 81 | /* update block information version error */ 82 | #define NGX_HTTP_TFS_EXIT_UPDATE_BLOCK_INFO_VERSION_ERROR -5011 83 | /* access mode error */ 84 | #define NGX_HTTP_TFS_EXIT_ACCESS_MODE_ERROR -5012 85 | /* play log error */ 86 | #define NGX_HTTP_TFS_EXIT_PLAY_LOG_ERROR -5013 87 | /* current nameserver only read */ 88 | #define NGX_HTTP_TFS_EXIT_NAMESERVER_ONLY_READ -5014 89 | /* current block already exist */ 90 | #define NGX_HTTP_TFS_EXIT_BLOCK_ALREADY_EXIST -5015 91 | /* create block by block id failed */ 92 | #define NGX_HTTP_TFS_EXIT_CREATE_BLOCK_BY_ID_ERROR -5016 93 | /* server object not found in XXX */ 94 | #define NGX_HTTP_TFS_EIXT_SERVER_OBJECT_NOT_FOUND -5017 95 | /* update relation error */ 96 | #define NGX_HTTP_TFS_EXIT_UPDATE_RELATION_ERROR -5018 97 | /* nameserver in safe_mode_time, discard newblk packet */ 98 | #define NGX_HTTP_TFS_EXIT_DISCARD_NEWBLK_ERROR -5019 99 | 100 | /* write offset error */ 101 | #define NGX_HTTP_TFS_EXIT_WRITE_OFFSET_ERROR -8001 102 | /* read offset error */ 103 | #define NGX_HTTP_TFS_EXIT_READ_OFFSET_ERROR -8002 104 | /* block id is zero, fatal error */ 105 | #define NGX_HTTP_TFS_EXIT_BLOCKID_ZERO_ERROR -8003 106 | /* block is used up, fatal error */ 107 | #define NGX_HTTP_TFS_EXIT_BLOCK_EXHAUST_ERROR -8004 108 | /* need extend too much physcial block when extend block */ 109 | #define NGX_HTTP_TFS_EXIT_PHYSICALBLOCK_NUM_ERROR -8005 110 | /* can't find logic block */ 111 | #define NGX_HTTP_TFS_EXIT_NO_LOGICBLOCK_ERROR -8006 112 | /* input point is null */ 113 | #define NGX_HTTP_TFS_EXIT_POINTER_NULL -8007 114 | /* cat find unused fileid in limited times */ 115 | #define NGX_HTTP_TFS_EXIT_CREATE_FILEID_ERROR -8008 116 | /* block id conflict */ 117 | #define NGX_HTTP_TFS_EXIT_BLOCKID_CONFLICT_ERROR -8009 118 | /* LogicBlock already Exists */ 119 | #define NGX_HTTP_TFS_EXIT_BLOCK_EXIST_ERROR -8010 120 | /* compact block error */ 121 | #define NGX_HTTP_TFS_EXIT_COMPACT_BLOCK_ERROR -8011 122 | /* read or write length is less than required */ 123 | #define NGX_HTTP_TFS_EXIT_DISK_OPER_INCOMPLETE -8012 124 | /* datafile is NULL / crc / getdata error */ 125 | #define NGX_HTTP_TFS_EXIT_DATA_FILE_ERROR -8013 126 | /* too much data file */ 127 | #define NGX_HTTP_TFS_EXIT_DATAFILE_OVERLOAD -8014 128 | /* data file is expired */ 129 | #define NGX_HTTP_TFS_EXIT_DATAFILE_EXPIRE_ERROR -8015 130 | /* file flag or id error when read file */ 131 | #define NGX_HTTP_TFS_EXIT_FILE_INFO_ERROR -8016 132 | /* fileid is same in rename file */ 133 | #define NGX_HTTP_TFS_EXIT_RENAME_FILEID_SAME_ERROR -8017 134 | /* file status error(in unlinkfile) */ 135 | #define NGX_HTTP_TFS_EXIT_FILE_STATUS_ERROR -8018 136 | /* action is not defined(in unlinkfile) */ 137 | #define NGX_HTTP_TFS_EXIT_FILE_ACTION_ERROR -8019 138 | /* file system is not inited */ 139 | #define NGX_HTTP_TFS_EXIT_FS_NOTINIT_ERROR -8020 140 | /* file system's bit map conflict */ 141 | #define NGX_HTTP_TFS_EXIT_BITMAP_CONFLICT_ERROR -8021 142 | /* physical block is already exist in file system */ 143 | #define NGX_HTTP_TFS_EXIT_PHYSIC_UNEXPECT_FOUND_ERROR -8022 144 | #define NGX_HTTP_TFS_EXIT_BLOCK_SETED_ERROR -8023 145 | /* index is loaded when create or load */ 146 | #define NGX_HTTP_TFS_EXIT_INDEX_ALREADY_LOADED_ERROR -8024 147 | /* meta not found in index */ 148 | #define NGX_HTTP_TFS_EXIT_META_NOT_FOUND_ERROR -8025 149 | /* meta found in index when insert */ 150 | #define NGX_HTTP_TFS_EXIT_META_UNEXPECT_FOUND_ERROR -8026 151 | /* require offset is out of index size */ 152 | #define NGX_HTTP_TFS_EXIT_META_OFFSET_ERROR -8027 153 | /* bucket size is conflict with before */ 154 | #define NGX_HTTP_TFS_EXIT_BUCKET_CONFIGURE_ERROR -8028 155 | /* index already exist when create index */ 156 | #define NGX_HTTP_TFS_EXIT_INDEX_UNEXPECT_EXIST_ERROR -8029 157 | /* index is corrupted, and index is created */ 158 | #define NGX_HTTP_TFS_EXIT_INDEX_CORRUPT_ERROR -8030 159 | /* ds version error */ 160 | #define NGX_HTTP_TFS_EXIT_BLOCK_DS_VERSION_ERROR -8031 161 | /* ns version error */ 162 | #define NGX_HTTP_TFS_EXIT_BLOCK_NS_VERSION_ERROR -8332 163 | /* offset is out of physical block size */ 164 | #define NGX_HTTP_TFS_EXIT_PHYSIC_BLOCK_OFFSET_ERROR -8033 165 | /* file size is little than fileinfo */ 166 | #define NGX_HTTP_TFS_EXIT_READ_FILE_SIZE_ERROR -8034 167 | /* connect to ds fail */ 168 | #define NGX_HTTP_TFS_EXIT_DS_CONNECT_ERROR -8035 169 | /* too much block checker */ 170 | #define NGX_HTTP_TFS_EXIT_BLOCK_CHECKER_OVERLOAD -8036 171 | /* fallocate is not implement */ 172 | #define NGX_HTTP_TFS_EXIT_FALLOCATE_NOT_IMPLEMENT -8037 173 | /* sync file failed */ 174 | #define NGX_HTTP_TFS_EXIT_SYNC_FILE_ERROR -8038 175 | 176 | #define NGX_HTTP_TFS_EXIT_SESSION_EXIST_ERROR -9001 177 | #define NGX_HTTP_TFS_EXIT_SESSIONID_INVALID_ERROR -9002 178 | #define NGX_HTTP_TFS_EXIT_APP_NOTEXIST_ERROR -9010 179 | #define NGX_HTTP_TFS_EXIT_APPID_PERMISSION_DENY -9011 180 | 181 | #define NGX_HTTP_TFS_EXIT_SYSTEM_ERROR -10000 182 | #define NGX_HTTP_TFS_EXIT_REGISTER_OPLOG_SYNC_ERROR -12000 183 | #define NGX_HTTP_TFS_EXIT_MAKEDIR_ERROR -13000 184 | 185 | #define NGX_HTTP_TFS_EXIT_UNKNOWN_SQL_ERROR -14000 186 | #define NGX_HTTP_TFS_EXIT_TARGET_EXIST_ERROR -14001 187 | #define NGX_HTTP_TFS_EXIT_PARENT_EXIST_ERROR -14002 188 | #define NGX_HTTP_TFS_EXIT_DELETE_DIR_WITH_FILE_ERROR -14003 189 | #define NGX_HTTP_TFS_EXIT_VERSION_CONFLICT_ERROR -14004 190 | #define NGX_HTTP_TFS_EXIT_NOT_CREATE_ERROR -14005 191 | #define NGX_HTTP_TFS_EXIT_CLUSTER_ID_ERROR -14006 192 | #define NGX_HTTP_TFS_EXIT_FRAG_META_OVERFLOW_ERROR -14007 193 | #define NGX_HTTP_TFS_EXIT_UPDATE_FRAG_INFO_ERROR -14008 194 | #define NGX_HTTP_TFS_EXIT_WRITE_EXIST_POS_ERROR -14009 195 | #define NGX_HTTP_TFS_EXIT_INVALID_FILE_NAME -14010 196 | #define NGX_HTTP_TFS_EXIT_MOVE_TO_SUB_DIR_ERROR -14011 197 | #define NGX_HTTP_TFS_EXIT_OVER_MAX_SUB_DIRS_COUNT -14012 198 | #define NGX_HTTP_TFS_EXIT_OVER_MAX_SUB_DIRS_DEEP -14013 199 | #define NGX_HTTP_TFS_EXIT_OVER_MAX_SUB_FILES_COUNT -14014 200 | 201 | /* server register fail */ 202 | #define NGX_HTTP_TFS_EXIT_REGISTER_ERROR -15000 203 | /* server register fail, server is existed */ 204 | #define NGX_HTTP_TFS_EXIT_REGISTER_EXIST_ERROR -15001 205 | /* renew lease fail, server is not existed */ 206 | #define NGX_HTTP_TFS_EXIT_REGISTER_NOT_EXIST_ERROR -15002 207 | /* table version error */ 208 | #define NGX_HTTP_TFS_EXIT_TABLE_VERSION_ERROR -15003 209 | /* bucket id invalid */ 210 | #define NGX_HTTP_TFS_EXIT_BUCKET_ID_INVLAID -15004 211 | /* bucket not exist */ 212 | #define NGX_HTTP_TFS_EXIT_BUCKET_NOT_EXIST -15005 213 | /* new table not exist */ 214 | #define NGX_HTTP_TFS_EXIT_NEW_TABLE_NOT_EXIST -15005 215 | /* new table invalid */ 216 | #define NGX_HTTP_TFS_EXIT_NEW_TABLE_INVALID -15005 217 | 218 | 219 | #endif /* _NGX_HTTP_TFS_ERRNO_H_INCLUDED_ */ 220 | -------------------------------------------------------------------------------- /src/ngx_http_tfs_json.c: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #include 8 | #include 9 | 10 | 11 | ngx_http_tfs_json_gen_t * 12 | ngx_http_tfs_json_init(ngx_log_t *log, ngx_pool_t *pool) 13 | { 14 | yajl_gen g; 15 | ngx_http_tfs_json_gen_t *tj_gen; 16 | 17 | g = yajl_gen_alloc(NULL); 18 | if (g == NULL) { 19 | ngx_log_error(NGX_LOG_ERR, log, errno, "alloc yajl_gen failed"); 20 | return NULL; 21 | } 22 | 23 | tj_gen = ngx_pcalloc(pool, sizeof(ngx_http_tfs_json_gen_t)); 24 | if (tj_gen == NULL) { 25 | return NULL; 26 | } 27 | 28 | yajl_gen_config(g, yajl_gen_beautify, 1); 29 | 30 | tj_gen->gen = g; 31 | tj_gen->pool = pool; 32 | tj_gen->log = log; 33 | 34 | return tj_gen; 35 | } 36 | 37 | 38 | void 39 | ngx_http_tfs_json_destroy(ngx_http_tfs_json_gen_t *tj_gen) 40 | { 41 | if (tj_gen != NULL) { 42 | yajl_gen_free(tj_gen->gen); 43 | } 44 | } 45 | 46 | 47 | ngx_chain_t * 48 | ngx_http_tfs_json_custom_file_info(ngx_http_tfs_json_gen_t *tj_gen, 49 | ngx_http_tfs_custom_meta_info_t *meta_info, uint8_t file_type) 50 | { 51 | size_t size; 52 | u_char time_buf[NGX_HTTP_TFS_GMT_TIME_SIZE]; 53 | yajl_gen g; 54 | uint32_t count; 55 | ngx_buf_t *b; 56 | ngx_int_t is_file; 57 | ngx_uint_t i; 58 | ngx_chain_t *cl; 59 | ngx_http_tfs_custom_file_t *file; 60 | 61 | g = tj_gen->gen; 62 | size = 0; 63 | 64 | if (file_type == NGX_HTTP_TFS_CUSTOM_FT_DIR) { 65 | yajl_gen_array_open(g); 66 | } 67 | 68 | for(; meta_info; meta_info = meta_info->next) { 69 | count = meta_info->file_count; 70 | file = meta_info->files; 71 | 72 | for (i = 0; i < count; i++) { 73 | yajl_gen_map_open(g); 74 | 75 | yajl_gen_string(g, (const unsigned char *) "NAME", 4); 76 | yajl_gen_string(g, (const unsigned char *) file[i].file_name.data, 77 | file[i].file_name.len); 78 | 79 | yajl_gen_string(g, (const unsigned char *) "PID", 3); 80 | yajl_gen_integer(g, file[i].file_info.pid); 81 | 82 | yajl_gen_string(g, (const unsigned char *) "ID", 2); 83 | yajl_gen_integer(g, file[i].file_info.id); 84 | 85 | yajl_gen_string(g, (const unsigned char *) "SIZE", 4); 86 | yajl_gen_integer(g, file[i].file_info.size); 87 | 88 | yajl_gen_string(g, (const unsigned char *) "IS_FILE", 7); 89 | if (file_type == NGX_HTTP_TFS_CUSTOM_FT_DIR) { 90 | is_file = (file[i].file_info.pid >> 63) & 0x01; 91 | } else { 92 | is_file = 1; 93 | } 94 | yajl_gen_bool(g, is_file); 95 | 96 | ngx_http_tfs_time(time_buf, file[i].file_info.create_time); 97 | yajl_gen_string(g, (const unsigned char *) "CREATE_TIME", 11); 98 | yajl_gen_string(g, time_buf, NGX_HTTP_TFS_GMT_TIME_SIZE); 99 | 100 | ngx_http_tfs_time(time_buf, file[i].file_info.modify_time); 101 | yajl_gen_string(g, (const unsigned char *) "MODIFY_TIME", 11); 102 | yajl_gen_string(g, time_buf, NGX_HTTP_TFS_GMT_TIME_SIZE); 103 | 104 | yajl_gen_string(g, (const unsigned char *) "VER_NO", 6); 105 | yajl_gen_integer(g, file[i].file_info.ver_no); 106 | 107 | yajl_gen_map_close(g); 108 | } 109 | } 110 | 111 | if (file_type == NGX_HTTP_TFS_CUSTOM_FT_DIR) { 112 | yajl_gen_array_close(g); 113 | } 114 | 115 | cl = ngx_alloc_chain_link(tj_gen->pool); 116 | if (cl == NULL) { 117 | return NULL; 118 | } 119 | cl->next = NULL; 120 | 121 | b = ngx_calloc_buf(tj_gen->pool); 122 | if (b == NULL) { 123 | return NULL; 124 | } 125 | 126 | yajl_gen_get_buf(g, (const unsigned char **) &b->pos, &size); 127 | b->last = b->pos + size; 128 | b->end = b->last; 129 | b->temporary = 1; 130 | b->flush = 1; 131 | /* b->last_buf = 1; */ 132 | cl->buf = b; 133 | 134 | return cl; 135 | } 136 | 137 | 138 | ngx_chain_t * 139 | ngx_http_tfs_json_file_name(ngx_http_tfs_json_gen_t *tj_gen, 140 | ngx_str_t *file_name) 141 | { 142 | size_t size; 143 | yajl_gen g; 144 | ngx_buf_t *b; 145 | ngx_chain_t *cl; 146 | 147 | g = tj_gen->gen; 148 | size = 0; 149 | 150 | yajl_gen_map_open(g); 151 | yajl_gen_string(g, (const unsigned char *) "TFS_FILE_NAME", 13); 152 | yajl_gen_string(g, (const unsigned char *) file_name->data, file_name->len); 153 | yajl_gen_map_close(g); 154 | 155 | cl = ngx_alloc_chain_link(tj_gen->pool); 156 | if (cl == NULL) { 157 | return NULL; 158 | } 159 | cl->next = NULL; 160 | 161 | b = ngx_calloc_buf(tj_gen->pool); 162 | if (b == NULL) { 163 | return NULL; 164 | } 165 | yajl_gen_get_buf(g, (const unsigned char **) &b->pos, &size); 166 | b->last = b->pos + size; 167 | b->end = b->last; 168 | b->temporary = 1; 169 | b->flush = 1; 170 | /* b->last_buf = 1; */ 171 | cl->buf = b; 172 | return cl; 173 | } 174 | 175 | 176 | ngx_chain_t * 177 | ngx_http_tfs_json_raw_file_info(ngx_http_tfs_json_gen_t *tj_gen, 178 | u_char* file_name, uint32_t block_id, 179 | ngx_http_tfs_raw_file_info_t *file_info) 180 | { 181 | size_t size; 182 | u_char time_buf[NGX_HTTP_TFS_GMT_TIME_SIZE]; 183 | yajl_gen g; 184 | ngx_buf_t *b; 185 | ngx_chain_t *cl; 186 | 187 | g = tj_gen->gen; 188 | size = 0; 189 | 190 | yajl_gen_map_open(g); 191 | 192 | yajl_gen_string(g, (const unsigned char *) "FILE_NAME", 9); 193 | yajl_gen_string(g, (const unsigned char *) file_name, 18); 194 | 195 | yajl_gen_string(g, (const unsigned char *) "BLOCK_ID", 8); 196 | yajl_gen_integer(g, block_id); 197 | 198 | yajl_gen_string(g, (const unsigned char *) "FILE_ID", 7); 199 | yajl_gen_integer(g, file_info->id); 200 | 201 | yajl_gen_string(g, (const unsigned char *) "OFFSET", 6); 202 | yajl_gen_integer(g, file_info->offset); 203 | 204 | yajl_gen_string(g, (const unsigned char *) "SIZE", 4); 205 | yajl_gen_integer(g, file_info->size); 206 | 207 | yajl_gen_string(g, (const unsigned char *) "OCCUPY_SIZE", 11); 208 | yajl_gen_integer(g, file_info->u_size); 209 | 210 | ngx_http_tfs_time(time_buf, file_info->modify_time); 211 | yajl_gen_string(g, (const unsigned char *) "MODIFY_TIME", 11); 212 | yajl_gen_string(g, time_buf, NGX_HTTP_TFS_GMT_TIME_SIZE); 213 | 214 | ngx_http_tfs_time(time_buf, file_info->create_time); 215 | yajl_gen_string(g, (const unsigned char *) "CREATE_TIME", 11); 216 | yajl_gen_string(g, time_buf, NGX_HTTP_TFS_GMT_TIME_SIZE); 217 | 218 | yajl_gen_string(g, (const unsigned char *) "STATUS", 6); 219 | yajl_gen_integer(g, file_info->flag); 220 | 221 | yajl_gen_string(g, (const unsigned char *) "CRC", 3); 222 | yajl_gen_integer(g, file_info->crc); 223 | 224 | yajl_gen_map_close(g); 225 | 226 | cl = ngx_alloc_chain_link(tj_gen->pool); 227 | if (cl == NULL) { 228 | return NULL; 229 | } 230 | cl->next = NULL; 231 | 232 | b = ngx_calloc_buf(tj_gen->pool); 233 | if (b == NULL) { 234 | return NULL; 235 | } 236 | 237 | yajl_gen_get_buf(g, (const unsigned char **) &b->pos, &size); 238 | 239 | b->last = b->pos + size; 240 | b->end = b->last; 241 | b->temporary = 1; 242 | b->flush = 1; 243 | 244 | /* b->last_buf = 1; */ 245 | 246 | cl->buf = b; 247 | 248 | return cl; 249 | } 250 | 251 | 252 | ngx_chain_t * 253 | ngx_http_tfs_json_appid(ngx_http_tfs_json_gen_t *tj_gen, 254 | uint64_t app_id) 255 | { 256 | size_t size; 257 | yajl_gen g; 258 | ngx_buf_t *b; 259 | ngx_chain_t *cl; 260 | 261 | g = tj_gen->gen; 262 | size = 0; 263 | 264 | u_char str_appid[NGX_INT64_LEN] = {'\0'}; 265 | ngx_sprintf(str_appid, "%uL", app_id); 266 | 267 | yajl_gen_map_open(g); 268 | yajl_gen_string(g, (const unsigned char *) "APP_ID", 6); 269 | yajl_gen_string(g, (const unsigned char *) str_appid, 270 | ngx_strlen(str_appid)); 271 | yajl_gen_map_close(g); 272 | 273 | cl = ngx_alloc_chain_link(tj_gen->pool); 274 | if (cl == NULL) { 275 | return NULL; 276 | } 277 | cl->next = NULL; 278 | 279 | b = ngx_calloc_buf(tj_gen->pool); 280 | if (b == NULL) { 281 | return NULL; 282 | } 283 | 284 | yajl_gen_get_buf(g, (const unsigned char **) &b->pos, &size); 285 | b->last = b->pos + size; 286 | b->end = b->last; 287 | b->temporary = 1; 288 | b->flush = 1; 289 | /* b->last_buf = 1; */ 290 | cl->buf = b; 291 | return cl; 292 | } 293 | 294 | 295 | ngx_chain_t * 296 | ngx_http_tfs_json_file_hole_info(ngx_http_tfs_json_gen_t *tj_gen, 297 | ngx_array_t *file_holes) 298 | { 299 | size_t size; 300 | yajl_gen g; 301 | ngx_buf_t *b; 302 | ngx_uint_t i; 303 | ngx_chain_t *cl; 304 | ngx_http_tfs_file_hole_info_t *file_hole_info; 305 | 306 | g = tj_gen->gen; 307 | size = 0; 308 | 309 | yajl_gen_array_open(g); 310 | 311 | file_hole_info = (ngx_http_tfs_file_hole_info_t *) file_holes->elts; 312 | for(i = 0; i < file_holes->nelts; i++, file_hole_info++) { 313 | yajl_gen_map_open(g); 314 | 315 | yajl_gen_string(g, (const unsigned char *) "OFFSET", 6); 316 | yajl_gen_integer(g, file_hole_info->offset); 317 | 318 | yajl_gen_string(g, (const unsigned char *) "LENGTH", 6); 319 | yajl_gen_integer(g, file_hole_info->length); 320 | 321 | yajl_gen_map_close(g); 322 | } 323 | 324 | yajl_gen_array_close(g); 325 | 326 | cl = ngx_alloc_chain_link(tj_gen->pool); 327 | if (cl == NULL) { 328 | return NULL; 329 | } 330 | cl->next = NULL; 331 | 332 | b = ngx_calloc_buf(tj_gen->pool); 333 | if (b == NULL) { 334 | return NULL; 335 | } 336 | 337 | yajl_gen_get_buf(g, (const unsigned char **) &b->pos, &size); 338 | b->last = b->pos + size; 339 | b->end = b->last; 340 | b->temporary = 1; 341 | b->flush = 1; 342 | /* b->last_buf = 1; */ 343 | cl->buf = b; 344 | return cl; 345 | } 346 | -------------------------------------------------------------------------------- /src/ngx_http_tfs_json.h: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #ifndef _NGX_HTTP_TFS_JSON_H_INCLUDED_ 8 | #define _NGX_HTTP_TFS_JSON_H_INCLUDED_ 9 | 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | 16 | typedef struct { 17 | yajl_gen gen; 18 | ngx_log_t *log; 19 | ngx_pool_t *pool; 20 | } ngx_http_tfs_json_gen_t; 21 | 22 | 23 | ngx_http_tfs_json_gen_t *ngx_http_tfs_json_init(ngx_log_t *log, 24 | ngx_pool_t *pool); 25 | 26 | void ngx_http_tfs_json_destroy(ngx_http_tfs_json_gen_t *tj_gen); 27 | 28 | ngx_chain_t *ngx_http_tfs_json_custom_file_info(ngx_http_tfs_json_gen_t *tj_gen, 29 | ngx_http_tfs_custom_meta_info_t *info, uint8_t file_type); 30 | 31 | ngx_chain_t *ngx_http_tfs_json_file_name(ngx_http_tfs_json_gen_t *tj_gen, 32 | ngx_str_t *file_name); 33 | 34 | ngx_chain_t *ngx_http_tfs_json_raw_file_info(ngx_http_tfs_json_gen_t *tj_gen, 35 | u_char *file_name, uint32_t block_id, 36 | ngx_http_tfs_raw_file_info_t *file_info); 37 | 38 | ngx_chain_t * ngx_http_tfs_json_appid(ngx_http_tfs_json_gen_t *tj_gen, 39 | uint64_t app_id); 40 | ngx_chain_t * ngx_http_tfs_json_file_hole_info(ngx_http_tfs_json_gen_t *tj_gen, 41 | ngx_array_t *file_holes); 42 | 43 | 44 | #endif /* _NGX_HTTP_TFS_JSON_H_INCLUDED_ */ 45 | -------------------------------------------------------------------------------- /src/ngx_http_tfs_local_block_cache.c: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #include 8 | 9 | 10 | static void ngx_http_tfs_local_block_cache_rbtree_insert_value( 11 | ngx_rbtree_node_t *temp, 12 | ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel); 13 | 14 | ngx_int_t 15 | ngx_http_tfs_local_block_cache_init_zone(ngx_shm_zone_t *shm_zone, void *data) 16 | { 17 | size_t len; 18 | ngx_http_tfs_local_block_cache_ctx_t *ctx; 19 | ngx_http_tfs_local_block_cache_ctx_t *octx = data; 20 | 21 | ctx = shm_zone->data; 22 | 23 | if (octx) { 24 | ctx->sh = octx->sh; 25 | ctx->shpool = octx->shpool; 26 | 27 | return NGX_OK; 28 | } 29 | 30 | ctx->shpool = (ngx_slab_pool_t *) shm_zone->shm.addr; 31 | 32 | if (shm_zone->shm.exists) { 33 | ctx->sh = ctx->shpool->data; 34 | 35 | return NGX_OK; 36 | } 37 | 38 | ctx->sh = ngx_slab_alloc(ctx->shpool, 39 | sizeof(ngx_http_tfs_block_cache_shctx_t)); 40 | if (ctx->sh == NULL) { 41 | return NGX_ERROR; 42 | } 43 | 44 | ctx->sh->discard_item_count = NGX_HTTP_TFS_BLOCK_CACHE_DISCARD_ITEM_COUNT; 45 | ctx->sh->hit_count = 0; 46 | ctx->sh->miss_count = 0; 47 | 48 | ctx->shpool->data = ctx->sh; 49 | 50 | ngx_rbtree_init(&ctx->sh->rbtree, &ctx->sh->sentinel, 51 | ngx_http_tfs_local_block_cache_rbtree_insert_value); 52 | 53 | ngx_queue_init(&ctx->sh->queue); 54 | 55 | len = sizeof(" in tfs block cache zone \"\"") + shm_zone->shm.name.len; 56 | 57 | ctx->shpool->log_ctx = ngx_slab_alloc(ctx->shpool, len); 58 | if (ctx->shpool->log_ctx == NULL) { 59 | return NGX_ERROR; 60 | } 61 | 62 | ngx_sprintf(ctx->shpool->log_ctx, " in tfs block cache zone \"%V\"%Z", 63 | &shm_zone->shm.name); 64 | 65 | return NGX_OK; 66 | } 67 | 68 | 69 | void 70 | ngx_http_tfs_local_block_cache_rbtree_insert_value(ngx_rbtree_node_t *temp, 71 | ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel) 72 | { 73 | ngx_rbtree_node_t **p; 74 | ngx_http_tfs_block_cache_node_t *tbn, *tbnt; 75 | 76 | for ( ;; ) { 77 | 78 | if (node->key < temp->key) { 79 | p = &temp->left; 80 | 81 | } else if (node->key > temp->key) { 82 | p = &temp->right; 83 | 84 | } else { /* node->key == temp->key */ 85 | tbn = (ngx_http_tfs_block_cache_node_t *) &node->color; 86 | tbnt = (ngx_http_tfs_block_cache_node_t *) &temp->color; 87 | 88 | p = (ngx_http_tfs_block_cache_cmp(&tbn->key, &tbnt->key) < 0) 89 | ? &temp->left : &temp->right; 90 | } 91 | 92 | if (*p == sentinel) { 93 | break; 94 | } 95 | 96 | temp = *p; 97 | } 98 | 99 | *p = node; 100 | node->parent = temp; 101 | node->left = sentinel; 102 | node->right = sentinel; 103 | ngx_rbt_red(node); 104 | } 105 | 106 | 107 | ngx_int_t 108 | ngx_http_tfs_local_block_cache_lookup(ngx_http_tfs_local_block_cache_ctx_t *ctx, 109 | ngx_pool_t *pool, ngx_log_t *log, 110 | ngx_http_tfs_block_cache_key_t* key, 111 | ngx_http_tfs_block_cache_value_t *value) 112 | { 113 | double hit_ratio; 114 | ngx_int_t rc; 115 | ngx_uint_t hash; 116 | ngx_slab_pool_t *shpool; 117 | ngx_rbtree_node_t *node, *sentinel; 118 | ngx_http_tfs_block_cache_node_t *bcn; 119 | 120 | ngx_log_debug2(NGX_LOG_DEBUG_HTTP, log, 0, 121 | "lookup local block cache, ns addr: %uL, block id: %uD", 122 | key->ns_addr, key->block_id); 123 | 124 | shpool = ctx->shpool; 125 | ngx_shmtx_lock(&shpool->mutex); 126 | 127 | node = ctx->sh->rbtree.root; 128 | sentinel = ctx->sh->rbtree.sentinel; 129 | 130 | hash = ngx_murmur_hash2((u_char*)key, NGX_HTTP_TFS_BLOCK_CACHE_KEY_SIZE); 131 | 132 | while (node != sentinel) { 133 | 134 | if (hash < node->key) { 135 | node = node->left; 136 | continue; 137 | } 138 | 139 | if (hash > node->key) { 140 | node = node->right; 141 | continue; 142 | } 143 | 144 | /* hash == node->key */ 145 | bcn = (ngx_http_tfs_block_cache_node_t *) &node->color; 146 | rc = ngx_http_tfs_block_cache_cmp(key, &bcn->key); 147 | if (rc == 0) { 148 | value->ds_count = bcn->count; 149 | value->ds_addrs = ngx_pcalloc(pool, 150 | value->ds_count * sizeof(uint64_t)); 151 | if (value->ds_addrs == NULL) { 152 | ngx_shmtx_unlock(&shpool->mutex); 153 | return NGX_ERROR; 154 | } 155 | ngx_memcpy(value->ds_addrs, bcn->data, 156 | value->ds_count * sizeof(uint64_t)); 157 | ngx_queue_remove(&bcn->queue); 158 | ngx_queue_insert_head(&ctx->sh->queue, &bcn->queue); 159 | ctx->sh->hit_count++; 160 | if (ctx->sh->hit_count >= NGX_HTTP_TFS_BLOCK_CACHE_STAT_COUNT) { 161 | hit_ratio = 100 * (double)((double)ctx->sh->hit_count 162 | / (double)(ctx->sh->hit_count 163 | + ctx->sh->miss_count)); 164 | ngx_log_error(NGX_LOG_INFO, log, 0, 165 | "local block cache hit_ratio: %.2f%%", 166 | hit_ratio); 167 | ctx->sh->hit_count = 0; 168 | ctx->sh->miss_count = 0; 169 | } 170 | ngx_shmtx_unlock(&shpool->mutex); 171 | return NGX_OK; 172 | } 173 | node = (rc < 0) ? node->left : node->right; 174 | } 175 | ctx->sh->miss_count++; 176 | ngx_shmtx_unlock(&shpool->mutex); 177 | 178 | ngx_log_debug2(NGX_LOG_DEBUG_HTTP, log, 0, 179 | "lookup local block cache, " 180 | "ns addr: %uL, block id: %uD not found", 181 | key->ns_addr, key->block_id); 182 | 183 | return NGX_DECLINED; 184 | } 185 | 186 | 187 | ngx_int_t 188 | ngx_http_tfs_local_block_cache_insert(ngx_http_tfs_local_block_cache_ctx_t *ctx, 189 | ngx_log_t *log, 190 | ngx_http_tfs_block_cache_key_t *key, 191 | ngx_http_tfs_block_cache_value_t *value) 192 | { 193 | size_t n; 194 | ngx_slab_pool_t *shpool; 195 | ngx_rbtree_node_t *node; 196 | ngx_http_tfs_block_cache_node_t *bcn; 197 | 198 | ngx_log_debug2(NGX_LOG_DEBUG_HTTP, log, 0, 199 | "insert local block cache, ns addr: %uL, block id: %uD", 200 | key->ns_addr, key->block_id); 201 | 202 | shpool = ctx->shpool; 203 | 204 | ngx_shmtx_lock(&shpool->mutex); 205 | 206 | n = offsetof(ngx_rbtree_node_t, color) 207 | + offsetof(ngx_http_tfs_block_cache_node_t, data) 208 | + value->ds_count * sizeof(uint64_t); 209 | 210 | node = ngx_slab_alloc_locked(shpool, n); 211 | if (node == NULL) { // full, discard 212 | ngx_http_tfs_local_block_cache_discard(ctx); 213 | node = ngx_slab_alloc_locked(shpool, n); 214 | if (node == NULL) { 215 | ngx_shmtx_unlock(&shpool->mutex); 216 | return NGX_ERROR; 217 | } 218 | } 219 | 220 | bcn = (ngx_http_tfs_block_cache_node_t *) &node->color; 221 | 222 | node->key = ngx_murmur_hash2((u_char*)key, 223 | NGX_HTTP_TFS_BLOCK_CACHE_KEY_SIZE); 224 | ngx_memcpy(&bcn->key, key, NGX_HTTP_TFS_BLOCK_CACHE_KEY_SIZE); 225 | bcn->len = (u_char) value->ds_count * sizeof(uint64_t); 226 | bcn->count = value->ds_count; 227 | ngx_memcpy(bcn->data, value->ds_addrs, bcn->len); 228 | 229 | ngx_rbtree_insert(&(ctx->sh->rbtree), node); 230 | ngx_queue_insert_head(&ctx->sh->queue, &bcn->queue); 231 | 232 | ngx_shmtx_unlock(&shpool->mutex); 233 | 234 | return NGX_OK; 235 | } 236 | 237 | 238 | void 239 | ngx_http_tfs_local_block_cache_remove(ngx_http_tfs_local_block_cache_ctx_t *ctx, 240 | ngx_log_t *log, ngx_http_tfs_block_cache_key_t* key) 241 | { 242 | ngx_int_t rc; 243 | ngx_uint_t hash; 244 | ngx_slab_pool_t *shpool; 245 | ngx_rbtree_node_t *node, *sentinel; 246 | ngx_http_tfs_block_cache_node_t *bcn; 247 | 248 | ngx_log_debug2(NGX_LOG_DEBUG_HTTP, log, 0, 249 | "remove local block cache, ns addr: %uL, block id: %uD", 250 | key->ns_addr, key->block_id); 251 | 252 | shpool = ctx->shpool; 253 | ngx_shmtx_lock(&shpool->mutex); 254 | 255 | node = ctx->sh->rbtree.root; 256 | sentinel = ctx->sh->rbtree.sentinel; 257 | 258 | hash = ngx_murmur_hash2((u_char*)key, NGX_HTTP_TFS_BLOCK_CACHE_KEY_SIZE); 259 | 260 | while (node != sentinel) { 261 | 262 | if (hash < node->key) { 263 | node = node->left; 264 | continue; 265 | } 266 | 267 | if (hash > node->key) { 268 | node = node->right; 269 | continue; 270 | } 271 | 272 | /* hash == node->key */ 273 | do { 274 | bcn = (ngx_http_tfs_block_cache_node_t *) &node->color; 275 | rc = ngx_http_tfs_block_cache_cmp(key, &bcn->key); 276 | if (rc == 0) { 277 | ngx_rbtree_delete(&ctx->sh->rbtree, node); 278 | ngx_slab_free_locked(ctx->shpool, node); 279 | ngx_queue_remove(&bcn->queue); 280 | ngx_shmtx_unlock(&shpool->mutex); 281 | return; 282 | } 283 | node = (rc < 0) ? node->left : node->right; 284 | } while (node != sentinel && hash == node->key); 285 | break; 286 | } 287 | ngx_shmtx_unlock(&shpool->mutex); 288 | 289 | ngx_log_debug2(NGX_LOG_DEBUG_HTTP, log, 0, 290 | "remove local block cache, " 291 | "ns addr: %uL, block id: %uD not found", 292 | key->ns_addr, key->block_id); 293 | 294 | } 295 | 296 | 297 | void 298 | ngx_http_tfs_local_block_cache_discard( 299 | ngx_http_tfs_local_block_cache_ctx_t *ctx) 300 | { 301 | ngx_uint_t i; 302 | ngx_queue_t *q, *h, *p; 303 | ngx_rbtree_node_t *node; 304 | ngx_http_tfs_block_cache_node_t *bcn; 305 | 306 | h = &ctx->sh->queue; 307 | if (ngx_queue_empty(h)) { 308 | return; 309 | } 310 | q = ngx_queue_last(h); 311 | 312 | for (i = 0; i < ctx->sh->discard_item_count; i++) { 313 | if (q == ngx_queue_sentinel(h)) { 314 | ngx_queue_init(h); 315 | return; 316 | } 317 | 318 | p = ngx_queue_prev(q); 319 | 320 | bcn = ngx_queue_data(q, ngx_http_tfs_block_cache_node_t, queue); 321 | 322 | node = (ngx_rbtree_node_t *) 323 | ((u_char *) bcn - offsetof(ngx_rbtree_node_t, color)); 324 | 325 | ngx_rbtree_delete(&ctx->sh->rbtree, node); 326 | 327 | ngx_slab_free_locked(ctx->shpool, node); 328 | 329 | q = p; 330 | } 331 | 332 | q->next = h; 333 | h->prev = q; 334 | } 335 | 336 | 337 | ngx_int_t 338 | ngx_http_tfs_local_block_cache_batch_lookup( 339 | ngx_http_tfs_local_block_cache_ctx_t *ctx, 340 | ngx_pool_t *pool, ngx_log_t *log, ngx_array_t *keys, ngx_array_t *kvs) 341 | { 342 | double hit_ratio; 343 | ngx_int_t rc; 344 | ngx_uint_t i, hash, hit_count; 345 | ngx_slab_pool_t *shpool; 346 | ngx_rbtree_node_t *node, *sentinel; 347 | ngx_http_tfs_block_cache_kv_t *kv; 348 | ngx_http_tfs_block_cache_key_t *key; 349 | ngx_http_tfs_block_cache_node_t *bcn; 350 | ngx_http_tfs_block_cache_value_t *value; 351 | 352 | ngx_log_debug1(NGX_LOG_DEBUG_HTTP, log, 0, 353 | "batch lookup local block cache, block count: %ui", 354 | keys->nelts); 355 | 356 | key = keys->elts; 357 | shpool = ctx->shpool; 358 | rc = NGX_ERROR; 359 | ngx_shmtx_lock(&shpool->mutex); 360 | 361 | sentinel = ctx->sh->rbtree.sentinel; 362 | hit_count = 0; 363 | 364 | for (i = 0; i < keys->nelts; i++, key++) { 365 | node = ctx->sh->rbtree.root; 366 | hash = ngx_murmur_hash2((u_char*)key, 367 | NGX_HTTP_TFS_BLOCK_CACHE_KEY_SIZE); 368 | 369 | while (node != sentinel) { 370 | 371 | if (hash < node->key) { 372 | node = node->left; 373 | continue; 374 | } 375 | 376 | if (hash > node->key) { 377 | node = node->right; 378 | continue; 379 | } 380 | 381 | /* hash == node->key */ 382 | bcn = (ngx_http_tfs_block_cache_node_t *) &node->color; 383 | rc = ngx_http_tfs_block_cache_cmp(key, &bcn->key); 384 | if (rc == 0) { 385 | value = ngx_pcalloc(pool, 386 | sizeof(ngx_http_tfs_block_cache_value_t)); 387 | if (value == NULL) { 388 | ngx_shmtx_unlock(&shpool->mutex); 389 | return NGX_ERROR; 390 | } 391 | 392 | value->ds_count = bcn->count; 393 | value->ds_addrs = ngx_pcalloc(pool, 394 | value->ds_count * sizeof(uint64_t)); 395 | if (value->ds_addrs == NULL) { 396 | ngx_shmtx_unlock(&shpool->mutex); 397 | return NGX_ERROR; 398 | } 399 | ngx_memcpy(value->ds_addrs, bcn->data, 400 | value->ds_count * sizeof(uint64_t)); 401 | 402 | kv = (ngx_http_tfs_block_cache_kv_t *)ngx_array_push(kvs); 403 | kv->key = key; 404 | kv->value = value; 405 | 406 | ngx_queue_remove(&bcn->queue); 407 | ngx_queue_insert_head(&ctx->sh->queue, &bcn->queue); 408 | hit_count++; 409 | break; 410 | } 411 | node = (rc < 0) ? node->left : node->right; 412 | } 413 | 414 | if (node == sentinel) { 415 | ctx->sh->miss_count++; 416 | } 417 | } 418 | 419 | ctx->sh->hit_count += hit_count; 420 | if (ctx->sh->hit_count >= NGX_HTTP_TFS_BLOCK_CACHE_STAT_COUNT) { 421 | hit_ratio = 100 * (double)((double)ctx->sh->hit_count 422 | / (double)(ctx->sh->hit_count 423 | + ctx->sh->miss_count)); 424 | ngx_log_error(NGX_LOG_INFO, log, 0, 425 | "local block cache hit_ratio: %.2f%%", 426 | hit_ratio); 427 | ctx->sh->hit_count = 0; 428 | ctx->sh->miss_count = 0; 429 | } 430 | 431 | ngx_shmtx_unlock(&shpool->mutex); 432 | 433 | ngx_log_debug1(NGX_LOG_DEBUG_HTTP, log, 0, 434 | "batch lookup local block cache, hit_count: %ui", 435 | kvs->nelts); 436 | 437 | /* not all hit */ 438 | if (hit_count < keys->nelts) { 439 | rc = NGX_DECLINED; 440 | } 441 | 442 | return rc; 443 | } 444 | -------------------------------------------------------------------------------- /src/ngx_http_tfs_local_block_cache.h: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #ifndef _NGX_HTTP_TFS_LOCAL_BLOCK_CACHE_H_INCLUDED_ 8 | #define _NGX_HTTP_TFS_LOCAL_BLOCK_CACHE_H_INCLUDED_ 9 | 10 | #include 11 | 12 | 13 | typedef struct { 14 | u_char color; 15 | u_char dummy; 16 | ngx_queue_t queue; 17 | 18 | ngx_http_tfs_block_cache_key_t key; 19 | 20 | u_char len; 21 | u_short count; 22 | u_char data[1]; 23 | } ngx_http_tfs_block_cache_node_t; 24 | 25 | 26 | ngx_int_t ngx_http_tfs_local_block_cache_init_zone(ngx_shm_zone_t *shm_zone, 27 | void *data); 28 | 29 | ngx_int_t ngx_http_tfs_local_block_cache_lookup( 30 | ngx_http_tfs_local_block_cache_ctx_t *ctx, 31 | ngx_pool_t *pool, ngx_log_t *log, 32 | ngx_http_tfs_block_cache_key_t *key, 33 | ngx_http_tfs_block_cache_value_t *value); 34 | 35 | ngx_int_t ngx_http_tfs_local_block_cache_insert( 36 | ngx_http_tfs_local_block_cache_ctx_t *ctx, 37 | ngx_log_t *log, ngx_http_tfs_block_cache_key_t *key, 38 | ngx_http_tfs_block_cache_value_t *value); 39 | 40 | void ngx_http_tfs_local_block_cache_remove( 41 | ngx_http_tfs_local_block_cache_ctx_t *ctx, 42 | ngx_log_t *log, ngx_http_tfs_block_cache_key_t *key); 43 | 44 | void ngx_http_tfs_local_block_cache_discard( 45 | ngx_http_tfs_local_block_cache_ctx_t *ctx); 46 | 47 | ngx_int_t ngx_http_tfs_local_block_cache_batch_lookup( 48 | ngx_http_tfs_local_block_cache_ctx_t *ctx, 49 | ngx_pool_t *pool, ngx_log_t *log, ngx_array_t *keys, ngx_array_t *kvs); 50 | 51 | 52 | 53 | #endif /* _NGX_HTTP_TFS_LOCAL_BLOCK_CACHE_H_INCLUDED_ */ 54 | -------------------------------------------------------------------------------- /src/ngx_http_tfs_meta_server_message.h: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #ifndef _NGX_HTTP_TFS_META_SERVER_MESSAGE_H_INCLUDED_ 8 | #define _NGX_HTTP_TFS_META_SERVER_MESSAGE_H_INCLUDED_ 9 | 10 | 11 | #include 12 | 13 | 14 | ngx_chain_t *ngx_http_tfs_meta_server_create_message(ngx_http_tfs_t *t); 15 | ngx_int_t ngx_http_tfs_meta_server_parse_message(ngx_http_tfs_t *t); 16 | ngx_http_tfs_inet_t *ngx_http_tfs_select_meta_server(ngx_http_tfs_t *t); 17 | 18 | 19 | #endif /* _NGX_HTTP_TFS_META_SERVER_MESSAGE_H_INCLUDED_ */ 20 | -------------------------------------------------------------------------------- /src/ngx_http_tfs_name_server_message.h: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #ifndef _NGX_HTTP_TFS_NAME_SERVER_MESSAGE_H_INCLUDED_ 8 | #define _NGX_HTTP_TFS_NAME_SERVER_MESSAGE_H_INCLUDED_ 9 | 10 | 11 | #include 12 | 13 | 14 | ngx_chain_t *ngx_http_tfs_name_server_create_message(ngx_http_tfs_t *t); 15 | ngx_int_t ngx_http_tfs_name_server_parse_message(ngx_http_tfs_t *t); 16 | ngx_int_t ngx_http_tfs_select_name_server(ngx_http_tfs_t *t, 17 | ngx_http_tfs_rcs_info_t *rc_info, ngx_http_tfs_inet_t *addr, 18 | ngx_str_t *addr_text); 19 | 20 | 21 | #endif /* _NGX_HTTP_TFS_NAME_SERVER_MESSAGE_H_INCLUDED_ */ 22 | -------------------------------------------------------------------------------- /src/ngx_http_tfs_peer_connection.c: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | 12 | static ngx_str_t rcs_name = ngx_string("rc server"); 13 | static ngx_str_t ns_name = ngx_string("name server"); 14 | static ngx_str_t ds_name = ngx_string("data server"); 15 | static ngx_str_t rs_name = ngx_string("root server"); 16 | static ngx_str_t ms_name = ngx_string("meta server"); 17 | 18 | 19 | ngx_int_t 20 | ngx_http_tfs_peer_init(ngx_http_tfs_t *t) 21 | { 22 | char *addr; 23 | uint16_t port; 24 | ngx_http_connection_pool_t *conn_pool; 25 | ngx_http_tfs_peer_connection_t *rc_server, *name_server, *root_server, 26 | *meta_server, *data_server; 27 | 28 | conn_pool = t->main_conf->conn_pool; 29 | 30 | t->tfs_peer_servers = ngx_pcalloc(t->pool, 31 | sizeof(ngx_http_tfs_peer_connection_t) * NGX_HTTP_TFS_SERVER_COUNT); 32 | if (t->tfs_peer_servers == NULL) { 33 | return NGX_ERROR; 34 | } 35 | 36 | name_server = &t->tfs_peer_servers[NGX_HTTP_TFS_NAME_SERVER]; 37 | data_server = &t->tfs_peer_servers[NGX_HTTP_TFS_DATA_SERVER]; 38 | root_server = &t->tfs_peer_servers[NGX_HTTP_TFS_ROOT_SERVER]; 39 | meta_server = &t->tfs_peer_servers[NGX_HTTP_TFS_META_SERVER]; 40 | 41 | /* rc server */ 42 | if (t->loc_conf->upstream->enable_rcs) { 43 | rc_server = &t->tfs_peer_servers[NGX_HTTP_TFS_RC_SERVER]; 44 | rc_server->peer.sockaddr = t->loc_conf->upstream->ups_addr->sockaddr; 45 | rc_server->peer.socklen = t->loc_conf->upstream->ups_addr->socklen; 46 | rc_server->peer.log = t->log; 47 | rc_server->peer.name = &rcs_name; 48 | rc_server->peer.data = conn_pool; 49 | rc_server->peer.get = conn_pool->get_peer; 50 | rc_server->peer.free = conn_pool->free_peer; 51 | rc_server->peer.log_error = NGX_ERROR_ERR; 52 | addr = inet_ntoa(((struct sockaddr_in*) 53 | (rc_server->peer.sockaddr))->sin_addr); 54 | port = ntohs(((struct sockaddr_in*) 55 | (rc_server->peer.sockaddr))->sin_port); 56 | ngx_sprintf(rc_server->peer_addr_text, "%s:%d", addr, port); 57 | 58 | } else { 59 | name_server->peer.sockaddr = t->loc_conf->upstream->ups_addr->sockaddr; 60 | name_server->peer.socklen = t->loc_conf->upstream->ups_addr->socklen; 61 | addr = inet_ntoa(((struct sockaddr_in*) 62 | (name_server->peer.sockaddr))->sin_addr); 63 | port = ntohs(((struct sockaddr_in*) 64 | (name_server->peer.sockaddr))->sin_port); 65 | ngx_sprintf(name_server->peer_addr_text, "%s:%d", addr, port); 66 | } 67 | 68 | /* name server */ 69 | name_server->peer.log = t->log; 70 | name_server->peer.name = &ns_name; 71 | name_server->peer.data = conn_pool; 72 | name_server->peer.get = conn_pool->get_peer; 73 | name_server->peer.free = conn_pool->free_peer; 74 | name_server->peer.log_error = NGX_ERROR_ERR; 75 | 76 | /* data server */ 77 | data_server->peer.log = t->log; 78 | data_server->peer.name = &ds_name; 79 | data_server->peer.data = conn_pool; 80 | data_server->peer.get = conn_pool->get_peer; 81 | data_server->peer.free = conn_pool->free_peer; 82 | data_server->peer.log_error = NGX_ERROR_ERR; 83 | 84 | if (t->r_ctx.version == 1) { 85 | t->tfs_peer_count = 3; 86 | 87 | } else { 88 | /* root server */ 89 | root_server->peer.log = t->log; 90 | root_server->peer.name = &rs_name; 91 | root_server->peer.data = conn_pool; 92 | root_server->peer.get = conn_pool->get_peer; 93 | root_server->peer.free = conn_pool->free_peer; 94 | root_server->peer.log_error = NGX_ERROR_ERR; 95 | 96 | /* meta server */ 97 | meta_server->peer.log = t->log; 98 | meta_server->peer.name = &ms_name; 99 | meta_server->peer.data = conn_pool; 100 | meta_server->peer.get = conn_pool->get_peer; 101 | meta_server->peer.free = conn_pool->free_peer; 102 | meta_server->peer.log_error = NGX_ERROR_ERR; 103 | 104 | t->tfs_peer_count = 5; 105 | } 106 | 107 | return NGX_OK; 108 | } 109 | 110 | 111 | static ngx_http_tfs_peer_connection_t * 112 | ngx_http_tfs_select_peer_v1(ngx_http_tfs_t *t) 113 | { 114 | switch (t->r_ctx.action.code) { 115 | case NGX_HTTP_TFS_ACTION_REMOVE_FILE: 116 | switch (t->state) { 117 | case NGX_HTTP_TFS_STATE_REMOVE_START: 118 | t->create_request = ngx_http_tfs_create_rcs_request; 119 | t->process_request_body = ngx_http_tfs_process_rcs; 120 | t->input_filter = NULL; 121 | return &t->tfs_peer_servers[NGX_HTTP_TFS_RC_SERVER]; 122 | 123 | case NGX_HTTP_TFS_STATE_REMOVE_GET_GROUP_COUNT: 124 | case NGX_HTTP_TFS_STATE_REMOVE_GET_GROUP_SEQ: 125 | case NGX_HTTP_TFS_STATE_REMOVE_GET_BLK_INFO: 126 | t->create_request = ngx_http_tfs_create_ns_request; 127 | t->process_request_body = ngx_http_tfs_process_ns; 128 | t->input_filter = NULL; 129 | t->retry_handler = NULL; 130 | return &t->tfs_peer_servers[NGX_HTTP_TFS_NAME_SERVER]; 131 | 132 | case NGX_HTTP_TFS_STATE_REMOVE_STAT_FILE: 133 | t->create_request = ngx_http_tfs_create_ds_request; 134 | t->process_request_body = ngx_http_tfs_process_ds; 135 | t->input_filter = NULL; 136 | t->retry_handler = ngx_http_tfs_retry_ds; 137 | return &t->tfs_peer_servers[NGX_HTTP_TFS_DATA_SERVER]; 138 | 139 | case NGX_HTTP_TFS_STATE_REMOVE_READ_META_SEGMENT: 140 | t->create_request = ngx_http_tfs_create_ds_request; 141 | t->process_request_body = ngx_http_tfs_process_ds_read; 142 | t->input_filter = ngx_http_tfs_process_ds_input_filter; 143 | t->retry_handler = ngx_http_tfs_retry_ds; 144 | return &t->tfs_peer_servers[NGX_HTTP_TFS_DATA_SERVER]; 145 | 146 | case NGX_HTTP_TFS_STATE_REMOVE_DELETE_DATA: 147 | t->create_request = ngx_http_tfs_create_ds_request; 148 | t->process_request_body = ngx_http_tfs_process_ds; 149 | t->input_filter = NULL; 150 | t->retry_handler = NULL; 151 | return &t->tfs_peer_servers[NGX_HTTP_TFS_DATA_SERVER]; 152 | 153 | case NGX_HTTP_TFS_STATE_REMOVE_DONE: 154 | return t->tfs_peer; 155 | 156 | default: 157 | return NULL; 158 | } 159 | break; 160 | case NGX_HTTP_TFS_ACTION_READ_FILE: 161 | switch (t->state) { 162 | case NGX_HTTP_TFS_STATE_READ_START: 163 | t->create_request = ngx_http_tfs_create_rcs_request; 164 | t->process_request_body = ngx_http_tfs_process_rcs; 165 | t->input_filter = NULL; 166 | return &t->tfs_peer_servers[NGX_HTTP_TFS_RC_SERVER]; 167 | 168 | case NGX_HTTP_TFS_STATE_READ_GET_BLK_INFO: 169 | t->create_request = ngx_http_tfs_create_ns_request; 170 | t->process_request_body = ngx_http_tfs_process_ns; 171 | t->input_filter = NULL; 172 | t->retry_handler = ngx_http_tfs_retry_ns; 173 | return &t->tfs_peer_servers[NGX_HTTP_TFS_NAME_SERVER]; 174 | 175 | case NGX_HTTP_TFS_STATE_READ_READ_DATA: 176 | t->create_request = ngx_http_tfs_create_ds_request; 177 | t->process_request_body = ngx_http_tfs_process_ds_read; 178 | t->input_filter = ngx_http_tfs_process_ds_input_filter; 179 | t->retry_handler = ngx_http_tfs_retry_ds; 180 | return &t->tfs_peer_servers[NGX_HTTP_TFS_DATA_SERVER]; 181 | 182 | case NGX_HTTP_TFS_STATE_READ_DONE: 183 | t->input_filter = NULL; 184 | return t->tfs_peer; 185 | 186 | default: 187 | return NULL; 188 | } 189 | break; 190 | case NGX_HTTP_TFS_ACTION_WRITE_FILE: 191 | switch (t->state) { 192 | case NGX_HTTP_TFS_STATE_WRITE_START: 193 | t->create_request = ngx_http_tfs_create_rcs_request; 194 | t->process_request_body = ngx_http_tfs_process_rcs; 195 | t->input_filter = NULL; 196 | return &t->tfs_peer_servers[NGX_HTTP_TFS_RC_SERVER]; 197 | 198 | case NGX_HTTP_TFS_STATE_WRITE_CLUSTER_ID_NS: 199 | case NGX_HTTP_TFS_STATE_WRITE_GET_BLK_INFO: 200 | t->create_request = ngx_http_tfs_create_ns_request; 201 | t->process_request_body = ngx_http_tfs_process_ns; 202 | t->input_filter = NULL; 203 | t->retry_handler = ngx_http_tfs_retry_ns; 204 | return &t->tfs_peer_servers[NGX_HTTP_TFS_NAME_SERVER]; 205 | 206 | case NGX_HTTP_TFS_STATE_WRITE_STAT_DUP_FILE: 207 | case NGX_HTTP_TFS_STATE_WRITE_CREATE_FILE_NAME: 208 | case NGX_HTTP_TFS_STATE_WRITE_WRITE_DATA: 209 | case NGX_HTTP_TFS_STATE_WRITE_CLOSE_FILE: 210 | t->create_request = ngx_http_tfs_create_ds_request; 211 | t->process_request_body = ngx_http_tfs_process_ds; 212 | t->input_filter = NULL; 213 | /* FIXME: it's better to retry_ns instead of ds when write failed */ 214 | t->retry_handler = ngx_http_tfs_retry_ds; 215 | return &t->tfs_peer_servers[NGX_HTTP_TFS_DATA_SERVER]; 216 | 217 | case NGX_HTTP_TFS_STATE_WRITE_DONE: 218 | t->input_filter = NULL; 219 | return t->tfs_peer; 220 | default: 221 | return NULL; 222 | } 223 | break; 224 | case NGX_HTTP_TFS_ACTION_STAT_FILE: 225 | switch (t->state) { 226 | case NGX_HTTP_TFS_STATE_STAT_START: 227 | t->create_request = ngx_http_tfs_create_rcs_request; 228 | t->process_request_body = ngx_http_tfs_process_rcs; 229 | t->input_filter = NULL; 230 | return &t->tfs_peer_servers[NGX_HTTP_TFS_RC_SERVER]; 231 | 232 | case NGX_HTTP_TFS_STATE_STAT_GET_BLK_INFO: 233 | t->create_request = ngx_http_tfs_create_ns_request; 234 | t->process_request_body = ngx_http_tfs_process_ns; 235 | t->input_filter = NULL; 236 | t->retry_handler = ngx_http_tfs_retry_ns; 237 | return &t->tfs_peer_servers[NGX_HTTP_TFS_NAME_SERVER]; 238 | 239 | case NGX_HTTP_TFS_STATE_STAT_STAT_FILE: 240 | t->create_request = ngx_http_tfs_create_ds_request; 241 | t->process_request_body = ngx_http_tfs_process_ds; 242 | t->input_filter = NULL; 243 | t->retry_handler = ngx_http_tfs_retry_ds; 244 | return &t->tfs_peer_servers[NGX_HTTP_TFS_DATA_SERVER]; 245 | 246 | default: 247 | return NULL; 248 | } 249 | break; 250 | 251 | case NGX_HTTP_TFS_ACTION_KEEPALIVE: 252 | t->create_request = ngx_http_tfs_create_rcs_request; 253 | t->process_request_body = ngx_http_tfs_process_rcs; 254 | t->input_filter = NULL; 255 | return &t->tfs_peer_servers[NGX_HTTP_TFS_RC_SERVER]; 256 | 257 | default: 258 | break; 259 | } 260 | 261 | return NULL; 262 | } 263 | 264 | 265 | static ngx_http_tfs_peer_connection_t * 266 | ngx_http_tfs_select_peer_v2(ngx_http_tfs_t *t) 267 | { 268 | switch (t->r_ctx.action.code) { 269 | case NGX_HTTP_TFS_ACTION_GET_APPID: 270 | t->create_request = ngx_http_tfs_create_rcs_request; 271 | t->process_request_body = ngx_http_tfs_process_rcs; 272 | return &t->tfs_peer_servers[NGX_HTTP_TFS_RC_SERVER]; 273 | 274 | case NGX_HTTP_TFS_ACTION_REMOVE_FILE: 275 | switch (t->state) { 276 | case NGX_HTTP_TFS_STATE_REMOVE_START: 277 | t->create_request = ngx_http_tfs_create_rcs_request; 278 | t->process_request_body = ngx_http_tfs_process_rcs; 279 | return &t->tfs_peer_servers[NGX_HTTP_TFS_RC_SERVER]; 280 | 281 | case NGX_HTTP_TFS_STATE_REMOVE_GET_META_TABLE: 282 | t->create_request = ngx_http_tfs_create_rs_request; 283 | t->process_request_body = ngx_http_tfs_process_rs; 284 | return &t->tfs_peer_servers[NGX_HTTP_TFS_ROOT_SERVER]; 285 | 286 | case NGX_HTTP_TFS_STATE_REMOVE_NOTIFY_MS: 287 | case NGX_HTTP_TFS_STATE_REMOVE_GET_FRAG_INFO: 288 | t->create_request = ngx_http_tfs_create_ms_request; 289 | t->process_request_body = ngx_http_tfs_process_ms; 290 | return &t->tfs_peer_servers[NGX_HTTP_TFS_META_SERVER]; 291 | 292 | case NGX_HTTP_TFS_STATE_REMOVE_GET_GROUP_COUNT: 293 | case NGX_HTTP_TFS_STATE_REMOVE_GET_GROUP_SEQ: 294 | case NGX_HTTP_TFS_STATE_REMOVE_GET_BLK_INFO: 295 | t->create_request = ngx_http_tfs_create_ns_request; 296 | t->process_request_body = ngx_http_tfs_process_ns; 297 | return &t->tfs_peer_servers[NGX_HTTP_TFS_NAME_SERVER]; 298 | 299 | case NGX_HTTP_TFS_STATE_REMOVE_DELETE_DATA: 300 | t->create_request = ngx_http_tfs_create_ds_request; 301 | t->process_request_body = ngx_http_tfs_process_ds; 302 | return &t->tfs_peer_servers[NGX_HTTP_TFS_DATA_SERVER]; 303 | 304 | case NGX_HTTP_TFS_STATE_REMOVE_DONE: 305 | return t->tfs_peer; 306 | default: 307 | return NULL; 308 | } 309 | break; 310 | case NGX_HTTP_TFS_ACTION_READ_FILE: 311 | switch (t->state) { 312 | case NGX_HTTP_TFS_STATE_READ_START: 313 | t->create_request = ngx_http_tfs_create_rcs_request; 314 | t->process_request_body = ngx_http_tfs_process_rcs; 315 | t->input_filter = NULL; 316 | t->retry_handler = NULL; 317 | return &t->tfs_peer_servers[NGX_HTTP_TFS_RC_SERVER]; 318 | 319 | case NGX_HTTP_TFS_STATE_READ_GET_META_TABLE: 320 | t->create_request = ngx_http_tfs_create_rs_request; 321 | t->process_request_body = ngx_http_tfs_process_rs; 322 | t->input_filter = NULL; 323 | t->retry_handler = NULL; 324 | return &t->tfs_peer_servers[NGX_HTTP_TFS_ROOT_SERVER]; 325 | 326 | case NGX_HTTP_TFS_STATE_READ_GET_FRAG_INFO: 327 | t->create_request = ngx_http_tfs_create_ms_request; 328 | t->process_request_body = ngx_http_tfs_process_ms; 329 | t->input_filter = NULL; 330 | t->retry_handler = NULL; 331 | return &t->tfs_peer_servers[NGX_HTTP_TFS_META_SERVER]; 332 | 333 | case NGX_HTTP_TFS_STATE_READ_GET_BLK_INFO: 334 | t->create_request = ngx_http_tfs_create_ns_request; 335 | t->process_request_body = ngx_http_tfs_process_ns; 336 | t->input_filter = NULL; 337 | t->retry_handler = ngx_http_tfs_retry_ns; 338 | return &t->tfs_peer_servers[NGX_HTTP_TFS_NAME_SERVER]; 339 | 340 | case NGX_HTTP_TFS_STATE_READ_READ_DATA: 341 | t->create_request = ngx_http_tfs_create_ds_request; 342 | t->process_request_body = ngx_http_tfs_process_ds_read; 343 | t->input_filter = ngx_http_tfs_process_ds_input_filter; 344 | t->retry_handler = ngx_http_tfs_retry_ds; 345 | return &t->tfs_peer_servers[NGX_HTTP_TFS_DATA_SERVER]; 346 | 347 | case NGX_HTTP_TFS_STATE_READ_DONE: 348 | t->input_filter = NULL; 349 | t->retry_handler = NULL; 350 | return t->tfs_peer; 351 | default: 352 | return NULL; 353 | } 354 | break; 355 | case NGX_HTTP_TFS_ACTION_WRITE_FILE: 356 | switch (t->state) { 357 | case NGX_HTTP_TFS_STATE_WRITE_START: 358 | t->create_request = ngx_http_tfs_create_rcs_request; 359 | t->process_request_body = ngx_http_tfs_process_rcs; 360 | t->input_filter = NULL; 361 | t->retry_handler = NULL; 362 | return &t->tfs_peer_servers[NGX_HTTP_TFS_RC_SERVER]; 363 | 364 | case NGX_HTTP_TFS_STATE_WRITE_GET_META_TABLE: 365 | t->create_request = ngx_http_tfs_create_rs_request; 366 | t->process_request_body = ngx_http_tfs_process_rs; 367 | t->input_filter = NULL; 368 | t->retry_handler = NULL; 369 | return &t->tfs_peer_servers[NGX_HTTP_TFS_ROOT_SERVER]; 370 | 371 | case NGX_HTTP_TFS_STATE_WRITE_CLUSTER_ID_MS: 372 | case NGX_HTTP_TFS_STATE_WRITE_WRITE_MS: 373 | t->create_request = ngx_http_tfs_create_ms_request; 374 | t->process_request_body = ngx_http_tfs_process_ms; 375 | t->input_filter = NULL; 376 | t->retry_handler = NULL; 377 | return &t->tfs_peer_servers[NGX_HTTP_TFS_META_SERVER]; 378 | 379 | case NGX_HTTP_TFS_STATE_WRITE_CLUSTER_ID_NS: 380 | case NGX_HTTP_TFS_STATE_WRITE_GET_BLK_INFO: 381 | t->create_request = ngx_http_tfs_create_ns_request; 382 | t->process_request_body = ngx_http_tfs_process_ns; 383 | t->input_filter = NULL; 384 | t->retry_handler = ngx_http_tfs_retry_ns; 385 | return &t->tfs_peer_servers[NGX_HTTP_TFS_NAME_SERVER]; 386 | 387 | case NGX_HTTP_TFS_STATE_WRITE_CREATE_FILE_NAME: 388 | case NGX_HTTP_TFS_STATE_WRITE_WRITE_DATA: 389 | case NGX_HTTP_TFS_STATE_WRITE_CLOSE_FILE: 390 | t->create_request = ngx_http_tfs_create_ds_request; 391 | t->process_request_body = ngx_http_tfs_process_ds; 392 | t->input_filter = NULL; 393 | /* FIXME: it's better to retry_ns instead of ds when write failed */ 394 | t->retry_handler = ngx_http_tfs_retry_ds; 395 | return &t->tfs_peer_servers[NGX_HTTP_TFS_DATA_SERVER]; 396 | 397 | case NGX_HTTP_TFS_STATE_WRITE_DONE: 398 | t->input_filter = NULL; 399 | t->retry_handler = NULL; 400 | return t->tfs_peer; 401 | default: 402 | return NULL; 403 | } 404 | break; 405 | case NGX_HTTP_TFS_ACTION_CREATE_FILE: 406 | case NGX_HTTP_TFS_ACTION_CREATE_DIR: 407 | case NGX_HTTP_TFS_ACTION_LS_FILE: 408 | case NGX_HTTP_TFS_ACTION_LS_DIR: 409 | case NGX_HTTP_TFS_ACTION_MOVE_DIR: 410 | case NGX_HTTP_TFS_ACTION_MOVE_FILE: 411 | case NGX_HTTP_TFS_ACTION_REMOVE_DIR: 412 | switch (t->state) { 413 | case NGX_HTTP_TFS_STATE_ACTION_START: 414 | t->create_request = ngx_http_tfs_create_rcs_request; 415 | t->process_request_body = ngx_http_tfs_process_rcs; 416 | t->input_filter = NULL; 417 | return &t->tfs_peer_servers[NGX_HTTP_TFS_RC_SERVER]; 418 | 419 | case NGX_HTTP_TFS_STATE_ACTION_GET_META_TABLE: 420 | t->create_request = ngx_http_tfs_create_rs_request; 421 | t->process_request_body = ngx_http_tfs_process_rs; 422 | t->input_filter = NULL; 423 | return &t->tfs_peer_servers[NGX_HTTP_TFS_ROOT_SERVER]; 424 | 425 | case NGX_HTTP_TFS_STATE_ACTION_PROCESS: 426 | t->create_request = ngx_http_tfs_create_ms_request; 427 | if (t->r_ctx.action.code == NGX_HTTP_TFS_ACTION_LS_DIR) { 428 | t->process_request_body = ngx_http_tfs_process_ms_ls_dir; 429 | t->input_filter = ngx_http_tfs_process_ms_input_filter; 430 | } else { 431 | t->process_request_body = ngx_http_tfs_process_ms; 432 | t->input_filter = NULL; 433 | } 434 | return &t->tfs_peer_servers[NGX_HTTP_TFS_META_SERVER]; 435 | 436 | case NGX_HTTP_TFS_STATE_ACTION_DONE: 437 | t->input_filter = NULL; 438 | return t->tfs_peer; 439 | default: 440 | return NULL; 441 | } 442 | break; 443 | default: 444 | break; 445 | } 446 | 447 | return NULL; 448 | } 449 | 450 | 451 | ngx_http_tfs_peer_connection_t * 452 | ngx_http_tfs_select_peer(ngx_http_tfs_t *t) 453 | { 454 | if (t->r_ctx.version == 1) { 455 | return ngx_http_tfs_select_peer_v1(t); 456 | } 457 | 458 | if (t->r_ctx.version == 2) { 459 | return ngx_http_tfs_select_peer_v2(t); 460 | } 461 | 462 | return NULL; 463 | } 464 | 465 | -------------------------------------------------------------------------------- /src/ngx_http_tfs_peer_connection.h: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #ifndef _NGX_HTTP_TFS_PEER_CONNECTION_INCLUDED_ 8 | #define _NGX_HTTP_TFS_PEER_CONNECTION_INCLUDED_ 9 | 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | 16 | struct ngx_http_tfs_peer_connection_s { 17 | ngx_peer_connection_t peer; 18 | u_char peer_addr_text[24]; 19 | ngx_buf_t body_buffer; 20 | ngx_pool_t *pool; 21 | }; 22 | 23 | 24 | ngx_int_t ngx_http_tfs_peer_init(ngx_http_tfs_t *t); 25 | ngx_http_tfs_peer_connection_t *ngx_http_tfs_select_peer(ngx_http_tfs_t *t); 26 | 27 | 28 | #endif /* _NGX_HTTP_TFS_PEER_CONNECTION_INCLUDED_ */ 29 | -------------------------------------------------------------------------------- /src/ngx_http_tfs_protocol.h: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #ifndef _NGX_HTTP_TFS_PROTOCOL_H_ 8 | #define _NGX_HTTP_TFS_PROTOCOL_H_ 9 | 10 | 11 | #define NGX_PACKED __attribute__ ((__packed__)) 12 | 13 | #define NGX_HTTP_TFS_PACKET_FLAG 0x4E534654 /* TFSN */ 14 | #define NGX_HTTP_TFS_PACKET_VERSION 2 15 | 16 | #define NGX_HTTP_TFS_READ 0 17 | #define NGX_HTTP_TFS_READ_V2 1 18 | 19 | #define NGX_HTTP_TFS_RAW_FILE_INFO_SIZE sizeof(ngx_http_tfs_raw_file_info_t) 20 | #define NGX_HTTP_TFS_READ_V2_TAIL_LEN \ 21 | sizeof(ngx_http_tfs_ds_readv2_response_tail_t) 22 | 23 | typedef enum 24 | { 25 | NGX_HTTP_TFS_STATUS_MESSAGE = 1, 26 | NGX_HTTP_TFS_GET_BLOCK_INFO_MESSAGE = 2, 27 | NGX_HTTP_TFS_SET_BLOCK_INFO_MESSAGE = 3, 28 | NGX_HTTP_TFS_CARRY_BLOCK_MESSAGE = 4, 29 | NGX_HTTP_TFS_SET_DATASERVER_MESSAGE = 5, 30 | NGX_HTTP_TFS_UPDATE_BLOCK_INFO_MESSAGE = 6, 31 | NGX_HTTP_TFS_READ_DATA_MESSAGE = 7, 32 | NGX_HTTP_TFS_RESP_READ_DATA_MESSAGE = 8, 33 | NGX_HTTP_TFS_WRITE_DATA_MESSAGE = 9, 34 | NGX_HTTP_TFS_CLOSE_FILE_MESSAGE = 10, 35 | NGX_HTTP_TFS_UNLINK_FILE_MESSAGE = 11, 36 | NGX_HTTP_TFS_REPLICATE_BLOCK_MESSAGE = 12, 37 | NGX_HTTP_TFS_COMPACT_BLOCK_MESSAGE = 13, 38 | NGX_HTTP_TFS_GET_SERVER_STATUS_MESSAGE = 14, 39 | NGX_HTTP_TFS_SHOW_SERVER_INFORMATION_MESSAGE = 15, 40 | NGX_HTTP_TFS_SUSPECT_DATASERVER_MESSAGE = 16, 41 | NGX_HTTP_TFS_FILE_INFO_MESSAGE = 17, 42 | NGX_HTTP_TFS_RESP_FILE_INFO_MESSAGE = 18, 43 | NGX_HTTP_TFS_RENAME_FILE_MESSAGE = 19, 44 | NGX_HTTP_TFS_CLIENT_CMD_MESSAGE = 20, 45 | NGX_HTTP_TFS_CREATE_FILENAME_MESSAGE = 21, 46 | NGX_HTTP_TFS_RESP_CREATE_FILENAME_MESSAGE = 22, 47 | NGX_HTTP_TFS_ROLLBACK_MESSAGE = 23, 48 | NGX_HTTP_TFS_RESP_HEART_MESSAGE = 24, 49 | NGX_HTTP_TFS_RESET_BLOCK_VERSION_MESSAGE = 25, 50 | NGX_HTTP_TFS_BLOCK_FILE_INFO_MESSAGE = 26, 51 | NGX_HTTP_TFS_LEGACY_UNIQUE_FILE_MESSAGE = 27, 52 | NGX_HTTP_TFS_LEGACY_RETRY_COMMAND_MESSAGE = 28, 53 | NGX_HTTP_TFS_NEW_BLOCK_MESSAGE = 29, 54 | NGX_HTTP_TFS_REMOVE_BLOCK_MESSAGE = 30, 55 | NGX_HTTP_TFS_LIST_BLOCK_MESSAGE = 31, 56 | NGX_HTTP_TFS_RESP_LIST_BLOCK_MESSAGE = 32, 57 | NGX_HTTP_TFS_BLOCK_WRITE_COMPLETE_MESSAGE = 33, 58 | NGX_HTTP_TFS_BLOCK_RAW_META_MESSAGE = 34, 59 | NGX_HTTP_TFS_WRITE_RAW_DATA_MESSAGE = 35, 60 | NGX_HTTP_TFS_WRITE_INFO_BATCH_MESSAGE = 36, 61 | NGX_HTTP_TFS_BLOCK_COMPACT_COMPLETE_MESSAGE = 37, 62 | NGX_HTTP_TFS_READ_DATA_MESSAGE_V2 = 38, 63 | NGX_HTTP_TFS_RESP_READ_DATA_MESSAGE_V2 = 39, 64 | NGX_HTTP_TFS_LIST_BITMAP_MESSAGE =40, 65 | NGX_HTTP_TFS_RESP_LIST_BITMAP_MESSAGE = 41, 66 | NGX_HTTP_TFS_RELOAD_CONFIG_MESSAGE = 42, 67 | NGX_HTTP_TFS_SERVER_META_INFO_MESSAGE = 43, 68 | NGX_HTTP_TFS_RESP_SERVER_META_INFO_MESSAGE = 44, 69 | NGX_HTTP_TFS_READ_RAW_DATA_MESSAGE = 45, 70 | NGX_HTTP_TFS_RESP_READ_RAW_DATA_MESSAGE = 46, 71 | NGX_HTTP_TFS_REPLICATE_INFO_MESSAGE = 47, 72 | NGX_HTTP_TFS_ACCESS_STAT_INFO_MESSAGE = 48, 73 | NGX_HTTP_TFS_READ_SCALE_IMAGE_MESSAGE = 49, 74 | NGX_HTTP_TFS_OPLOG_SYNC_MESSAGE = 50, 75 | NGX_HTTP_TFS_OPLOG_SYNC_RESPONSE_MESSAGE = 51, 76 | NGX_HTTP_TFS_MASTER_AND_SLAVE_HEART_MESSAGE = 52, 77 | NGX_HTTP_TFS_MASTER_AND_SLAVE_HEART_RESPONSE_MESSAGE = 53, 78 | NGX_HTTP_TFS_HEARTBEAT_AND_NS_HEART_MESSAGE = 54, 79 | NGX_HTTP_TFS_OWNER_CHECK_MESSAGE = 55, 80 | NGX_HTTP_TFS_GET_BLOCK_LIST_MESSAGE = 56, 81 | NGX_HTTP_TFS_CRC_ERROR_MESSAGE = 57, 82 | NGX_HTTP_TFS_ADMIN_CMD_MESSAGE = 58, 83 | NGX_HTTP_TFS_BATCH_GET_BLOCK_INFO_MESSAGE = 59, 84 | NGX_HTTP_TFS_BATCH_SET_BLOCK_INFO_MESSAGE = 60, 85 | NGX_HTTP_TFS_REMOVE_BLOCK_RESPONSE_MESSAGE = 61, 86 | NGX_HTTP_TFS_READ_DATA_MESSAGE_V3 = 62, 87 | NGX_HTTP_TFS_RESP_READ_DATA_MESSAGE_V3 = 63, 88 | NGX_HTTP_TFS_DUMP_PLAN_MESSAGE = 64, 89 | NGX_HTTP_TFS_DUMP_PLAN_RESPONSE_MESSAGE = 65, 90 | NGX_HTTP_TFS_REQ_RC_LOGIN_MESSAGE = 66, 91 | NGX_HTTP_TFS_RSP_RC_LOGIN_MESSAGE = 67, 92 | NGX_HTTP_TFS_REQ_RC_KEEPALIVE_MESSAGE = 68, 93 | NGX_HTTP_TFS_RSP_RC_KEEPALIVE_MESSAGE = 69, 94 | NGX_HTTP_TFS_REQ_RC_LOGOUT_MESSAGE = 70, 95 | NGX_HTTP_TFS_REQ_RC_RELOAD_MESSAGE = 71, 96 | NGX_HTTP_TFS_GET_DATASERVER_INFORMATION_MESSAGE = 72, 97 | NGX_HTTP_TFS_GET_DATASERVER_INFORMATION_RESPONSE_MESSAGE = 73, 98 | NGX_HTTP_TFS_FILEPATH_ACTION_MESSAGE = 74, 99 | NGX_HTTP_TFS_WRITE_FILEPATH_MESSAGE = 75, 100 | NGX_HTTP_TFS_READ_FILEPATH_MESSAGE = 76, 101 | NGX_HTTP_TFS_RESP_READ_FILEPATH_MESSAGE = 77, 102 | NGX_HTTP_TFS_LS_FILEPATH_MESSAGE = 78, 103 | NGX_HTTP_TFS_RESP_LS_FILEPATH_MESSAGE = 79, 104 | NGX_HTTP_TFS_REQ_RT_UPDATE_TABLE_MESSAGE = 80, 105 | NGX_HTTP_TFS_RSP_RT_UPDATE_TABLE_MESSAGE = 81, 106 | NGX_HTTP_TFS_REQ_RT_MS_KEEPALIVE_MESSAGE = 82, 107 | NGX_HTTP_TFS_RSP_RT_MS_KEEPALIVE_MESSAGE = 83, 108 | NGX_HTTP_TFS_REQ_RT_GET_TABLE_MESSAGE = 84, 109 | NGX_HTTP_TFS_RSP_RT_GET_TABLE_MESSAGE = 85, 110 | NGX_HTTP_TFS_REQ_RT_RS_KEEPALIVE_MESSAGE = 86, 111 | NGX_HTTP_TFS_RSP_RT_RS_KEEPALIVE_MESSAGE = 87, 112 | NGX_HTTP_TFS_LOCAL_PACKET = 500 113 | } ngx_http_tfs_status_msg_e; 114 | 115 | 116 | typedef enum 117 | { 118 | NGX_HTTP_TFS_STATUS_MESSAGE_OK = 0, 119 | NGX_HTTP_TFS_STATUS_MESSAGE_ERROR, 120 | NGX_HTTP_TFS_STATUS_NEED_SEND_BLOCK_INFO, 121 | NGX_HTTP_TFS_STATUS_MESSAGE_PING, 122 | NGX_HTTP_TFS_STATUS_MESSAGE_REMOVE, 123 | NGX_HTTP_TFS_STATUS_MESSAGE_BLOCK_FULL, 124 | NGX_HTTP_TFS_STATUS_MESSAGE_ACCESS_DENIED 125 | } ngx_http_tfs_message_status_t; 126 | 127 | 128 | typedef enum 129 | { 130 | NGX_HTTP_TFS_ACTION_NON = 0, 131 | NGX_HTTP_TFS_ACTION_CREATE_DIR = 1, 132 | NGX_HTTP_TFS_ACTION_CREATE_FILE = 2, 133 | NGX_HTTP_TFS_ACTION_REMOVE_DIR = 3, 134 | NGX_HTTP_TFS_ACTION_REMOVE_FILE = 4, 135 | NGX_HTTP_TFS_ACTION_MOVE_DIR = 5, 136 | NGX_HTTP_TFS_ACTION_MOVE_FILE = 6, 137 | NGX_HTTP_TFS_ACTION_READ_FILE = 7, 138 | NGX_HTTP_TFS_ACTION_LS_DIR = 8, 139 | NGX_HTTP_TFS_ACTION_LS_FILE = 9, 140 | NGX_HTTP_TFS_ACTION_WRITE_FILE = 10, 141 | NGX_HTTP_TFS_ACTION_STAT_FILE = 11, 142 | NGX_HTTP_TFS_ACTION_KEEPALIVE = 12, 143 | NGX_HTTP_TFS_ACTION_GET_APPID = 13, 144 | } ngx_http_tfs_action_e; 145 | 146 | 147 | typedef enum 148 | { 149 | NGX_HTTP_TFS_OPEN_MODE_DEFAULT = 0, 150 | NGX_HTTP_TFS_OPEN_MODE_READ = 1, 151 | NGX_HTTP_TFS_OPEN_MODE_WRITE = 2, 152 | NGX_HTTP_TFS_OPEN_MODE_CREATE = 4, 153 | NGX_HTTP_TFS_OPEN_MODE_NEWBLK = 8, 154 | NGX_HTTP_TFS_OPEN_MODE_NOLEASE = 16, 155 | NGX_HTTP_TFS_OPEN_MODE_STAT = 32, 156 | NGX_HTTP_TFS_OPEN_MODE_LARGE = 64, 157 | NGX_HTTP_TFS_OPEN_MODE_UNLINK = 128 158 | } ngx_http_tfs_open_mode_e; 159 | 160 | 161 | typedef enum 162 | { 163 | NGX_HTTP_TFS_CUSTOM_FT_FILE = 1, 164 | NGX_HTTP_TFS_CUSTOM_FT_DIR, 165 | NGX_HTTP_TFS_CUSTOM_FT_PWRITE_FILE 166 | } ngx_http_tfs_custom_file_type_e; 167 | 168 | 169 | typedef enum 170 | { 171 | NGX_HTTP_TFS_CLIENT_CMD_EXPBLK = 1, 172 | NGX_HTTP_TFS_CLIENT_CMD_LOADBLK, 173 | NGX_HTTP_TFS_CLIENT_CMD_COMPACT, 174 | NGX_HTTP_TFS_CLIENT_CMD_IMMEDIATELY_REPL, 175 | NGX_HTTP_TFS_CLIENT_CMD_REPAIR_GROUP, 176 | NGX_HTTP_TFS_CLIENT_CMD_SET_PARAM, 177 | NGX_HTTP_TFS_CLIENT_CMD_UNLOADBLK, 178 | NGX_HTTP_TFS_CLIENT_CMD_FORCE_DATASERVER_REPORT, 179 | NGX_HTTP_TFS_CLIENT_CMD_ROTATE_LOG, 180 | NGX_HTTP_TFS_CLIENT_CMD_GET_BALANCE_PERCENT, 181 | NGX_HTTP_TFS_CLIENT_CMD_SET_BALANCE_PERCENT 182 | } ngx_http_tfs_ns_ctl_type_e; 183 | 184 | 185 | typedef enum 186 | { 187 | NGX_HTTP_TFS_CLOSE_FILE_MASTER = 100, 188 | NGX_HTTP_TFS_CLOSE_FILE_SLAVER 189 | } ngx_http_tfs_close_mode_e; 190 | 191 | 192 | typedef enum 193 | { 194 | NGX_HTTP_TFS_REMOVE_FILE_MASTER = 0, 195 | NGX_HTTP_TFS_REMOVE_FILE_SLAVER 196 | } ngx_http_tfs_remove_mode_e; 197 | 198 | 199 | typedef enum 200 | { 201 | NGX_HTTP_TFS_FILE_DEFAULT_OPTION = 0, 202 | NGX_HTTP_TFS_FILE_NO_SYNC_LOG = 1, 203 | NGX_HTTP_TFS_FILE_CLOSE_FLAG_WRITE_DATA_FAILED = 2 204 | } ngx_http_tfs_close_option_e; 205 | 206 | 207 | typedef enum 208 | { 209 | NGX_HTTP_TFS_UNLINK_DELETE = 0, 210 | NGX_HTTP_TFS_UNLINK_UNDELETE = 2, 211 | NGX_HTTP_TFS_UNLINK_CONCEAL = 4, 212 | NGX_HTTP_TFS_UNLINK_REVEAL = 6 213 | } ngx_http_tfs_unlink_type_e; 214 | 215 | 216 | typedef enum 217 | { 218 | NGX_HTTP_TFS_FILE_NORMAL = 0, 219 | NGX_HTTP_TFS_FILE_DELETED = 1, 220 | NGX_HTTP_TFS_FILE_INVALID = 2, 221 | NGX_HTTP_TFS_FILE_CONCEAL = 4 222 | } ngx_http_tfs_file_status_e; 223 | 224 | 225 | typedef enum 226 | { 227 | NGX_HTTP_TFS_ACCESS_FORBIDEN = 0, 228 | NGX_HTTP_TFS_ACCESS_READ_ONLY = 1, 229 | NGX_HTTP_TFS_ACCESS_READ_AND_WRITE = 2, 230 | } ngx_http_tfs_access_type_e; 231 | 232 | 233 | typedef struct { 234 | uint32_t flag; 235 | uint32_t len; 236 | uint16_t type; 237 | uint16_t version; 238 | uint64_t id; 239 | uint32_t crc; 240 | } NGX_PACKED ngx_http_tfs_header_t; 241 | 242 | 243 | typedef struct { 244 | int32_t code; 245 | uint32_t error_len; 246 | u_char error_str[]; 247 | } NGX_PACKED ngx_http_tfs_status_msg_t; 248 | 249 | 250 | /* root server */ 251 | typedef struct { 252 | ngx_http_tfs_header_t header; 253 | uint8_t reserse; 254 | } NGX_PACKED ngx_http_tfs_rs_request_t; 255 | 256 | 257 | typedef struct { 258 | uint64_t version; 259 | uint64_t length; 260 | u_char table[]; 261 | } NGX_PACKED ngx_http_tfs_rs_response_t; 262 | 263 | 264 | /* meta server */ 265 | typedef struct { 266 | uint32_t block_id; 267 | uint64_t file_id; 268 | int64_t offset; 269 | uint32_t size; 270 | } NGX_PACKED ngx_http_tfs_meta_frag_meta_info_t; 271 | 272 | 273 | typedef struct { 274 | uint32_t cluster_id; 275 | 276 | /* highest is split flag */ 277 | uint32_t frag_count; 278 | ngx_http_tfs_meta_frag_meta_info_t frag_meta[]; 279 | } NGX_PACKED ngx_http_tfs_meta_frag_info_t; 280 | 281 | 282 | typedef struct { 283 | ngx_http_tfs_header_t header; 284 | uint64_t app_id; 285 | uint64_t user_id; 286 | uint32_t file_len; 287 | u_char file_path_s[]; 288 | } NGX_PACKED ngx_http_tfs_ms_base_msg_header_t; 289 | 290 | 291 | typedef struct { 292 | ngx_http_tfs_header_t header; 293 | uint64_t app_id; 294 | uint64_t user_id; 295 | int64_t pid; 296 | uint32_t file_len; 297 | u_char file_path[]; 298 | } NGX_PACKED ngx_http_tfs_ms_ls_msg_header_t; 299 | 300 | 301 | typedef struct { 302 | /* ignore header */ 303 | uint8_t still_have; 304 | uint32_t count; 305 | } NGX_PACKED ngx_http_tfs_ms_ls_response_t; 306 | 307 | 308 | typedef struct { 309 | /* ignore header */ 310 | uint8_t still_have; 311 | ngx_http_tfs_meta_frag_info_t frag_info; 312 | } NGX_PACKED ngx_http_tfs_ms_read_response_t; 313 | 314 | 315 | /* rc server */ 316 | typedef struct { 317 | ngx_http_tfs_header_t header; 318 | uint32_t appkey_len; 319 | u_char appkey[]; 320 | /* uint64_t app_ip */ 321 | } NGX_PACKED ngx_http_tfs_rcs_login_msg_header_t; 322 | 323 | 324 | /* name server */ 325 | typedef struct { 326 | ngx_http_tfs_header_t header; 327 | uint32_t mode; 328 | 329 | uint32_t block_id; 330 | uint32_t fs_count; 331 | u_char fs_id[]; 332 | } NGX_PACKED ngx_http_tfs_ns_block_info_request_t; 333 | 334 | 335 | typedef struct { 336 | ngx_http_tfs_header_t header; 337 | uint32_t mode; 338 | 339 | uint32_t block_count; 340 | uint32_t block_ids[]; 341 | } NGX_PACKED ngx_http_tfs_ns_batch_block_info_request_t; 342 | 343 | 344 | typedef struct { 345 | /* ignore header */ 346 | uint32_t block_id; 347 | uint32_t ds_count; 348 | uint64_t ds_addrs[]; 349 | } NGX_PACKED ngx_http_tfs_ns_block_info_response_t; 350 | 351 | 352 | typedef struct { 353 | /* ignore header */ 354 | uint32_t block_count; 355 | } NGX_PACKED ngx_http_tfs_ns_batch_block_info_response_t; 356 | 357 | 358 | typedef struct { 359 | ngx_http_tfs_header_t header; 360 | 361 | int32_t cmd; 362 | int64_t value1; 363 | int32_t value2; 364 | int32_t value3; 365 | int64_t value4; 366 | } NGX_PACKED ngx_http_tfs_ns_ctl_request_t; 367 | 368 | 369 | /* data server */ 370 | typedef struct { 371 | ngx_http_tfs_header_t base_header; 372 | uint32_t block_id; 373 | uint64_t file_id; 374 | } NGX_PACKED ngx_http_tfs_ds_msg_header_t; 375 | 376 | 377 | typedef struct { 378 | ngx_http_tfs_ds_msg_header_t header; 379 | int32_t offset; 380 | uint32_t length; 381 | uint8_t flag; 382 | } NGX_PACKED ngx_http_tfs_ds_read_request_t; 383 | 384 | 385 | typedef struct { 386 | ngx_http_tfs_header_t header; 387 | int32_t data_len; 388 | } NGX_PACKED ngx_http_tfs_ds_read_response_t; 389 | 390 | 391 | typedef struct { 392 | /* ignore header */ 393 | uint32_t block_id; 394 | uint64_t file_id; 395 | uint64_t file_number; 396 | } NGX_PACKED ngx_http_tfs_ds_cf_reponse_t; 397 | 398 | 399 | typedef struct { 400 | ngx_http_tfs_ds_msg_header_t header; 401 | int32_t offset; 402 | uint32_t length; 403 | int32_t is_server; 404 | uint64_t file_number; 405 | } NGX_PACKED ngx_http_tfs_ds_write_request_t; 406 | 407 | 408 | typedef struct { 409 | ngx_http_tfs_ds_msg_header_t header; 410 | uint32_t server_mode; 411 | } NGX_PACKED ngx_http_tfs_ds_unlink_request_t; 412 | 413 | 414 | typedef struct { 415 | ngx_http_tfs_ds_msg_header_t header; 416 | int32_t mode; 417 | uint32_t crc; 418 | uint64_t file_number; 419 | } NGX_PACKED ngx_http_tfs_ds_close_request_t; 420 | 421 | 422 | typedef struct { 423 | int64_t pid; 424 | int64_t id; 425 | uint32_t create_time; 426 | uint32_t modify_time; 427 | uint64_t size; 428 | uint16_t ver_no; 429 | } NGX_PACKED ngx_http_tfs_custom_file_info_t; 430 | 431 | 432 | typedef struct { 433 | uint64_t id; 434 | int32_t offset; 435 | int32_t size; 436 | int32_t u_size; 437 | int32_t modify_time; 438 | int32_t create_time; 439 | int32_t flag; 440 | uint32_t crc; 441 | } NGX_PACKED ngx_http_tfs_raw_file_info_t; 442 | 443 | 444 | typedef struct { 445 | ngx_http_tfs_ds_msg_header_t header; 446 | uint32_t mode; 447 | } NGX_PACKED ngx_http_tfs_ds_stat_request_t; 448 | 449 | 450 | typedef struct { 451 | /* ignore header */ 452 | uint32_t data_len; 453 | ngx_http_tfs_raw_file_info_t file_info; 454 | } NGX_PACKED ngx_http_tfs_ds_stat_response_t; 455 | 456 | 457 | typedef struct { 458 | uint32_t file_info_len; 459 | ngx_http_tfs_raw_file_info_t file_info; 460 | } NGX_PACKED ngx_http_tfs_ds_readv2_response_tail_t; 461 | 462 | 463 | typedef struct { 464 | uint32_t count; /* segment count */ 465 | 466 | /* total size of all data segments */ 467 | uint64_t size; 468 | u_char reserve[64]; 469 | } NGX_PACKED ngx_http_tfs_segment_head_t; 470 | 471 | 472 | typedef struct { 473 | /* ignore header */ 474 | int32_t data_len; 475 | ngx_http_tfs_segment_head_t seg_head; 476 | uint32_t file_info_len; 477 | ngx_http_tfs_raw_file_info_t file_info; 478 | } NGX_PACKED ngx_http_tfs_ds_sp_readv2_response_t; 479 | 480 | 481 | typedef struct { 482 | ngx_str_t file_name; 483 | ngx_http_tfs_custom_file_info_t file_info; 484 | } ngx_http_tfs_custom_file_t; 485 | 486 | 487 | typedef struct ngx_http_tfs_custom_meta_info_s ngx_http_tfs_custom_meta_info_t; 488 | 489 | struct ngx_http_tfs_custom_meta_info_s { 490 | uint32_t file_count; 491 | uint32_t rest_file_count; 492 | uint32_t file_index; 493 | ngx_http_tfs_custom_file_t *files; 494 | ngx_http_tfs_custom_meta_info_t *next; 495 | }; 496 | 497 | 498 | typedef struct { 499 | uint64_t offset; 500 | uint64_t length; 501 | } ngx_http_tfs_file_hole_info_t; 502 | 503 | 504 | 505 | 506 | #endif /* _NGX_HTTP_TFS_PROTOCOL_H_ */ 507 | -------------------------------------------------------------------------------- /src/ngx_http_tfs_raw_fsname.c: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #include 8 | 9 | #define NGX_HTTP_TFS_KEY_MASK_LEN 10 /* strlen(NGX_HTTP_TFS_KEY_MASK) */ 10 | 11 | static const u_char* NGX_HTTP_TFS_KEY_MASK = (u_char *)"Taobao-inc"; 12 | 13 | static const u_char enc_table[] = "0JoU8EaN3xf19hIS2d.6p" 14 | "ZRFBYurMDGw7K5m4CyXsbQjg_vTOAkcHVtzqWilnLPe"; 15 | 16 | static const u_char dec_table[] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, \ 17 | 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,18,0,0,11,16,8, \ 18 | 36,34,19,32,4,12,0,0,0,0,0,0,0,49,24,37,29,5,23,30,52,14,1,33,61,28,7, \ 19 | 48,62,42,22,15,47,3,53,57,39,25,21,0,0,0,0,45,0,6,41,51,17,63,10,44,13,\ 20 | 58,43,50,59,35,60,2,20,56,27,40,54,26,46,31,9,38,55,0,0,0,0,0,0,0,0,0, \ 21 | 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, \ 22 | 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, \ 23 | 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, \ 24 | 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; 25 | 26 | 27 | static void 28 | xor_mask(const u_char* source, const int32_t len, u_char* target) 29 | { 30 | int32_t i = 0; 31 | 32 | for (; i < len; i++) { 33 | target[i] = 34 | source[i] ^ NGX_HTTP_TFS_KEY_MASK[i % NGX_HTTP_TFS_KEY_MASK_LEN]; 35 | } 36 | } 37 | 38 | 39 | ngx_int_t 40 | ngx_http_tfs_raw_fsname_parse(ngx_str_t *tfs_name, ngx_str_t *suffix, 41 | ngx_http_tfs_raw_fsname_t* fsname) 42 | { 43 | ngx_uint_t suffix_len; 44 | 45 | if (fsname != NULL && tfs_name->data != NULL && tfs_name->data[0] != '\0') { 46 | ngx_memzero(fsname, sizeof(ngx_http_tfs_raw_fsname_t)); 47 | fsname->file_type = ngx_http_tfs_raw_fsname_check_file_type(tfs_name); 48 | if (fsname->file_type == NGX_HTTP_TFS_INVALID_FILE_TYPE) { 49 | return NGX_ERROR; 50 | } else { 51 | /* if two suffix exist, check consistency */ 52 | if (suffix != NULL 53 | && suffix->data != NULL 54 | && tfs_name->len > NGX_HTTP_TFS_FILE_NAME_LEN) 55 | { 56 | suffix_len = tfs_name->len - NGX_HTTP_TFS_FILE_NAME_LEN; 57 | if (suffix->len != suffix_len) { 58 | return NGX_ERROR; 59 | } 60 | suffix_len = suffix->len > suffix_len ? suffix_len :suffix->len; 61 | if (ngx_memcmp(suffix->data, 62 | tfs_name->data + NGX_HTTP_TFS_FILE_NAME_LEN, 63 | suffix_len)) 64 | { 65 | return NGX_ERROR; 66 | } 67 | } 68 | 69 | ngx_http_tfs_raw_fsname_decode(tfs_name->data + 2, 70 | (u_char*) &(fsname->file)); 71 | if (suffix != NULL && suffix->data == NULL) { 72 | suffix->data = tfs_name->data + NGX_HTTP_TFS_FILE_NAME_LEN; 73 | suffix->len = tfs_name->len - NGX_HTTP_TFS_FILE_NAME_LEN; 74 | } 75 | 76 | ngx_http_tfs_raw_fsname_set_suffix(fsname, suffix); 77 | if (fsname->cluster_id == 0) { 78 | fsname->cluster_id = tfs_name->data[1] - '0'; 79 | } 80 | } 81 | } 82 | 83 | return NGX_OK; 84 | } 85 | 86 | 87 | u_char* 88 | ngx_http_tfs_raw_fsname_get_name(ngx_http_tfs_raw_fsname_t* fsname, 89 | unsigned large_flag, ngx_int_t simple_name) 90 | { 91 | if (fsname != NULL) { 92 | if (simple_name) { 93 | /* zero suffix */ 94 | fsname->file.suffix = 0; 95 | } 96 | 97 | ngx_http_tfs_raw_fsname_encode((u_char*) &(fsname->file), 98 | fsname->file_name + 2); 99 | 100 | if (large_flag) { 101 | fsname->file_name[0] = NGX_HTTP_TFS_LARGE_FILE_KEY_CHAR; 102 | 103 | } else { 104 | fsname->file_name[0] = NGX_HTTP_TFS_SMALL_FILE_KEY_CHAR; 105 | } 106 | fsname->file_name[1] = (u_char) ('0' + fsname->cluster_id); 107 | fsname->file_name[NGX_HTTP_TFS_FILE_NAME_LEN] = '\0'; 108 | 109 | return fsname->file_name; 110 | } 111 | 112 | return NULL; 113 | } 114 | 115 | 116 | ngx_http_tfs_raw_file_type_e 117 | ngx_http_tfs_raw_fsname_check_file_type(ngx_str_t *tfs_name) 118 | { 119 | ngx_http_tfs_raw_file_type_e file_type = NGX_HTTP_TFS_INVALID_FILE_TYPE; 120 | 121 | if (tfs_name->data != NULL 122 | && tfs_name->len >= NGX_HTTP_TFS_FILE_NAME_LEN) 123 | { 124 | if (tfs_name->data[0] == NGX_HTTP_TFS_LARGE_FILE_KEY_CHAR) { 125 | file_type = NGX_HTTP_TFS_LARGE_FILE_TYPE; 126 | 127 | } else if (tfs_name->data[0] == NGX_HTTP_TFS_SMALL_FILE_KEY_CHAR) { 128 | file_type = NGX_HTTP_TFS_SMALL_FILE_TYPE; 129 | } 130 | } 131 | 132 | return file_type; 133 | } 134 | 135 | 136 | void 137 | ngx_http_tfs_raw_fsname_encode(u_char *input, u_char *output) 138 | { 139 | u_char buffer[NGX_HTTP_TFS_FILE_NAME_EXCEPT_SUFFIX_LEN]; 140 | uint32_t value; 141 | ngx_uint_t i, k; 142 | 143 | k = 0; 144 | 145 | if (input != NULL && output != NULL) { 146 | xor_mask(input, NGX_HTTP_TFS_FILE_NAME_EXCEPT_SUFFIX_LEN, buffer); 147 | for (i = 0; i < NGX_HTTP_TFS_FILE_NAME_EXCEPT_SUFFIX_LEN; i += 3) { 148 | value = ((buffer[i] << 16) & 0xff0000) 149 | + ((buffer[i + 1] << 8) & 0xff00) + (buffer[i + 2] & 0xff); 150 | output[k++] = enc_table[value >> 18]; 151 | output[k++] = enc_table[(value >> 12) & 0x3f]; 152 | output[k++] = enc_table[(value >> 6) & 0x3f]; 153 | output[k++] = enc_table[value & 0x3f]; 154 | } 155 | } 156 | } 157 | 158 | 159 | void 160 | ngx_http_tfs_raw_fsname_decode(u_char *input, u_char *output) 161 | { 162 | u_char buffer[NGX_HTTP_TFS_FILE_NAME_EXCEPT_SUFFIX_LEN]; 163 | uint32_t value; 164 | ngx_uint_t i, k; 165 | 166 | k = 0; 167 | 168 | if (input != NULL && output != NULL) { 169 | for (i = 0; i < NGX_HTTP_TFS_FILE_NAME_LEN - 2; i += 4) { 170 | value = (dec_table[input[i] & 0xff] << 18) 171 | + (dec_table[input[i + 1] & 0xff] << 12) 172 | + (dec_table[input[i + 2] & 0xff] << 6) 173 | + dec_table[input[i + 3] & 0xff]; 174 | buffer[k++] = (u_char) ((value >> 16) & 0xff); 175 | buffer[k++] = (u_char) ((value >> 8) & 0xff); 176 | buffer[k++] = (u_char) (value & 0xff); 177 | } 178 | xor_mask(buffer, NGX_HTTP_TFS_FILE_NAME_EXCEPT_SUFFIX_LEN, output); 179 | } 180 | } 181 | -------------------------------------------------------------------------------- /src/ngx_http_tfs_raw_fsname.h: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #ifndef _NGX_HTTP_TFS_RAW_FSNAME_H_INCLUDED_ 8 | #define _NGX_HTTP_TFS_RAW_FSNAME_H_INCLUDED_ 9 | 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | 16 | typedef enum 17 | { 18 | NGX_HTTP_TFS_INVALID_FILE_TYPE = 0, 19 | NGX_HTTP_TFS_SMALL_FILE_TYPE, 20 | NGX_HTTP_TFS_LARGE_FILE_TYPE 21 | } ngx_http_tfs_raw_file_type_e; 22 | 23 | 24 | typedef struct { 25 | uint32_t block_id; 26 | uint32_t seq_id; 27 | uint32_t suffix; 28 | } ngx_http_tfs_raw_fsname_filebits_t; 29 | 30 | 31 | typedef struct { 32 | u_char file_name[NGX_HTTP_TFS_FILE_NAME_BUFF_LEN]; 33 | 34 | ngx_http_tfs_raw_fsname_filebits_t file; 35 | 36 | uint32_t cluster_id; 37 | ngx_http_tfs_raw_file_type_e file_type; 38 | } ngx_http_tfs_raw_fsname_t; 39 | 40 | 41 | #define ngx_http_tfs_raw_fsname_set_suffix(fsname, fs_suffix) do { \ 42 | if ((fs_suffix != NULL) \ 43 | && (fs_suffix->data != NULL) \ 44 | && (fs_suffix->len != 0)) \ 45 | { \ 46 | fsname->file.suffix = ngx_http_tfs_raw_fsname_hash(fs_suffix->data, \ 47 | fs_suffix->len); \ 48 | } \ 49 | } while(0) 50 | 51 | 52 | #define ngx_http_tfs_raw_fsname_set_file_id(fsname, id) \ 53 | fsname->file.suffix = (id >> 32); \ 54 | fsname->file.seq_id = (id & 0xFFFFFFFF) 55 | 56 | 57 | #define ngx_http_tfs_raw_fsname_get_file_id(fsname) \ 58 | ((((uint64_t)(fsname.file.suffix)) << 32) | fsname.file.seq_id) 59 | 60 | 61 | #define ngx_http_tfs_group_seq_match(block_id, group_count, group_seq) \ 62 | ((block_id % group_count) == (ngx_uint_t) group_seq) 63 | 64 | 65 | ngx_http_tfs_raw_file_type_e ngx_http_tfs_raw_fsname_check_file_type( 66 | ngx_str_t *tfs_name); 67 | void ngx_http_tfs_raw_fsname_encode(u_char * input, u_char *output); 68 | void ngx_http_tfs_raw_fsname_decode(u_char * input, u_char *output); 69 | 70 | ngx_int_t ngx_http_tfs_raw_fsname_parse(ngx_str_t *tfs_name, ngx_str_t *suffix, 71 | ngx_http_tfs_raw_fsname_t *fsname); 72 | u_char* ngx_http_tfs_raw_fsname_get_name(ngx_http_tfs_raw_fsname_t *fsname, 73 | unsigned large_flag, ngx_int_t no_suffix); 74 | 75 | 76 | #endif /* _NGX_HTTP_TFS_RAW_FSNAME_H_INCLUDED_ */ 77 | -------------------------------------------------------------------------------- /src/ngx_http_tfs_rc_server_info.c: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #include 8 | 9 | 10 | static void ngx_http_tfs_rcs_rbtree_insert_value(ngx_rbtree_node_t *temp, 11 | ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel); 12 | 13 | 14 | ngx_http_tfs_rcs_info_t * 15 | ngx_http_tfs_rcs_lookup(ngx_http_request_t *r, ngx_http_tfs_rc_ctx_t *ctx, 16 | ngx_str_t appkey) 17 | { 18 | ngx_int_t rc; 19 | ngx_uint_t hash; 20 | ngx_rbtree_node_t *node, *sentinel; 21 | ngx_http_tfs_rcs_info_t *tr; 22 | 23 | node = ctx->sh->rbtree.root; 24 | sentinel = ctx->sh->rbtree.sentinel; 25 | 26 | hash = ngx_murmur_hash2(appkey.data, appkey.len); 27 | 28 | while (node != sentinel) { 29 | 30 | if (hash < node->key) { 31 | node = node->left; 32 | continue; 33 | } 34 | 35 | if (hash > node->key) { 36 | node = node->right; 37 | continue; 38 | } 39 | 40 | /* hash == node->key */ 41 | 42 | tr = (ngx_http_tfs_rcs_info_t *) &node->color; 43 | rc = ngx_memn2cmp(appkey.data, tr->appkey.data, appkey.len, 44 | tr->appkey.len); 45 | 46 | if (rc == 0) { 47 | ngx_queue_remove(&tr->queue); 48 | ngx_queue_insert_head(&ctx->sh->queue, &tr->queue); 49 | 50 | return tr; 51 | } 52 | 53 | node = (rc < 0) ? node->left : node->right; 54 | } 55 | 56 | return NULL; 57 | } 58 | 59 | 60 | void 61 | ngx_http_tfs_rc_server_destroy_node(ngx_http_tfs_rc_ctx_t *rc_ctx, 62 | ngx_http_tfs_rcs_info_t *rc_info_node) 63 | { 64 | ngx_str_t *block_cache_info; 65 | ngx_uint_t i, j; 66 | ngx_rbtree_node_t *node; 67 | ngx_http_tfs_group_info_t *group_info; 68 | ngx_http_tfs_logical_cluster_t *logical_cluster; 69 | ngx_http_tfs_physical_cluster_t *physical_cluster; 70 | ngx_http_tfs_cluster_group_info_t *cluster_group_info; 71 | ngx_http_tfs_tair_server_addr_info_t *dup_server_info; 72 | 73 | if (rc_info_node == NULL) { 74 | return; 75 | } 76 | 77 | if (rc_info_node->session_id.len <= 0 78 | || rc_info_node->session_id.data == NULL) 79 | { 80 | goto last_free; 81 | } 82 | 83 | ngx_slab_free_locked(rc_ctx->shpool, rc_info_node->session_id.data); 84 | ngx_str_null(&rc_info_node->session_id); 85 | 86 | if (rc_info_node->rc_servers_count <= 0 87 | || rc_info_node->rc_servers == NULL) 88 | { 89 | goto last_free; 90 | } 91 | 92 | ngx_slab_free_locked(rc_ctx->shpool, rc_info_node->rc_servers); 93 | block_cache_info = &rc_info_node->remote_block_cache_info; 94 | rc_info_node->rc_servers = NULL; 95 | 96 | logical_cluster = rc_info_node->logical_clusters; 97 | for (i = 0; i < rc_info_node->logical_cluster_count; i++) { 98 | if (logical_cluster->need_duplicate) { 99 | dup_server_info = &logical_cluster->dup_server_info; 100 | 101 | for (i = 0; i < NGX_HTTP_TFS_TAIR_SERVER_ADDR_PART_COUNT; i++) { 102 | if (dup_server_info->server[i].data == NULL) { 103 | goto last_free; 104 | } 105 | ngx_slab_free_locked(rc_ctx->shpool, 106 | dup_server_info->server[i].data); 107 | ngx_str_null(&dup_server_info->server[i]); 108 | } 109 | } 110 | 111 | physical_cluster = logical_cluster->rw_clusters; 112 | for (j = 0; j < logical_cluster->rw_cluster_count; j++) { 113 | if (physical_cluster[j].cluster_id_text.len <= 0 114 | || physical_cluster[j].cluster_id_text.data == NULL) 115 | { 116 | goto last_free; 117 | } 118 | ngx_slab_free_locked(rc_ctx->shpool, 119 | physical_cluster[j].cluster_id_text.data); 120 | ngx_str_null(&physical_cluster[j].cluster_id_text); 121 | physical_cluster[j].cluster_id = 0; 122 | 123 | if (physical_cluster[j].ns_vip_text.len <= 0 124 | || physical_cluster[j].ns_vip_text.data == NULL) 125 | { 126 | goto last_free; 127 | } 128 | ngx_slab_free_locked(rc_ctx->shpool, 129 | physical_cluster[j].ns_vip_text.data); 130 | ngx_str_null(&physical_cluster[j].ns_vip_text); 131 | physical_cluster++; 132 | } 133 | logical_cluster++; 134 | } 135 | 136 | if (block_cache_info->len <= 0 || block_cache_info->data == NULL) 137 | { 138 | goto last_free; 139 | } 140 | 141 | ngx_slab_free_locked(rc_ctx->shpool, block_cache_info->data); 142 | ngx_str_null(&rc_info_node->remote_block_cache_info); 143 | 144 | cluster_group_info = rc_info_node->unlink_clusters; 145 | for (i = 0; i < rc_info_node->unlink_cluster_count; i++) { 146 | for (j = 0; j < cluster_group_info[i].info_count; j++) { 147 | group_info = &cluster_group_info[i].group_info[j]; 148 | if (group_info->ns_vip_text.len <= 0 149 | || group_info->ns_vip_text.data == NULL) 150 | { 151 | break; 152 | } 153 | ngx_slab_free_locked(rc_ctx->shpool, group_info->ns_vip_text.data); 154 | ngx_str_null(&group_info->ns_vip_text); 155 | } 156 | } 157 | 158 | last_free: 159 | node = (ngx_rbtree_node_t *) 160 | ((u_char *) rc_info_node - offsetof(ngx_rbtree_node_t, color)); 161 | ngx_slab_free_locked(rc_ctx->shpool, node); 162 | } 163 | 164 | 165 | void 166 | ngx_http_tfs_rc_server_expire(ngx_http_tfs_rc_ctx_t *ctx) 167 | { 168 | ngx_queue_t *q, *kp_q; 169 | ngx_rbtree_node_t *node; 170 | ngx_http_tfs_rcs_info_t *rc_info_node; 171 | 172 | if (ngx_queue_empty(&ctx->sh->queue)) { 173 | return; 174 | } 175 | 176 | q = ngx_queue_last(&ctx->sh->queue); 177 | 178 | rc_info_node = ngx_queue_data(q, ngx_http_tfs_rcs_info_t, queue); 179 | kp_q = &rc_info_node->kp_queue; 180 | 181 | ngx_queue_remove(q); 182 | ngx_queue_remove(kp_q); 183 | 184 | node = (ngx_rbtree_node_t *) 185 | ((u_char *) rc_info_node - offsetof(ngx_rbtree_node_t, color)); 186 | 187 | ngx_rbtree_delete(&ctx->sh->rbtree, node); 188 | 189 | ngx_http_tfs_rc_server_destroy_node(ctx, rc_info_node); 190 | } 191 | 192 | 193 | ngx_int_t 194 | ngx_http_tfs_rc_server_init_zone(ngx_shm_zone_t *shm_zone, void *data) 195 | { 196 | ngx_http_tfs_rc_ctx_t *octx = data; 197 | 198 | size_t len; 199 | ngx_http_tfs_rc_ctx_t *ctx; 200 | 201 | ctx = shm_zone->data; 202 | 203 | if (octx) { 204 | 205 | ctx->sh = octx->sh; 206 | ctx->shpool = octx->shpool; 207 | 208 | return NGX_OK; 209 | } 210 | 211 | ctx->shpool = (ngx_slab_pool_t *) shm_zone->shm.addr; 212 | 213 | if (shm_zone->shm.exists) { 214 | ctx->sh = ctx->shpool->data; 215 | 216 | return NGX_OK; 217 | } 218 | 219 | ctx->sh = ngx_slab_alloc(ctx->shpool, sizeof(ngx_http_tfs_rc_shctx_t)); 220 | if (ctx->sh == NULL) { 221 | return NGX_ERROR; 222 | } 223 | 224 | ctx->shpool->data = ctx->sh; 225 | 226 | ngx_rbtree_init(&ctx->sh->rbtree, &ctx->sh->sentinel, 227 | ngx_http_tfs_rcs_rbtree_insert_value); 228 | ngx_queue_init(&ctx->sh->queue); 229 | ngx_queue_init(&ctx->sh->kp_queue); 230 | 231 | len = sizeof(" in tfs rc servers zone \"\"") + shm_zone->shm.name.len; 232 | 233 | ctx->shpool->log_ctx = ngx_slab_alloc(ctx->shpool, len); 234 | if (ctx->shpool->log_ctx == NULL) { 235 | return NGX_ERROR; 236 | } 237 | 238 | ngx_sprintf(ctx->shpool->log_ctx, " in tfs rc servers zone \"%V\"%Z", 239 | &shm_zone->shm.name); 240 | 241 | return NGX_OK; 242 | } 243 | 244 | 245 | static void 246 | ngx_http_tfs_rcs_rbtree_insert_value(ngx_rbtree_node_t *temp, 247 | ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel) 248 | { 249 | ngx_int_t rc; 250 | ngx_rbtree_node_t **p; 251 | ngx_http_tfs_rcs_info_t *trn, *trnt; 252 | 253 | for ( ;; ) { 254 | 255 | if (node->key < temp->key) { 256 | 257 | p = &temp->left; 258 | 259 | } else if (node->key > temp->key) { 260 | 261 | p = &temp->right; 262 | 263 | } else { /* node->key == temp->key */ 264 | 265 | trn = (ngx_http_tfs_rcs_info_t *) &node->color; 266 | trnt = (ngx_http_tfs_rcs_info_t *) &temp->color; 267 | 268 | rc = ngx_memn2cmp(trn->appkey.data, trnt->appkey.data, 269 | trn->appkey.len, trn->appkey.len); 270 | if (rc < 0) { 271 | p = &temp->left; 272 | 273 | } else if (rc > 0) { 274 | p = &temp->right; 275 | 276 | } else { 277 | return; 278 | } 279 | } 280 | 281 | if (*p == sentinel) { 282 | break; 283 | } 284 | 285 | temp = *p; 286 | } 287 | 288 | *p = node; 289 | node->parent = temp; 290 | node->left = sentinel; 291 | node->right = sentinel; 292 | ngx_rbt_red(node); 293 | } 294 | 295 | 296 | void 297 | ngx_http_tfs_rcs_set_group_info_by_addr(ngx_http_tfs_rcs_info_t *rc_info, 298 | ngx_int_t group_count, ngx_int_t group_seq, ngx_http_tfs_inet_t addr) 299 | { 300 | ngx_uint_t i, j; 301 | ngx_http_tfs_group_info_t *group_info; 302 | ngx_http_tfs_cluster_group_info_t *cluster_group_info; 303 | 304 | cluster_group_info = rc_info->unlink_clusters; 305 | 306 | for (i = 0; i < rc_info->unlink_cluster_count; i++) { 307 | group_info = cluster_group_info[i].group_info; 308 | 309 | for (j = 0; j < cluster_group_info[i].info_count; j++) { 310 | 311 | if (group_info[j].group_seq >= 0) { 312 | continue; 313 | } 314 | 315 | if (ngx_memcmp(&group_info[j].ns_vip, &addr, 316 | sizeof(ngx_http_tfs_inet_t)) == 0) 317 | { 318 | group_info[j].group_seq = group_seq; 319 | cluster_group_info[i].group_count = group_count; 320 | return; 321 | } 322 | } 323 | } 324 | } 325 | -------------------------------------------------------------------------------- /src/ngx_http_tfs_rc_server_info.h: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #ifndef _NGX_HTTP_TFS_RC_SERVER_INFO_H_INCLUDED_ 8 | #define _NGX_HTTP_TFS_RC_SERVER_INFO_H_INCLUDED_ 9 | 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | 18 | #define ngx_http_tfs_cluster_is_master(cluster_id) \ 19 | (cluster_id[2] == 'M' || cluster_id[2] == 'm') 20 | 21 | #define ngx_http_tfs_get_cluster_id(cluster_id_data) \ 22 | (cluster_id_data[1] - '0') 23 | 24 | 25 | typedef struct { 26 | int32_t group_seq; /* get from ns */ 27 | ngx_str_t ns_vip_text; 28 | uint8_t is_master; 29 | ngx_http_tfs_inet_t ns_vip; 30 | } ngx_http_tfs_group_info_t; 31 | 32 | 33 | typedef struct { 34 | uint32_t cluster_id; 35 | int32_t group_count; /* get from ns */ 36 | uint32_t info_count; 37 | ngx_http_tfs_group_info_t group_info[NGX_HTTP_TFS_MAX_CLUSTER_ID_COUNT]; 38 | } ngx_http_tfs_cluster_group_info_t; 39 | 40 | 41 | typedef struct { 42 | uint32_t cluster_stat; 43 | uint32_t access_type; 44 | uint32_t cluster_id; /* get from ns */ 45 | ngx_str_t cluster_id_text; 46 | ngx_str_t ns_vip_text; 47 | ngx_http_tfs_inet_t ns_vip; 48 | } ngx_http_tfs_physical_cluster_t; 49 | 50 | 51 | typedef struct { 52 | uint8_t need_duplicate; 53 | uint32_t dup_server_addr_hash; 54 | ngx_http_tfs_tair_server_addr_info_t dup_server_info; 55 | 56 | /* for read and write */ 57 | uint32_t rw_cluster_count; 58 | ngx_http_tfs_physical_cluster_t rw_clusters[NGX_HTTP_TFS_MAX_CLUSTER_COUNT]; 59 | } ngx_http_tfs_logical_cluster_t; 60 | 61 | 62 | typedef struct { 63 | u_char color; 64 | u_char dummy; 65 | ngx_queue_t queue; 66 | /* for keep alive, fixed sequence */ 67 | ngx_queue_t kp_queue; 68 | 69 | ngx_str_t appkey; 70 | uint64_t app_id; 71 | ngx_str_t session_id; 72 | uint32_t rc_servers_count; 73 | uint64_t *rc_servers; 74 | 75 | /* logical cluster */ 76 | uint32_t logical_cluster_count; 77 | ngx_http_tfs_logical_cluster_t logical_clusters[NGX_HTTP_TFS_MAX_CLUSTER_COUNT]; 78 | 79 | uint8_t need_duplicate; 80 | 81 | uint32_t report_interval; 82 | uint64_t modify_time; 83 | uint64_t meta_root_server; 84 | ngx_str_t remote_block_cache_info; 85 | 86 | /* for unlink & update */ 87 | uint8_t unlink_cluster_count; 88 | ngx_http_tfs_cluster_group_info_t unlink_clusters[NGX_HTTP_TFS_MAX_CLUSTER_COUNT]; 89 | 90 | uint32_t use_remote_block_cache; 91 | } ngx_http_tfs_rcs_info_t; 92 | 93 | 94 | typedef struct { 95 | ngx_rbtree_t rbtree; 96 | ngx_rbtree_node_t sentinel; 97 | ngx_queue_t queue; 98 | /* for keep alive, fixed sequence */ 99 | ngx_queue_t kp_queue; 100 | } ngx_http_tfs_rc_shctx_t; 101 | 102 | 103 | typedef struct { 104 | ngx_http_tfs_rc_shctx_t *sh; 105 | ngx_slab_pool_t *shpool; 106 | } ngx_http_tfs_rc_ctx_t; 107 | 108 | 109 | ngx_int_t ngx_http_tfs_rc_server_update(ngx_http_request_t *r, 110 | ngx_http_tfs_rc_ctx_t *t, ngx_http_tfs_rcs_info_t *info); 111 | ngx_int_t ngx_http_tfs_rc_server_init_zone(ngx_shm_zone_t *shm_zone, 112 | void *data); 113 | void ngx_http_tfs_rc_server_expire(ngx_http_tfs_rc_ctx_t *ctx); 114 | ngx_http_tfs_rcs_info_t *ngx_http_tfs_rcs_lookup(ngx_http_request_t *r, 115 | ngx_http_tfs_rc_ctx_t *ctx, ngx_str_t appkey); 116 | void ngx_http_tfs_rc_server_destroy_node(ngx_http_tfs_rc_ctx_t *ctx, 117 | ngx_http_tfs_rcs_info_t *rc_info_node); 118 | void ngx_http_tfs_rcs_set_group_info_by_addr(ngx_http_tfs_rcs_info_t *rc_info, 119 | ngx_int_t group_count, ngx_int_t seq_id, ngx_http_tfs_inet_t addr); 120 | 121 | 122 | #endif /* _NGX_HTTP_TFS_RC_SERVER_INFO_H_INCLUDED_ */ 123 | 124 | -------------------------------------------------------------------------------- /src/ngx_http_tfs_rc_server_message.h: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #ifndef _NGX_TFS_RC_SERVER_MESSAGE_H_INCLUDED_ 8 | #define _NGX_TFS_RC_SERVER_MESSAGE_H_INCLUDED_ 9 | 10 | 11 | #include 12 | 13 | 14 | ngx_chain_t *ngx_http_tfs_rc_server_create_message(ngx_http_tfs_t *t); 15 | ngx_int_t ngx_http_tfs_rc_server_parse_message(ngx_http_tfs_t *t); 16 | 17 | 18 | #endif /* _NGX_TFS_RC_SERVER_MESSAGE_H_INCLUDED_ */ 19 | -------------------------------------------------------------------------------- /src/ngx_http_tfs_remote_block_cache.c: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | 13 | static void ngx_http_tfs_remote_block_cache_get_handler( 14 | ngx_http_tair_key_value_t *kv, ngx_int_t rc, void *data); 15 | static void ngx_http_tfs_remote_block_cache_dummy_handler(ngx_int_t rc, 16 | void *data); 17 | 18 | static void ngx_http_tfs_remote_block_cache_mget_handler(ngx_array_t *kvs, 19 | ngx_int_t rc, void *data); 20 | 21 | ngx_int_t 22 | ngx_http_tfs_remote_block_cache_lookup( 23 | ngx_http_tfs_remote_block_cache_ctx_t *ctx, 24 | ngx_pool_t *pool, ngx_log_t *log, ngx_http_tfs_block_cache_key_t* key) 25 | { 26 | ngx_int_t rc; 27 | ngx_http_tair_data_t tair_key; 28 | 29 | ngx_log_debug2(NGX_LOG_DEBUG_HTTP, log, 0, 30 | "lookup remote block cache, ns addr: %uL, block id: %uD", 31 | key->ns_addr, key->block_id); 32 | 33 | tair_key.type = NGX_HTTP_TAIR_INT; 34 | tair_key.data = (u_char *)key; 35 | tair_key.len = NGX_HTTP_TFS_BLOCK_CACHE_KEY_SIZE; 36 | 37 | rc = ngx_http_tfs_tair_get_helper( 38 | ctx->tair_instance, 39 | pool, log, 40 | &tair_key, 41 | ngx_http_tfs_remote_block_cache_get_handler, 42 | (void *)ctx); 43 | 44 | return rc; 45 | } 46 | 47 | 48 | static void 49 | ngx_http_tfs_remote_block_cache_get_handler(ngx_http_tair_key_value_t *kv, 50 | ngx_int_t rc, void *data) 51 | { 52 | u_char *p, *q; 53 | uint32_t ds_count; 54 | ngx_http_tfs_t *t; 55 | ngx_http_tfs_inet_t *addr; 56 | ngx_http_tfs_segment_data_t *segment_data; 57 | ngx_http_tfs_block_cache_key_t key; 58 | ngx_http_tfs_block_cache_value_t value; 59 | ngx_http_tfs_remote_block_cache_ctx_t *ctx = data; 60 | 61 | t = ctx->data; 62 | segment_data = &t->file.segment_data[t->file.segment_index]; 63 | if (rc == NGX_HTTP_ETAIR_SUCCESS) { 64 | q = kv->key.data; 65 | p = kv->value->data; 66 | if (p != NULL 67 | && (kv->value->len 68 | > NGX_HTTP_TFS_REMOTE_BLOCK_CACHE_VALUE_BASE_SIZE)) 69 | { 70 | key.ns_addr = *(uint64_t *)q; 71 | q += sizeof(uint64_t); 72 | key.block_id = *(uint32_t *)q; 73 | 74 | ds_count = *(uint32_t *)p; 75 | p += sizeof(uint32_t); 76 | 77 | if (ds_count > 0) { 78 | segment_data->block_info.ds_count = ds_count; 79 | segment_data->block_info.ds_addrs = ngx_pcalloc(t->pool, 80 | sizeof(ngx_http_tfs_inet_t) * ds_count); 81 | if (segment_data->block_info.ds_addrs == NULL) { 82 | ngx_http_tfs_finalize_request(t->data, t, 83 | NGX_HTTP_INTERNAL_SERVER_ERROR); 84 | return; 85 | } 86 | ngx_memcpy(segment_data->block_info.ds_addrs, p, 87 | ds_count * sizeof(ngx_http_tfs_inet_t)); 88 | 89 | /* insert local block cache */ 90 | if (t->block_cache_ctx.use_cache 91 | & NGX_HTTP_TFS_LOCAL_BLOCK_CACHE) 92 | { 93 | value.ds_count = ds_count; 94 | value.ds_addrs = 95 | (uint64_t *)segment_data->block_info.ds_addrs; 96 | ngx_http_tfs_local_block_cache_insert( 97 | t->block_cache_ctx.local_ctx, 98 | t->log, &key, &value); 99 | } 100 | 101 | /* skip GET_BLK_INFO state */ 102 | t->state += 1; 103 | 104 | segment_data->cache_hit = NGX_HTTP_TFS_REMOTE_BLOCK_CACHE; 105 | segment_data->block_info_src = NGX_HTTP_TFS_FROM_CACHE; 106 | 107 | /* select data server */ 108 | addr = ngx_http_tfs_select_data_server(t, segment_data); 109 | 110 | ngx_http_tfs_peer_set_addr(t->pool, 111 | &t->tfs_peer_servers[NGX_HTTP_TFS_DATA_SERVER], 112 | addr); 113 | 114 | } else { 115 | /* remote block cache invalid, need remove it */ 116 | ngx_http_tfs_remote_block_cache_remove(ctx, t->pool, t->log, 117 | &key); 118 | } 119 | } 120 | 121 | } else { 122 | ngx_log_debug2(NGX_LOG_DEBUG_HTTP, t->log, 0, 123 | "lookup remote block cache, " 124 | "ns addr: %V, block id: %uD not found", 125 | &t->name_server_addr_text, 126 | segment_data->segment_info.block_id); 127 | } 128 | 129 | ngx_http_tfs_finalize_state(t, NGX_OK); 130 | } 131 | 132 | 133 | ngx_int_t 134 | ngx_http_tfs_remote_block_cache_insert( 135 | ngx_http_tfs_remote_block_cache_ctx_t *ctx, 136 | ngx_pool_t *pool, ngx_log_t *log, 137 | ngx_http_tfs_block_cache_key_t *key, 138 | ngx_http_tfs_block_cache_value_t *value) 139 | { 140 | ngx_int_t rc; 141 | ngx_pool_t *tmp_pool; 142 | ngx_http_tair_data_t tair_key; 143 | ngx_http_tair_data_t tair_value; 144 | 145 | ngx_log_debug2(NGX_LOG_DEBUG_HTTP, log, 0, 146 | "insert remote block cache, " 147 | "ns addr: %uL, block id: %uD", 148 | key->ns_addr, key->block_id); 149 | 150 | tair_key.type = NGX_HTTP_TAIR_INT; 151 | tair_key.data = (u_char *)key; 152 | tair_key.len = NGX_HTTP_TFS_BLOCK_CACHE_KEY_SIZE; 153 | 154 | tair_value.len = NGX_HTTP_TFS_REMOTE_BLOCK_CACHE_VALUE_BASE_SIZE 155 | + value->ds_count * sizeof(uint64_t); 156 | tair_value.data = ngx_pcalloc(pool, tair_value.len); 157 | if (tair_value.data == NULL) { 158 | return NGX_ERROR; 159 | } 160 | *(uint32_t*)tair_value.data = value->ds_count; 161 | ngx_memcpy(tair_value.data+ NGX_HTTP_TFS_REMOTE_BLOCK_CACHE_VALUE_BASE_SIZE, 162 | value->ds_addrs, value->ds_count * sizeof(uint64_t)); 163 | tair_value.type = NGX_HTTP_TAIR_INT; 164 | 165 | /* since we do not care returns, 166 | * we make a tmp pool and destroy it in callback */ 167 | tmp_pool = ngx_create_pool(4096, log); 168 | if (tmp_pool == NULL) { 169 | return NGX_ERROR; 170 | } 171 | 172 | rc = ngx_http_tfs_tair_put_helper( 173 | ctx->tair_instance, 174 | tmp_pool, log, 175 | &tair_key, &tair_value, 176 | 0/*expire*/, 0/* do not care version */, 177 | ngx_http_tfs_remote_block_cache_dummy_handler, 178 | (void *)tmp_pool); 179 | 180 | return rc; 181 | } 182 | 183 | 184 | static void 185 | ngx_http_tfs_remote_block_cache_dummy_handler(ngx_int_t rc, void *data) 186 | { 187 | ngx_pool_t *pool = (ngx_pool_t *)data; 188 | ngx_destroy_pool(pool); 189 | 190 | return; 191 | } 192 | 193 | 194 | void 195 | ngx_http_tfs_remote_block_cache_remove( 196 | ngx_http_tfs_remote_block_cache_ctx_t *ctx, 197 | ngx_pool_t *pool, ngx_log_t *log, ngx_http_tfs_block_cache_key_t* key) 198 | { 199 | ngx_int_t rc; 200 | ngx_pool_t *tmp_pool; 201 | ngx_array_t tair_keys; 202 | ngx_http_tair_data_t *tair_key; 203 | 204 | ngx_log_debug2(NGX_LOG_DEBUG_HTTP, log, 0, 205 | "remove remote block cache, ns addr: %uL, block id: %uD", 206 | key->ns_addr, key->block_id); 207 | 208 | rc = ngx_array_init(&tair_keys, pool, 1, sizeof(ngx_http_tair_data_t)); 209 | if (rc == NGX_ERROR) { 210 | return; 211 | } 212 | tair_key = (ngx_http_tair_data_t*) ngx_array_push(&tair_keys); 213 | 214 | tair_key->type = NGX_HTTP_TAIR_INT; 215 | tair_key->data = (u_char *)key; 216 | tair_key->len = NGX_HTTP_TFS_BLOCK_CACHE_KEY_SIZE; 217 | 218 | /* since we do not care returns, 219 | * we make a tmp pool and destroy it in callback */ 220 | tmp_pool = ngx_create_pool(4096, log); 221 | if (tmp_pool == NULL) { 222 | return; 223 | } 224 | 225 | (void) ngx_http_tfs_tair_delete_helper( 226 | ctx->tair_instance, 227 | tmp_pool, log, 228 | &tair_keys, 229 | ngx_http_tfs_remote_block_cache_dummy_handler, 230 | (void *)tmp_pool); 231 | 232 | } 233 | 234 | 235 | ngx_int_t 236 | ngx_http_tfs_remote_block_cache_batch_lookup( 237 | ngx_http_tfs_remote_block_cache_ctx_t *ctx, 238 | ngx_pool_t *pool, ngx_log_t *log, ngx_array_t* keys) 239 | { 240 | ngx_int_t rc; 241 | ngx_uint_t i; 242 | ngx_array_t *tair_kvs; 243 | ngx_http_tair_key_value_t *tair_kv; 244 | ngx_http_tfs_block_cache_key_t *key; 245 | 246 | tair_kvs = ngx_array_create(pool, keys->nelts, 247 | sizeof(ngx_http_tair_key_value_t)); 248 | if (tair_kvs == NULL) { 249 | return NGX_ERROR; 250 | } 251 | 252 | key = keys->elts; 253 | for (i = 0; i < keys->nelts; i++, key++) { 254 | ngx_log_debug2(NGX_LOG_DEBUG_HTTP, log, 0, 255 | "batch lookup remote block cache, " 256 | "ns addr: %uL, block id: %uD", 257 | key->ns_addr, key->block_id); 258 | 259 | tair_kv = (ngx_http_tair_key_value_t *)ngx_array_push(tair_kvs); 260 | if (tair_kv == NULL) { 261 | return NGX_ERROR; 262 | } 263 | 264 | tair_kv->key.type = NGX_HTTP_TAIR_INT; 265 | tair_kv->key.data = (u_char *)key; 266 | tair_kv->key.len = NGX_HTTP_TFS_BLOCK_CACHE_KEY_SIZE; 267 | } 268 | 269 | rc = ngx_http_tfs_tair_mget_helper( 270 | ctx->tair_instance, 271 | pool, log, 272 | tair_kvs, 273 | ngx_http_tfs_remote_block_cache_mget_handler, 274 | (void *)ctx); 275 | return rc; 276 | } 277 | 278 | 279 | static void 280 | ngx_http_tfs_remote_block_cache_mget_handler(ngx_array_t *kvs, ngx_int_t rc, 281 | void *data) 282 | { 283 | u_char *p, *q; 284 | uint32_t ds_count, block_count; 285 | ngx_uint_t i, j, hit_count; 286 | ngx_http_tfs_t *t; 287 | ngx_http_tair_key_value_t *kv; 288 | ngx_http_tfs_segment_data_t *segment_data; 289 | ngx_http_tfs_block_cache_key_t key; 290 | ngx_http_tfs_block_cache_value_t value; 291 | ngx_http_tfs_remote_block_cache_ctx_t *ctx = data; 292 | 293 | t = ctx->data; 294 | 295 | segment_data = &t->file.segment_data[t->file.segment_index]; 296 | block_count = t->file.segment_count - t->file.segment_index; 297 | if (block_count > NGX_HTTP_TFS_MAX_SEND_FRAG_COUNT) { 298 | block_count = NGX_HTTP_TFS_MAX_SEND_FRAG_COUNT; 299 | } 300 | 301 | if (rc == NGX_OK) { 302 | kv = kvs->elts; 303 | hit_count = 0; 304 | for (i = 0; i < kvs->nelts; i++, kv++) { 305 | if (kv->rc != NGX_HTTP_ETAIR_SUCCESS) { 306 | continue; 307 | } 308 | q = kv->key.data; 309 | p = kv->value->data; 310 | if (p != NULL 311 | && (kv->value->len 312 | > NGX_HTTP_TFS_REMOTE_BLOCK_CACHE_VALUE_BASE_SIZE)) 313 | { 314 | key.ns_addr = *(uint64_t *)q; 315 | q += sizeof(uint64_t); 316 | key.block_id = *(uint32_t *)q; 317 | 318 | ds_count = *(uint32_t *)p; 319 | p += sizeof(uint32_t); 320 | 321 | if (ds_count > 0) { 322 | /* find out segment */ 323 | for (j = 0; j < block_count; j++) { 324 | if(segment_data[j].segment_info.block_id == key.block_id 325 | && segment_data[j].block_info.ds_addrs == NULL) 326 | { 327 | break; 328 | } 329 | } 330 | /* not found, some error happen */ 331 | if (j == block_count) { 332 | continue; 333 | } 334 | 335 | segment_data[j].block_info.ds_count = ds_count; 336 | segment_data[j].block_info.ds_addrs = ngx_pcalloc(t->pool, 337 | ds_count * sizeof(ngx_http_tfs_inet_t)); 338 | if (segment_data[j].block_info.ds_addrs == NULL) { 339 | ngx_http_tfs_finalize_request(t->data, t, 340 | NGX_HTTP_INTERNAL_SERVER_ERROR); 341 | return; 342 | } 343 | ngx_memcpy(segment_data[j].block_info.ds_addrs, p, 344 | ds_count * sizeof(ngx_http_tfs_inet_t)); 345 | 346 | if (t->block_cache_ctx.use_cache 347 | & NGX_HTTP_TFS_LOCAL_BLOCK_CACHE) 348 | { 349 | value.ds_count = ds_count; 350 | value.ds_addrs = 351 | (uint64_t *)segment_data[j].block_info.ds_addrs; 352 | ngx_http_tfs_local_block_cache_insert( 353 | t->block_cache_ctx.local_ctx, t->log, &key, &value); 354 | } 355 | 356 | hit_count++; 357 | segment_data[j].cache_hit = NGX_HTTP_TFS_REMOTE_BLOCK_CACHE; 358 | segment_data[j].block_info_src = NGX_HTTP_TFS_FROM_CACHE; 359 | 360 | ngx_log_debug2(NGX_LOG_DEBUG_HTTP, t->log, 0, 361 | "remote block cache hit, " 362 | "ns addr: %V, block id: %uD", 363 | &t->name_server_addr_text, 364 | segment_data[j].segment_info.block_id); 365 | 366 | } else { 367 | /* remote block cache invalid, need remove it */ 368 | ngx_http_tfs_remote_block_cache_remove(ctx, t->pool, t->log, 369 | &key); 370 | } 371 | } 372 | } 373 | 374 | if (hit_count > 0) { 375 | ngx_log_debug1(NGX_LOG_DEBUG_HTTP, t->log, 0, 376 | "batch lookup remote block cache, hit_count: %ui", 377 | hit_count); 378 | 379 | if (hit_count == kvs->nelts) { 380 | /* all cache hit, start batch process */ 381 | rc = ngx_http_tfs_batch_process_start(t); 382 | if (rc == NGX_ERROR) { 383 | ngx_http_tfs_finalize_request(t->data, t, 384 | NGX_HTTP_INTERNAL_SERVER_ERROR); 385 | return; 386 | } 387 | return; 388 | } 389 | } 390 | 391 | } else { 392 | ngx_log_debug0(NGX_LOG_DEBUG_HTTP, t->log, 0, 393 | "remote block cache miss"); 394 | } 395 | 396 | ngx_http_tfs_finalize_state(t, NGX_OK); 397 | } 398 | 399 | 400 | #ifdef NGX_HTTP_TFS_USE_TAIR 401 | ngx_int_t 402 | ngx_http_tfs_get_remote_block_cache_instance( 403 | ngx_http_tfs_remote_block_cache_ctx_t *ctx, 404 | ngx_str_t *server_addr) 405 | { 406 | size_t server_addr_len; 407 | uint32_t server_addr_hash; 408 | ngx_int_t rc, i; 409 | ngx_str_t *st; 410 | ngx_array_t config_server; 411 | ngx_http_tfs_t *t; 412 | ngx_http_tfs_tair_instance_t *instance; 413 | ngx_http_etair_server_conf_t *server; 414 | ngx_http_tfs_tair_server_addr_info_t server_addr_info; 415 | 416 | if (server_addr->len == 0 417 | || server_addr->data == NULL) 418 | { 419 | return NGX_ERROR; 420 | } 421 | 422 | t = ctx->data; 423 | server_addr_len = server_addr->len; 424 | server_addr_hash = ngx_murmur_hash2(server_addr->data, server_addr_len); 425 | 426 | instance = ctx->tair_instance; 427 | if (instance->server != NULL) { 428 | if (instance->server_addr_hash == server_addr_hash) { 429 | return NGX_OK; 430 | } 431 | 432 | ngx_http_etair_destory_server(instance->server, 433 | (ngx_cycle_t *) ngx_cycle); 434 | instance->server = NULL; 435 | } 436 | 437 | rc = ngx_http_tfs_parse_tair_server_addr_info(&server_addr_info, 438 | server_addr->data, 439 | server_addr_len, 440 | t->pool, 0); 441 | if (rc == NGX_ERROR) { 442 | return NGX_ERROR; 443 | } 444 | 445 | rc = ngx_array_init(&config_server, t->pool, 446 | NGX_HTTP_TFS_TAIR_CONFIG_SERVER_COUNT, 447 | sizeof(ngx_str_t)); 448 | if (rc == NGX_ERROR) { 449 | return NGX_ERROR; 450 | } 451 | 452 | for (i = 0; i < NGX_HTTP_TFS_TAIR_CONFIG_SERVER_COUNT; i++) { 453 | if (server_addr_info.server[i].len > 0 ) { 454 | st = (ngx_str_t *) ngx_array_push(&config_server); 455 | *st = server_addr_info.server[i]; 456 | } 457 | } 458 | 459 | server = &server_addr_info.server[NGX_HTTP_TFS_TAIR_CONFIG_SERVER_COUNT]; 460 | instance->server = ngx_http_etair_create_server(server, 461 | &config_server, 462 | t->main_conf->tair_timeout, 463 | (ngx_cycle_t *) ngx_cycle); 464 | if (instance->server == NULL) { 465 | return NGX_ERROR; 466 | } 467 | instance->server_addr_hash = server_addr_hash; 468 | instance->area = server_addr_info.area; 469 | 470 | return NGX_OK; 471 | } 472 | 473 | #else 474 | 475 | ngx_int_t 476 | ngx_http_tfs_get_remote_block_cache_instance( 477 | ngx_http_tfs_remote_block_cache_ctx_t *ctx, 478 | ngx_str_t *server_addr) 479 | { 480 | return NGX_ERROR; 481 | } 482 | 483 | #endif 484 | 485 | -------------------------------------------------------------------------------- /src/ngx_http_tfs_remote_block_cache.h: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #ifndef _NGX_HTTP_TFS_REMOTE_BLOCK_CACHE_H_INCLUDED_ 8 | #define _NGX_HTTP_TFS_REMOTE_BLOCK_CACHE_H_INCLUDED_ 9 | 10 | 11 | #include 12 | 13 | 14 | ngx_int_t ngx_http_tfs_remote_block_cache_lookup( 15 | ngx_http_tfs_remote_block_cache_ctx_t *ctx, 16 | ngx_pool_t *pool, ngx_log_t *log, ngx_http_tfs_block_cache_key_t* key); 17 | 18 | ngx_int_t ngx_http_tfs_remote_block_cache_insert( 19 | ngx_http_tfs_remote_block_cache_ctx_t *ctx, 20 | ngx_pool_t *pool, ngx_log_t *log, ngx_http_tfs_block_cache_key_t *key, 21 | ngx_http_tfs_block_cache_value_t *value); 22 | 23 | void ngx_http_tfs_remote_block_cache_remove( 24 | ngx_http_tfs_remote_block_cache_ctx_t *ctx, 25 | ngx_pool_t *pool, ngx_log_t *log, ngx_http_tfs_block_cache_key_t *key); 26 | 27 | ngx_int_t ngx_http_tfs_remote_block_cache_batch_lookup( 28 | ngx_http_tfs_remote_block_cache_ctx_t *ctx, 29 | ngx_pool_t *pool, ngx_log_t *log, ngx_array_t* keys); 30 | 31 | ngx_int_t ngx_http_tfs_remote_block_cache_batch_insert( 32 | ngx_http_tfs_remote_block_cache_ctx_t *ctx, 33 | ngx_pool_t *pool, ngx_log_t *log, ngx_array_t *kvs); 34 | 35 | ngx_int_t ngx_http_tfs_get_remote_block_cache_instance( 36 | ngx_http_tfs_remote_block_cache_ctx_t *ctx, 37 | ngx_str_t *server_addr); 38 | 39 | 40 | #endif /* _NGX_HTTP_TFS_REMOTE_BLOCK_CACHE_H_INCLUDED_ */ 41 | -------------------------------------------------------------------------------- /src/ngx_http_tfs_restful.h: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #ifndef _NGX_HTTP_TFS_RESTFUL_H_INCLUDED_ 8 | #define _NGX_HTTP_TFS_RESTFUL_H_INCLUDED_ 9 | 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | 16 | typedef struct { 17 | ngx_str_t file_path_s; 18 | ngx_str_t file_path_d; 19 | 20 | ngx_str_t file_suffix; 21 | ngx_http_tfs_raw_fsname_t fsname; 22 | 23 | ngx_str_t appkey; 24 | uint64_t app_id; 25 | uint64_t user_id; 26 | 27 | /* action */ 28 | ngx_http_tfs_action_t action; 29 | int64_t offset; 30 | uint64_t size; 31 | 32 | ngx_int_t unlink_type; 33 | ngx_int_t simple_name; 34 | uint8_t version; 35 | uint8_t file_type; 36 | ngx_int_t large_file; 37 | 38 | ngx_int_t read_stat_type; 39 | ngx_int_t write_meta_segment; 40 | ngx_int_t no_dedup; 41 | ngx_int_t chk_file_hole; 42 | 43 | unsigned meta:1; 44 | unsigned get_appid:1; 45 | unsigned chk_exist:1; 46 | unsigned recursive:1; 47 | } ngx_http_tfs_restful_ctx_t; 48 | 49 | 50 | ngx_int_t ngx_http_restful_parse(ngx_http_request_t *r, 51 | ngx_http_tfs_restful_ctx_t *ctx); 52 | 53 | 54 | #endif /* _NGX_HTTP_TFS_RESTFUL_H_INCLUDED_ */ 55 | -------------------------------------------------------------------------------- /src/ngx_http_tfs_root_server_message.c: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #include 8 | #include 9 | 10 | 11 | ngx_chain_t * 12 | ngx_http_tfs_root_server_create_message(ngx_pool_t *pool) 13 | { 14 | ngx_buf_t *b; 15 | ngx_chain_t *cl; 16 | ngx_http_tfs_rs_request_t *req; 17 | 18 | b = ngx_create_temp_buf(pool, sizeof(ngx_http_tfs_rs_request_t)); 19 | if (b == NULL) { 20 | return NULL; 21 | } 22 | 23 | req = (ngx_http_tfs_rs_request_t *) b->pos; 24 | req->header.flag = NGX_HTTP_TFS_PACKET_FLAG; 25 | req->header.len = sizeof(uint8_t); 26 | req->header.type = NGX_HTTP_TFS_REQ_RT_GET_TABLE_MESSAGE; 27 | req->header.version = NGX_HTTP_TFS_PACKET_VERSION; 28 | req->header.crc = ngx_http_tfs_crc(NGX_HTTP_TFS_PACKET_FLAG, 29 | (const char *) (&req->header + 1), 30 | req->header.len); 31 | req->header.id = ngx_http_tfs_generate_packet_id(); 32 | 33 | b->last += sizeof(ngx_http_tfs_rs_request_t); 34 | 35 | cl = ngx_alloc_chain_link(pool); 36 | if (cl == NULL) { 37 | return NULL; 38 | } 39 | 40 | cl->buf = b; 41 | cl->next = NULL; 42 | 43 | return cl; 44 | } 45 | 46 | 47 | ngx_int_t 48 | ngx_http_tfs_root_server_parse_message(ngx_http_tfs_t *t) 49 | { 50 | ngx_int_t rc; 51 | ngx_http_tfs_peer_connection_t *tp; 52 | ngx_http_tfs_rs_response_t *resp; 53 | uLongf table_length; 54 | 55 | tp = t->tfs_peer; 56 | resp = (ngx_http_tfs_rs_response_t *) (tp->body_buffer.pos); 57 | table_length = NGX_HTTP_TFS_METASERVER_COUNT * sizeof(uint64_t); 58 | 59 | rc = uncompress((Bytef *) (t->loc_conf->meta_server_table.table), 60 | &table_length, resp->table, resp->length); 61 | if (rc != Z_OK) { 62 | ngx_log_error(NGX_LOG_ERR, t->log, errno, "uncompress error"); 63 | return NGX_ERROR; 64 | } 65 | 66 | t->loc_conf->meta_server_table.version = resp->version; 67 | 68 | return NGX_OK; 69 | } 70 | 71 | 72 | -------------------------------------------------------------------------------- /src/ngx_http_tfs_root_server_message.h: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #ifndef _NGX_HTTP_TFS_ROOT_SERVER_MESSAGE_H_INCLUDED_ 8 | #define _NGX_HTTP_TFS_ROOT_SERVER_MESSAGE_H_INCLUDED_ 9 | 10 | 11 | #include 12 | 13 | 14 | ngx_chain_t *ngx_http_tfs_root_server_create_message(ngx_pool_t *pool); 15 | ngx_int_t ngx_http_tfs_root_server_parse_message(ngx_http_tfs_t *t); 16 | 17 | 18 | #endif /* _NGX_HTTP_TFS_ROOT_SERVER_MESSAGE_H_INCLUDED_ */ 19 | -------------------------------------------------------------------------------- /src/ngx_http_tfs_server_handler.h: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #ifndef _NGX_HTTP_TFS_SERVER_HANDLER_H_INCLUDED_ 8 | #define _NGX_HTTP_TFS_SERVER_HANDLER_H_INCLUDED_ 9 | 10 | 11 | #include 12 | 13 | 14 | /* root server */ 15 | ngx_int_t ngx_http_tfs_create_rs_request(ngx_http_tfs_t *t); 16 | ngx_int_t ngx_http_tfs_process_rs(ngx_http_tfs_t *t); 17 | 18 | 19 | /* meta server */ 20 | ngx_int_t ngx_http_tfs_create_ms_request(ngx_http_tfs_t *t); 21 | ngx_int_t ngx_http_tfs_process_ms_input_filter(ngx_http_tfs_t *t); 22 | ngx_int_t ngx_http_tfs_process_ms(ngx_http_tfs_t *t); 23 | ngx_int_t ngx_http_tfs_process_ms_ls_dir(ngx_http_tfs_t *t); 24 | 25 | 26 | /* rc server */ 27 | ngx_int_t ngx_http_tfs_create_rcs_request(ngx_http_tfs_t *t); 28 | ngx_int_t ngx_http_tfs_process_rcs(ngx_http_tfs_t *t); 29 | 30 | 31 | /* name server */ 32 | ngx_int_t ngx_http_tfs_create_ns_request(ngx_http_tfs_t *t); 33 | ngx_int_t ngx_http_tfs_process_ns(ngx_http_tfs_t *t); 34 | 35 | 36 | /* data server */ 37 | ngx_int_t ngx_http_tfs_create_ds_request(ngx_http_tfs_t *t); 38 | ngx_int_t ngx_http_tfs_process_ds_input_filter(ngx_http_tfs_t *t); 39 | ngx_int_t ngx_http_tfs_process_ds(ngx_http_tfs_t *t); 40 | ngx_int_t ngx_http_tfs_process_ds_read(ngx_http_tfs_t *t); 41 | 42 | ngx_int_t ngx_http_tfs_retry_ds(ngx_http_tfs_t *t); 43 | ngx_int_t ngx_http_tfs_retry_ns(ngx_http_tfs_t *t); 44 | 45 | 46 | #endif /* */ 47 | 48 | -------------------------------------------------------------------------------- /src/ngx_http_tfs_tair_helper.c: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #include 8 | 9 | 10 | #ifdef NGX_HTTP_TFS_USE_TAIR 11 | 12 | ngx_int_t 13 | ngx_http_tfs_tair_get_helper(ngx_http_tfs_tair_instance_t *instance, 14 | ngx_pool_t *pool, ngx_log_t *log, 15 | ngx_http_tair_data_t *key, ngx_http_tair_get_handler_pt callback, 16 | void *data) 17 | { 18 | ngx_int_t rc; 19 | 20 | if (instance == NULL || key == NULL) { 21 | return NGX_ERROR; 22 | } 23 | 24 | rc = ngx_http_tair_get(instance->server, pool, log, *key, 25 | instance->area, callback, data); 26 | 27 | if (rc != NGX_OK) { 28 | return NGX_ERROR; 29 | } 30 | 31 | return NGX_DECLINED; 32 | } 33 | 34 | 35 | ngx_int_t ngx_http_tfs_tair_mget_helper(ngx_http_tfs_tair_instance_t *instance, 36 | ngx_pool_t *pool, ngx_log_t *log, 37 | ngx_array_t *kvs, ngx_http_tair_mget_handler_pt callback, void *data) 38 | { 39 | ngx_int_t rc; 40 | 41 | if (instance == NULL || kvs == NULL) { 42 | return NGX_ERROR; 43 | } 44 | 45 | rc = ngx_http_tair_mget(instance->server, pool, log, kvs, 46 | instance->area, callback, data); 47 | 48 | if (rc != NGX_OK) { 49 | return NGX_ERROR; 50 | } 51 | 52 | return NGX_DECLINED; 53 | } 54 | 55 | 56 | ngx_int_t 57 | ngx_http_tfs_tair_put_helper(ngx_http_tfs_tair_instance_t *instance, 58 | ngx_pool_t *pool, ngx_log_t *log, 59 | ngx_http_tair_data_t *key, ngx_http_tair_data_t *value, 60 | ngx_int_t expire, ngx_int_t version, 61 | ngx_http_tair_handler_pt callback, void *data) 62 | { 63 | ngx_int_t rc; 64 | 65 | if (instance == NULL || key == NULL || value == NULL) { 66 | return NGX_ERROR; 67 | } 68 | 69 | rc = ngx_http_tair_put(instance->server, pool, log, *key, 70 | *value, instance->area, 0/*nx*/, expire, 71 | version, callback, data); 72 | 73 | if (rc != NGX_OK) { 74 | return NGX_ERROR; 75 | } 76 | 77 | return NGX_DECLINED; 78 | } 79 | 80 | 81 | ngx_int_t 82 | ngx_http_tfs_tair_delete_helper(ngx_http_tfs_tair_instance_t *instance, 83 | ngx_pool_t *pool, ngx_log_t *log, 84 | ngx_array_t *keys, ngx_http_tair_handler_pt callback, void *data) 85 | { 86 | ngx_int_t rc; 87 | 88 | if (instance == NULL || keys == NULL) { 89 | return NGX_ERROR; 90 | } 91 | 92 | rc = ngx_http_tair_delete(instance->server, pool, log, keys, 93 | instance->area, callback, data); 94 | 95 | if (rc != NGX_OK) { 96 | return NGX_ERROR; 97 | } 98 | 99 | return NGX_DECLINED; 100 | } 101 | 102 | #else 103 | 104 | ngx_int_t 105 | ngx_http_tfs_tair_get_helper(ngx_http_tfs_tair_instance_t *instance, 106 | ngx_pool_t *pool, ngx_log_t *log, 107 | ngx_http_tair_data_t *key, ngx_http_tair_get_handler_pt callback, 108 | void *data) 109 | { 110 | return NGX_ERROR; 111 | } 112 | 113 | 114 | ngx_int_t ngx_http_tfs_tair_mget_helper(ngx_http_tfs_tair_instance_t *instance, 115 | ngx_pool_t *pool, ngx_log_t *log, 116 | ngx_array_t *kvs, ngx_http_tair_mget_handler_pt callback, void *data) 117 | { 118 | return NGX_ERROR; 119 | } 120 | 121 | 122 | ngx_int_t 123 | ngx_http_tfs_tair_put_helper(ngx_http_tfs_tair_instance_t *instance, 124 | ngx_pool_t *pool, ngx_log_t *log, 125 | ngx_http_tair_data_t *key, ngx_http_tair_data_t *value, 126 | ngx_int_t expire, ngx_int_t version, 127 | ngx_http_tair_handler_pt callback, void *data) 128 | { 129 | return NGX_ERROR; 130 | } 131 | 132 | 133 | ngx_int_t 134 | ngx_http_tfs_tair_delete_helper(ngx_http_tfs_tair_instance_t *instance, 135 | ngx_pool_t *pool, ngx_log_t *log, 136 | ngx_array_t *keys, ngx_http_tair_handler_pt callback, void *data) 137 | { 138 | return NGX_ERROR; 139 | } 140 | 141 | #endif 142 | 143 | 144 | ngx_int_t 145 | ngx_http_tfs_parse_tair_server_addr_info( 146 | ngx_http_tfs_tair_server_addr_info_t *info, 147 | u_char *addr, uint32_t len, void *pool, uint8_t shared_memory) 148 | { 149 | u_char *temp, *p; 150 | ssize_t info_size; 151 | ngx_int_t i; 152 | 153 | p = addr; 154 | 155 | for (i = 0; i < NGX_HTTP_TFS_TAIR_SERVER_ADDR_PART_COUNT; i++) { 156 | temp = ngx_strlchr(p, p + len, ';'); 157 | if (temp == NULL) { 158 | return NGX_ERROR; 159 | } 160 | 161 | info_size = temp - p; 162 | if (shared_memory) { 163 | info->server[i].data = 164 | ngx_slab_alloc_locked((ngx_slab_pool_t *)pool, info_size); 165 | } else { 166 | info->server[i].data = ngx_pcalloc((ngx_pool_t *)pool, info_size); 167 | } 168 | if (info->server[i].data == NULL) { 169 | return NGX_ERROR; 170 | } 171 | info->server[i].len = info_size; 172 | memcpy(info->server[i].data, p, info_size); 173 | 174 | p += info_size + 1; 175 | len -= (info_size + 1); 176 | if (len <= 0) { 177 | return NGX_ERROR; 178 | } 179 | } 180 | 181 | info->area = ngx_atoi(p, len); 182 | if (info->area == NGX_ERROR) { 183 | return NGX_ERROR; 184 | } 185 | 186 | return NGX_OK; 187 | } 188 | -------------------------------------------------------------------------------- /src/ngx_http_tfs_tair_helper.h: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #ifndef _NGX_HTTP_TFS_TAIR_HELPER_H_INCLUDED_ 8 | #define _NGX_HTTP_TFS_TAIR_HELPER_H_INCLUDED_ 9 | 10 | 11 | #include 12 | #ifdef NGX_HTTP_TFS_USE_TAIR 13 | #include 14 | #endif 15 | 16 | 17 | typedef struct { 18 | uint32_t server_addr_hash; 19 | ngx_int_t area; 20 | #ifdef NGX_HTTP_TFS_USE_TAIR 21 | ngx_http_etair_server_conf_t *server; 22 | #endif 23 | } ngx_http_tfs_tair_instance_t; 24 | 25 | 26 | typedef struct { 27 | ngx_str_t server[NGX_HTTP_TFS_TAIR_SERVER_ADDR_PART_COUNT]; 28 | ngx_int_t area; 29 | } ngx_http_tfs_tair_server_addr_info_t; 30 | 31 | #ifndef NGX_HTTP_TFS_USE_TAIR 32 | 33 | #define NGX_HTTP_TAIR_BYTEARRAY 9 34 | #define NGX_HTTP_TAIR_INT 1 35 | #define NGX_HTTP_ETAIR_SUCCESS 0 36 | 37 | typedef struct { 38 | 39 | size_t len; 40 | u_char *data; 41 | 42 | ngx_uint_t type; 43 | } ngx_http_tair_data_t; 44 | 45 | typedef struct { 46 | ngx_http_tair_data_t key; 47 | ngx_http_tair_data_t *value; 48 | 49 | ngx_int_t version; 50 | ngx_int_t exptime; 51 | 52 | ngx_int_t rc; 53 | } ngx_http_tair_key_value_t; 54 | 55 | typedef void (*ngx_http_tair_handler_pt)(ngx_int_t rc, void *data); 56 | typedef void (*ngx_http_tair_get_handler_pt)(ngx_http_tair_key_value_t *kv, 57 | ngx_int_t rc, void *data); 58 | typedef void (*ngx_http_tair_mget_handler_pt)(ngx_array_t *kvs, ngx_int_t rc, 59 | void *data); 60 | 61 | #endif 62 | 63 | ngx_int_t ngx_http_tfs_tair_get_helper(ngx_http_tfs_tair_instance_t *instance, 64 | ngx_pool_t *pool, ngx_log_t *log, 65 | ngx_http_tair_data_t *key, ngx_http_tair_get_handler_pt callback, 66 | void *data); 67 | 68 | ngx_int_t ngx_http_tfs_tair_mget_helper(ngx_http_tfs_tair_instance_t *instance, 69 | ngx_pool_t *pool, ngx_log_t *log, 70 | ngx_array_t *kvs, ngx_http_tair_mget_handler_pt callback, void *data); 71 | 72 | ngx_int_t 73 | ngx_http_tfs_tair_put_helper(ngx_http_tfs_tair_instance_t *instance, 74 | ngx_pool_t *pool, ngx_log_t *log, 75 | ngx_http_tair_data_t *key, ngx_http_tair_data_t *value, 76 | ngx_int_t expire, ngx_int_t version, 77 | ngx_http_tair_handler_pt callback, void *data); 78 | 79 | ngx_int_t 80 | ngx_http_tfs_tair_delete_helper(ngx_http_tfs_tair_instance_t *instance, 81 | ngx_pool_t *pool, ngx_log_t *log, 82 | ngx_array_t *keys, ngx_http_tair_handler_pt callback, void *data); 83 | 84 | ngx_int_t 85 | ngx_http_tfs_parse_tair_server_addr_info( 86 | ngx_http_tfs_tair_server_addr_info_t *info, 87 | u_char *addr, uint32_t len, void* pool, uint8_t shared_memory); 88 | 89 | 90 | 91 | #endif /* _NGX_HTTP_TFS_TAIR_HELPER_H_INCLUDED_ */ 92 | -------------------------------------------------------------------------------- /src/ngx_http_tfs_timers.c: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #include 8 | #include 9 | 10 | 11 | static void ngx_http_tfs_timeout_handler(ngx_event_t *event); 12 | 13 | 14 | ngx_http_tfs_timers_lock_t * 15 | ngx_http_tfs_timers_init(ngx_cycle_t *cycle, 16 | u_char *lock_file) 17 | { 18 | u_char *shared; 19 | size_t size; 20 | ngx_shm_t shm; 21 | ngx_http_tfs_timers_lock_t *lock; 22 | 23 | /* cl should be equal or bigger than cache line size */ 24 | 25 | size = 128; /* ngx_http_tfs_kp_mutex */ 26 | 27 | shm.size = size; 28 | shm.name.len = sizeof("nginx_tfs_keepalive_zone"); 29 | shm.name.data = (u_char *) "nginx_tfs_keepalive_zone"; 30 | shm.log = cycle->log; 31 | 32 | if (ngx_shm_alloc(&shm) != NGX_OK) { 33 | return NULL; 34 | } 35 | 36 | shared = shm.addr; 37 | 38 | lock = ngx_palloc(cycle->pool, sizeof(ngx_http_tfs_timers_lock_t)); 39 | if (lock == NULL) { 40 | return NULL; 41 | } 42 | 43 | lock->ngx_http_tfs_kp_mutex_ptr = (ngx_atomic_t *) shared; 44 | lock->ngx_http_tfs_kp_mutex.spin = (ngx_uint_t) -1; 45 | 46 | #if defined(nginx_version) && (nginx_version > 1001008) 47 | 48 | if (ngx_shmtx_create(&lock->ngx_http_tfs_kp_mutex, 49 | (ngx_shmtx_sh_t *) shared, lock_file) 50 | != NGX_OK) 51 | { 52 | return NULL; 53 | } 54 | 55 | #else 56 | 57 | if (ngx_shmtx_create(&lock->ngx_http_tfs_kp_mutex, shared, lock_file) 58 | != NGX_OK) 59 | { 60 | return NULL; 61 | } 62 | #endif 63 | 64 | return lock; 65 | } 66 | 67 | 68 | ngx_int_t 69 | ngx_http_tfs_add_rcs_timers(ngx_cycle_t *cycle, 70 | ngx_http_tfs_timers_data_t *data) 71 | { 72 | ngx_event_t *ev; 73 | 74 | ngx_log_debug0(NGX_LOG_DEBUG_HTTP, cycle->log, 0, 75 | "http check tfs rc servers"); 76 | 77 | ev = ngx_pcalloc(cycle->pool, sizeof(ngx_event_t)); 78 | if (ev == NULL) { 79 | return NGX_ERROR; 80 | } 81 | 82 | ev->handler = ngx_http_tfs_timeout_handler; 83 | ev->log = cycle->log; 84 | ev->data = data; 85 | ev->timer_set = 0; 86 | 87 | ngx_add_timer(ev, data->upstream->rcs_interval); 88 | 89 | return NGX_OK; 90 | } 91 | 92 | 93 | ngx_int_t 94 | ngx_http_tfs_timers_finalize_request_handler(ngx_http_tfs_t *t) 95 | { 96 | ngx_event_t *event; 97 | ngx_http_tfs_timers_data_t *data; 98 | 99 | ngx_log_debug0(NGX_LOG_DEBUG_HTTP, t->log, 0, "http tfs timers finalize"); 100 | 101 | event = t->finalize_data; 102 | data = event->data; 103 | 104 | ngx_destroy_pool(t->pool); 105 | ngx_shmtx_unlock(&data->lock->ngx_http_tfs_kp_mutex); 106 | ngx_add_timer(event, data->upstream->rcs_interval); 107 | return NGX_OK; 108 | } 109 | 110 | 111 | static void 112 | ngx_http_tfs_timeout_handler(ngx_event_t *event) 113 | { 114 | ngx_int_t rc; 115 | ngx_pool_t *pool; 116 | ngx_http_tfs_t *t; 117 | ngx_http_request_t *r; 118 | ngx_http_tfs_timers_data_t *data; 119 | 120 | data = event->data; 121 | if (ngx_shmtx_trylock(&data->lock->ngx_http_tfs_kp_mutex)) { 122 | if (ngx_queue_empty(&data->upstream->rc_ctx->sh->kp_queue)) { 123 | ngx_log_debug0(NGX_LOG_DEBUG_EVENT, event->log, 0, 124 | "empty rc keepalive queue"); 125 | ngx_shmtx_unlock(&data->lock->ngx_http_tfs_kp_mutex); 126 | ngx_add_timer(event, data->upstream->rcs_interval); 127 | return; 128 | } 129 | 130 | pool = ngx_create_pool(8192, event->log); 131 | if (pool == NULL) { 132 | ngx_shmtx_unlock(&data->lock->ngx_http_tfs_kp_mutex); 133 | return; 134 | } 135 | 136 | /* fake ngx_http_request_t */ 137 | r = ngx_pcalloc(pool, sizeof(ngx_http_request_t)); 138 | if (r == NULL) { 139 | ngx_shmtx_unlock(&data->lock->ngx_http_tfs_kp_mutex); 140 | return; 141 | } 142 | 143 | r->pool = pool; 144 | r->connection = ngx_pcalloc(pool, sizeof(ngx_connection_t)); 145 | if (r->connection == NULL) { 146 | ngx_shmtx_unlock(&data->lock->ngx_http_tfs_kp_mutex); 147 | return; 148 | } 149 | r->connection->log = event->log; 150 | /* in order to return from ngx_http_run_posted_requests() */ 151 | r->connection->destroyed = 1; 152 | 153 | t = ngx_pcalloc(pool, sizeof(ngx_http_tfs_t)); 154 | if (t == NULL) { 155 | ngx_shmtx_unlock(&data->lock->ngx_http_tfs_kp_mutex); 156 | return; 157 | } 158 | 159 | t->pool = pool; 160 | t->data = r; 161 | t->log = event->log; 162 | t->finalize_request = ngx_http_tfs_timers_finalize_request_handler; 163 | t->finalize_data = event; 164 | 165 | t->r_ctx.action.code = NGX_HTTP_TFS_ACTION_KEEPALIVE; 166 | t->r_ctx.version = 1; 167 | t->loc_conf = ngx_pcalloc(pool, sizeof(ngx_http_tfs_loc_conf_t)); 168 | if (t->loc_conf == NULL) { 169 | ngx_shmtx_unlock(&data->lock->ngx_http_tfs_kp_mutex); 170 | return; 171 | } 172 | t->loc_conf->upstream = data->upstream; 173 | t->main_conf = data->main_conf; 174 | 175 | rc = ngx_http_tfs_init(t); 176 | if (rc == NGX_ERROR) { 177 | ngx_destroy_pool(pool); 178 | ngx_shmtx_unlock(&data->lock->ngx_http_tfs_kp_mutex); 179 | } 180 | 181 | } else { 182 | ngx_log_debug0(NGX_LOG_DEBUG_EVENT, event->log, 0, 183 | "tfs kp mutex lock failed"); 184 | } 185 | } 186 | -------------------------------------------------------------------------------- /src/ngx_http_tfs_timers.h: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #ifndef _NGX_HTTP_TFS_TIMERS_H_INCLUDED_ 8 | #define _NGX_HTTP_TFS_TIMERS_H_INCLUDED_ 9 | 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | 17 | struct ngx_http_tfs_timers_lock_s { 18 | ngx_atomic_t *ngx_http_tfs_kp_mutex_ptr; 19 | ngx_shmtx_t ngx_http_tfs_kp_mutex; 20 | }; 21 | 22 | 23 | struct ngx_http_tfs_timers_data_s { 24 | ngx_http_tfs_main_conf_t *main_conf; 25 | ngx_http_tfs_upstream_t *upstream; 26 | ngx_http_tfs_timers_lock_t *lock; 27 | }; 28 | 29 | ngx_int_t ngx_http_tfs_add_rcs_timers(ngx_cycle_t *cycle, 30 | ngx_http_tfs_timers_data_t *data); 31 | ngx_http_tfs_timers_lock_t *ngx_http_tfs_timers_init(ngx_cycle_t *cycle, 32 | u_char *lock_file); 33 | 34 | 35 | #endif /* _NGX_HTTP_TFS_TIMERS_H_INCLUDED_ */ 36 | -------------------------------------------------------------------------------- /src/ngx_tfs_common.h: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2010-2013 Alibaba Group Holding Limited 4 | */ 5 | 6 | 7 | #ifndef _NGX_HTTP_TFS_COMMON_H_INCLUDED_ 8 | #define _NGX_HTTP_TFS_COMMON_H_INCLUDED_ 9 | 10 | 11 | #include 12 | #include 13 | 14 | #define NGX_HTTP_TFS_HEADER 0 15 | #define NGX_HTTP_TFS_BODY 1 16 | #define NGX_HTTP_TFS_METASERVER_COUNT 10240 17 | 18 | #define NGX_HTTP_TFS_YES 1 19 | #define NGX_HTTP_TFS_NO 0 20 | #define NGX_HTTP_TFS_AGAIN -20 21 | 22 | #define NGX_HTTP_TFS_NGINX_APPKEY "tfs" 23 | #define NGX_HTTP_TFS_DEFAULT_APPID 1 24 | #define NGX_HTTP_TFS_RCS_LOCK_FILE "nginx_rcs.lock" 25 | 26 | #define NGX_HTTP_TFS_MD5_RESULT_LEN 16 27 | #define NGX_HTTP_TFS_DUPLICATE_KEY_SIZE \ 28 | (sizeof(uint32_t) + NGX_HTTP_TFS_MD5_RESULT_LEN) 29 | #define NGX_HTTP_TFS_DUPLICATE_VALUE_BASE_SIZE sizeof(int32_t) 30 | #define NGX_HTTP_TFS_DUPLICATE_INITIAL_MAGIC_VERSION 0x0fffffff 31 | 32 | /* rcs, ns, ds, rs, ms */ 33 | #define NGX_HTTP_TFS_SERVER_COUNT 5 34 | /* master_conifg_server;slave_config_server;group */ 35 | #define NGX_HTTP_TFS_TAIR_SERVER_ADDR_PART_COUNT 3 36 | /* master && slave */ 37 | #define NGX_HTTP_TFS_TAIR_CONFIG_SERVER_COUNT 2 38 | 39 | #define NGX_HTTP_TFS_KEEPALIVE_ACTION "keepalive" 40 | 41 | #define NGX_HTTP_TFS_MAX_READ_FILE_SIZE (512 * 1024) 42 | #define NGX_HTTP_TFS_USE_LARGE_FILE_SIZE (15 * 1024 * 1024) 43 | #define NGX_HTTP_TFS_MAX_SIZE (ULLONG_MAX - 1) 44 | 45 | #define NGX_HTTP_TFS_ZERO_BUF_SIZE (512 * 1024) 46 | #define NGX_HTTP_TFS_INIT_FILE_HOLE_COUNT 5 47 | 48 | #define NGX_HTTP_TFS_APPEND_OFFSET -1 49 | 50 | /* tfs file name standard name length */ 51 | #define NGX_HTTP_TFS_SMALL_FILE_KEY_CHAR 'T' 52 | #define NGX_HTTP_TFS_LARGE_FILE_KEY_CHAR 'L' 53 | #define NGX_HTTP_TFS_FILE_NAME_LEN 18 54 | #define NGX_HTTP_TFS_FILE_NAME_BUFF_LEN 19 55 | #define NGX_HTTP_TFS_FILE_NAME_EXCEPT_SUFFIX_LEN 12 56 | #define NGX_HTTP_TFS_MAX_FILE_NAME_LEN 256 57 | #define NGX_HTTP_TFS_MAX_SUFFIX_LEN 109 /* 128 - 19 */ 58 | 59 | 60 | #define NGX_HTTP_TFS_MAX_CLUSTER_COUNT 10 61 | #define NGX_HTTP_TFS_MAX_CLUSTER_ID_COUNT 5 62 | 63 | #define NGX_HTTP_TFS_CMD_GET_CLUSTER_ID_NS 20 64 | #define NGX_HTTP_TFS_CMD_GET_GROUP_COUNT 22 65 | #define NGX_HTTP_TFS_CMD_GET_GROUP_SEQ 23 66 | 67 | #define NGX_HTTP_TFS_GMT_TIME_SIZE \ 68 | (sizeof("Mon, 28 Sep 1970 06:00:00 UTC+0800") - 1) 69 | 70 | #define NGX_HTTP_TFS_MAX_FRAGMENT_SIZE (2 * 1024 * 1024) 71 | #define NGX_HTTP_TFS_MAX_SEND_FRAG_COUNT 8 72 | 73 | 74 | #define NGX_HTTP_TFS_MUR_HASH_SEED 97 75 | 76 | #define NGX_HTTP_TFS_CLIENT_VERSION "NGINX" 77 | 78 | #define NGX_HTTP_TFS_MIN_TIMER_DELAY 1000 79 | 80 | #define NGX_HTTP_TFS_READ_STAT_NORMAL 0 81 | #define NGX_HTTP_TFS_READ_STAT_FORCE 1 82 | 83 | #define NGX_BSWAP_64(x) \ 84 | ((((x) & 0xff00000000000000ull) >> 56) \ 85 | | (((x) & 0x00ff000000000000ull) >> 40) \ 86 | | (((x) & 0x0000ff0000000000ull) >> 24) \ 87 | | (((x) & 0x000000ff00000000ull) >> 8) \ 88 | | (((x) & 0x00000000ff000000ull) << 8) \ 89 | | (((x) & 0x0000000000ff0000ull) << 24) \ 90 | | (((x) & 0x000000000000ff00ull) << 40) \ 91 | | (((x) & 0x00000000000000ffull) << 56)) 92 | 93 | #define ngx_http_tfs_clear_buf(b) \ 94 | (b)->pos = (b)->start; \ 95 | (b)->last = (b)->start; 96 | 97 | #if (NGX_HAVE_BIG_ENDIAN) 98 | 99 | #define ngx_hton64(x) x 100 | 101 | #define ngx_ntoh64(x) x 102 | 103 | #else 104 | 105 | #define ngx_hton64(x) \ 106 | NGX_BSWAP_64(x) 107 | 108 | #define ngx_ntoh64(x) \ 109 | NGX_BSWAP_64(x) 110 | 111 | 112 | #endif 113 | 114 | typedef struct ngx_http_tfs_s ngx_http_tfs_t; 115 | typedef struct ngx_http_tfs_peer_connection_s ngx_http_tfs_peer_connection_t; 116 | 117 | typedef struct ngx_http_tfs_main_conf_s ngx_http_tfs_main_conf_t; 118 | typedef struct ngx_http_tfs_loc_conf_s ngx_http_tfs_loc_conf_t; 119 | typedef struct ngx_http_tfs_upstream_s ngx_http_tfs_upstream_t; 120 | 121 | typedef struct ngx_http_tfs_inet_s ngx_http_tfs_inet_t; 122 | typedef struct ngx_http_tfs_meta_hh_s ngx_http_tfs_meta_hh_t; 123 | 124 | typedef struct ngx_http_tfs_segment_data_s ngx_http_tfs_segment_data_t; 125 | 126 | typedef struct ngx_http_tfs_timers_lock_s ngx_http_tfs_timers_lock_t; 127 | typedef struct ngx_http_tfs_timers_data_s ngx_http_tfs_timers_data_t; 128 | 129 | typedef struct { 130 | uint64_t size; 131 | } ngx_http_tfs_stat_info_t; 132 | 133 | 134 | typedef struct { 135 | uint32_t crc; 136 | uint32_t data_crc; 137 | } ngx_http_tfs_crc_t; 138 | 139 | 140 | typedef struct { 141 | uint16_t code; 142 | ngx_str_t msg; 143 | } ngx_http_tfs_action_t; 144 | 145 | 146 | struct ngx_http_tfs_meta_hh_s { 147 | uint64_t app_id; 148 | uint64_t user_id; 149 | }; 150 | 151 | 152 | struct ngx_http_tfs_inet_s { 153 | uint32_t ip; 154 | uint32_t port; 155 | }; 156 | 157 | typedef struct { 158 | ngx_http_tfs_inet_t table[NGX_HTTP_TFS_METASERVER_COUNT]; 159 | uint64_t version; 160 | } ngx_http_tfs_meta_table_t; 161 | 162 | 163 | typedef enum { 164 | NGX_HTTP_TFS_RC_SERVER = 0, 165 | NGX_HTTP_TFS_NAME_SERVER, 166 | NGX_HTTP_TFS_DATA_SERVER, 167 | NGX_HTTP_TFS_ROOT_SERVER, 168 | NGX_HTTP_TFS_META_SERVER, 169 | } ngx_http_tfs_peer_server_e; 170 | 171 | 172 | typedef enum { 173 | NGX_HTTP_TFS_STATE_WRITE_START = 0, 174 | NGX_HTTP_TFS_STATE_WRITE_GET_META_TABLE, 175 | NGX_HTTP_TFS_STATE_WRITE_CLUSTER_ID_MS, 176 | NGX_HTTP_TFS_STATE_WRITE_CLUSTER_ID_NS, 177 | NGX_HTTP_TFS_STATE_WRITE_GET_BLK_INFO, 178 | NGX_HTTP_TFS_STATE_WRITE_STAT_DUP_FILE, 179 | NGX_HTTP_TFS_STATE_WRITE_CREATE_FILE_NAME, 180 | NGX_HTTP_TFS_STATE_WRITE_WRITE_DATA, 181 | NGX_HTTP_TFS_STATE_WRITE_CLOSE_FILE, 182 | NGX_HTTP_TFS_STATE_WRITE_WRITE_MS, 183 | NGX_HTTP_TFS_STATE_WRITE_DONE, 184 | } ngx_http_tfs_state_write_e; 185 | 186 | 187 | typedef enum { 188 | NGX_HTTP_TFS_STATE_READ_START = 0, 189 | NGX_HTTP_TFS_STATE_READ_GET_META_TABLE, 190 | NGX_HTTP_TFS_STATE_READ_GET_FRAG_INFO, 191 | NGX_HTTP_TFS_STATE_READ_GET_BLK_INFO, 192 | NGX_HTTP_TFS_STATE_READ_READ_DATA, 193 | NGX_HTTP_TFS_STATE_READ_DONE, 194 | } ngx_http_tfs_state_read_e; 195 | 196 | 197 | typedef enum { 198 | NGX_HTTP_TFS_STATE_REMOVE_START = 0, 199 | NGX_HTTP_TFS_STATE_REMOVE_GET_META_TABLE, 200 | NGX_HTTP_TFS_STATE_REMOVE_GET_FRAG_INFO, 201 | NGX_HTTP_TFS_STATE_REMOVE_GET_GROUP_COUNT, 202 | NGX_HTTP_TFS_STATE_REMOVE_GET_GROUP_SEQ, 203 | NGX_HTTP_TFS_STATE_REMOVE_GET_BLK_INFO, 204 | NGX_HTTP_TFS_STATE_REMOVE_STAT_FILE, 205 | NGX_HTTP_TFS_STATE_REMOVE_READ_META_SEGMENT, 206 | NGX_HTTP_TFS_STATE_REMOVE_DELETE_DATA, 207 | NGX_HTTP_TFS_STATE_REMOVE_NOTIFY_MS, 208 | NGX_HTTP_TFS_STATE_REMOVE_DONE, 209 | } ngx_http_tfs_state_remove_e; 210 | 211 | 212 | typedef enum { 213 | NGX_HTTP_TFS_STATE_STAT_START = 0, 214 | NGX_HTTP_TFS_STATE_STAT_GET_BLK_INFO, 215 | NGX_HTTP_TFS_STATE_STAT_STAT_FILE, 216 | NGX_HTTP_TFS_STATE_STAT_DONE, 217 | } ngx_http_tfs_state_stat_e; 218 | 219 | 220 | typedef enum { 221 | NGX_HTTP_TFS_STATE_ACTION_START = 0, 222 | NGX_HTTP_TFS_STATE_ACTION_GET_META_TABLE, 223 | NGX_HTTP_TFS_STATE_ACTION_PROCESS, 224 | NGX_HTTP_TFS_STATE_ACTION_DONE, 225 | } ngx_http_tfs_state_action_e; 226 | 227 | 228 | static inline uint32_t 229 | ngx_http_tfs_crc(uint32_t crc, const char *data, size_t len) 230 | { 231 | size_t i; 232 | 233 | for (i = 0; i < len; ++i) { 234 | crc = (crc >> 8) ^ ngx_crc32_table256[(crc ^ *data++) & 0xff]; 235 | } 236 | 237 | return crc; 238 | } 239 | 240 | ngx_chain_t *ngx_http_tfs_alloc_chains(ngx_pool_t *pool, size_t count); 241 | void ngx_http_tfs_free_chains(ngx_chain_t **free, ngx_chain_t **out); 242 | 243 | ngx_int_t ngx_http_tfs_test_connect(ngx_connection_t *c); 244 | uint64_t ngx_http_tfs_generate_packet_id(void); 245 | 246 | ngx_int_t ngx_http_tfs_parse_headerin(ngx_http_request_t *r, 247 | ngx_str_t *header_name, ngx_str_t *value); 248 | 249 | ngx_int_t ngx_http_tfs_compute_buf_crc(ngx_http_tfs_crc_t *t_crc, ngx_buf_t *b, 250 | size_t size, ngx_log_t *log); 251 | 252 | ngx_int_t ngx_http_tfs_peer_set_addr(ngx_pool_t *pool, 253 | ngx_http_tfs_peer_connection_t *p, ngx_http_tfs_inet_t *addr); 254 | 255 | uint32_t ngx_http_tfs_murmur_hash(u_char *data, size_t len); 256 | 257 | ngx_int_t ngx_http_tfs_parse_inet(ngx_str_t *u, ngx_http_tfs_inet_t *addr); 258 | int32_t ngx_http_tfs_raw_fsname_hash(const u_char *str, const int32_t len); 259 | ngx_int_t ngx_http_tfs_get_local_ip(ngx_str_t device, struct sockaddr_in *addr); 260 | ngx_buf_t *ngx_http_tfs_copy_buf_chain(ngx_pool_t *pool, ngx_chain_t *in); 261 | ngx_int_t ngx_http_tfs_sum_md5(ngx_chain_t *body, u_char *md5_final, 262 | ssize_t *body_size, ngx_log_t *log); 263 | u_char *ngx_http_tfs_time(u_char *buf, time_t t); 264 | 265 | ngx_int_t ngx_http_tfs_status_message(ngx_buf_t *b, ngx_str_t *action, 266 | ngx_log_t *log); 267 | ngx_int_t ngx_http_tfs_get_parent_dir(ngx_str_t *file_path, 268 | ngx_int_t *dir_level); 269 | ngx_int_t ngx_http_tfs_set_output_file_name(ngx_http_tfs_t *t); 270 | long long ngx_http_tfs_atoll(u_char *line, size_t n); 271 | ngx_int_t ngx_http_tfs_atoull(u_char *line, size_t n, 272 | unsigned long long *value); 273 | void *ngx_http_tfs_prealloc(ngx_pool_t *pool, void *p, size_t old_size, 274 | size_t new_size); 275 | uint64_t ngx_http_tfs_get_chain_buf_size(ngx_chain_t *data); 276 | 277 | void ngx_http_tfs_dump_segment_data(ngx_http_tfs_segment_data_t *segment, 278 | ngx_log_t *log); 279 | ngx_http_tfs_t *ngx_http_tfs_alloc_st(ngx_http_tfs_t *t); 280 | 281 | #define ngx_http_tfs_free_st(t) \ 282 | t->next = t->parent->free_sts; \ 283 | t->parent->free_sts = t; \ 284 | 285 | 286 | #endif /* _NGX_HTTP_TFS_COMMON_H_INCLUDED_ */ 287 | --------------------------------------------------------------------------------