├── .gitignore ├── AUTHORS ├── Changelog.md ├── README.md ├── config ├── include ├── ngx_selective_cache_purge_module.h ├── ngx_selective_cache_purge_module_db.h └── ngx_selective_cache_purge_module_utils.h ├── nginx.conf ├── src ├── ngx_selective_cache_purge_module.c ├── ngx_selective_cache_purge_module_redis.c ├── ngx_selective_cache_purge_module_setup.c ├── ngx_selective_cache_purge_module_sync.c └── ngx_selective_cache_purge_module_utils.c └── test ├── Gemfile ├── Gemfile.lock ├── assets ├── cache.zip ├── cache_2.zip ├── cache_3.zip ├── cache_4.zip └── nginx-test.conf ├── cache_full_spec.rb ├── database_lock_spec.rb ├── module_spec.rb ├── nginx_configuration.rb ├── spec_helper.rb └── sync_memory_spec.rb /.gitignore: -------------------------------------------------------------------------------- 1 | # Object files 2 | *.o 3 | 4 | # Libraries 5 | *.lib 6 | *.a 7 | 8 | # Shared objects (inc. Windows DLLs) 9 | *.dll 10 | *.so 11 | *.so.* 12 | *.dylib 13 | 14 | # Executables 15 | *.exe 16 | *.out 17 | *.app 18 | 19 | # IDE 20 | .cproject 21 | .project 22 | *.sublime* 23 | 24 | # Mac OS X 25 | .DS_Store 26 | 27 | # Build Directory 28 | work/ 29 | !**/work/ 30 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Bruno Torres 2 | Danilo Moret 3 | Wandenberg Peixoto 4 | -------------------------------------------------------------------------------- /Changelog.md: -------------------------------------------------------------------------------- 1 | ### 0.8.0 2 | - Add support to use a redis server protected by password 3 | - Allow usage of multiple cache zones 4 | - Fix support to Nginx 1.9.11+ 5 | 6 | ### 0.7.0 7 | - Fix set initial value for some members in structures 8 | 9 | ### 0.6.1 10 | - Fix structure initialization after a reload in the server 11 | 12 | ### 0.6.0 13 | - Refactor to work in the log phase instead of header filter phase not interfering in the delivery of the content 14 | 15 | ### 0.5.5 16 | - Add support to connect to redis using unix socket 17 | 18 | ### 0.5.4 19 | - Fix scan on redis and purge files operations when purge request is canceled 20 | - Fix to not let worker in starvation when purging multiple files 21 | - Fix cleanup when problems occurs while checking if the file is on cache dir 22 | - Fix build on nginx 1.3.2+ 23 | - Refactor to use redis_nginx_adapter 24 | 25 | ### 0.5.3 26 | - Fix cleanup requests when server is restarting 27 | 28 | ### 0.5.2 29 | - Fix purge files which are on redis but not on nginx memory or cache path, avoiding the cache size be over the limit 30 | 31 | ### 0.5.1 32 | - Stop scan on redis if purge operation is canceled 33 | - Ensure only one purge operation by nginx worker 34 | 35 | ### 0.5.0 36 | - Replace sqlite by redis as db backend using async api 37 | - Fix purge files not on memory to not receive a 'md5 colision' message 38 | - Split memory diff and database store tasks to reduce the number of lock on cache memory 39 | - Compare up to 10000 entries at each interaction to reduce lock time 40 | 41 | ### 0.4.1 42 | - Fix mark old entries when working with multiple zones 43 | 44 | ### 0.4.0 45 | - Remove shared memory lock from select query, do it only if receive a SQLITE_BUSY 46 | - Remove old entries after sync memory with database 47 | - Remove unnecessary 'order by', which was making select slower 48 | 49 | ### 0.3.0 50 | - Fix when try to save on sqlite a cache entry that wasn't actually cached 51 | 52 | ### 0.2.0 53 | - Reduce the time inside a locked area when syncing memory to database 54 | - Fix compilation when --without-http-cache is used 55 | 56 | ### 0.1.2 57 | - Execute select queries in a locked block to avoid receive a SQLITE_BUSY response 58 | 59 | ### 0.1.1 60 | - Forcing sqlite to work on single thread mode 61 | 62 | ### 0.1.0 63 | - Initial release 64 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Nginx Selective Cache Purge Module 2 | ================================== 3 | 4 | A module to purge cache by GLOB patterns. The supported patterns are the same as supported by [Redis](http://redis.io/commands/KEYS). 5 | 6 | _This module is not distributed with the Nginx source. See [the installation instructions](#installation-instructions)._ 7 | 8 | 9 | Configuration 10 | ------------- 11 | 12 | An example: 13 | 14 | ```nginx 15 | pid logs/nginx.pid; 16 | error_log logs/nginx-main_error.log debug; 17 | 18 | # Development Mode 19 | # master_process off; 20 | # daemon off; 21 | worker_processes 1; 22 | worker_rlimit_core 500M; 23 | working_directory /tmp; 24 | debug_points abort; 25 | 26 | events { 27 | worker_connections 1024; 28 | #use kqueue; # MacOS 29 | use epoll; # Linux 30 | } 31 | 32 | http { 33 | default_type application/octet-stream; 34 | 35 | access_log logs/nginx-http_access.log; 36 | error_log logs/nginx-http_error.log; 37 | 38 | proxy_cache_path /tmp/cache_zone levels=1:2 keys_zone=zone:10m inactive=10d max_size=100m; 39 | proxy_cache_path /tmp/cache_other_zone levels=1:2 keys_zone=other_zone:1m inactive=1d max_size=10m; 40 | 41 | #selective_cache_purge_redis_unix_socket "/tmp/redis.sock"; 42 | # 43 | # or 44 | # 45 | #selective_cache_purge_redis_host "localhost"; 46 | #selective_cache_purge_redis_port 6379; 47 | 48 | selective_cache_purge_redis_database 1; 49 | 50 | server { 51 | listen 8080; 52 | server_name localhost; 53 | 54 | # purging by prefix 55 | location ~ /purge(.*) { 56 | selective_cache_purge_query "$1*"; 57 | } 58 | 59 | location / { 60 | proxy_pass http://localhost:8081; 61 | 62 | proxy_cache zone; 63 | proxy_cache_key "$uri"; 64 | proxy_cache_valid 200 1m; 65 | } 66 | } 67 | 68 | server { 69 | listen 8090; 70 | server_name localhost; 71 | 72 | # purging by extension 73 | location ~ /purge/.*(\..*)$ { 74 | #purge by extension 75 | selective_cache_purge_query "*$1"; 76 | } 77 | 78 | location / { 79 | proxy_pass http://localhost:8081; 80 | 81 | proxy_cache other_zone; 82 | proxy_cache_key "$uri"; 83 | proxy_cache_valid 200 1m; 84 | } 85 | } 86 | 87 | server { 88 | listen 8081; 89 | server_name localhost; 90 | 91 | location / { 92 | return 200 "requested url: $uri\n"; 93 | } 94 | } 95 | } 96 | ``` 97 | 98 | Installation instructions 99 | ------------------------- 100 | 101 | This module requires: 102 | - Redis 2.8 or newer. Install it with your favourite package manager - apt-get, yum, brew - or download [Redis](http://redis.io/download) and compile it. 103 | - hiredis 0.11.0. Install it with your favourite package manager - apt-get, yum, brew - or download [hiredis](https://github.com/redis/hiredis/releases) and compile it. 104 | - [redis_nginx_adapter](https://github.com/wandenberg/redis_nginx_adapter) library 105 | 106 | [Download Nginx Stable](http://nginx.org/en/download.html) source and uncompress it. You must then run ./configure with --add-module pointing to this project as usual, referencing the up-to-date hiredis/redis_nginx_adapter lib and include if they are not on your default lib and include folders. Something in the lines of: 107 | 108 | ```bash 109 | $ ./configure \ 110 | --with-ld-opt='-L/usr/lib/ ' \ 111 | --with-cc-opt='-I/usr/include/hiredis/ ' \ 112 | --add-module=/path/to/nginx-selective-cache-purge-module 113 | $ make 114 | $ make install 115 | ``` 116 | 117 | Running tests 118 | ------------- 119 | 120 | This project uses [nginx_test_helper](https://github.com/wandenberg/nginx_test_helper) on the test suite. So, after you've installed the module, you can just install the necessary gems: 121 | 122 | ```bash 123 | $ bundle install --gemfile=test/Gemfile 124 | ``` 125 | 126 | And run rspec pointing to where your Nginx binary is (default: /usr/local/nginx/sbin/nginx): 127 | 128 | ```bash 129 | $ NGINX_EXEC=/path/to/nginx rspec test/ 130 | ``` 131 | 132 | Changelog 133 | --------- 134 | 135 | This is still a work in progress. Be the change. And take a look on the [Changelog](Changelog.md). 136 | -------------------------------------------------------------------------------- /config: -------------------------------------------------------------------------------- 1 | if [ "$HTTP_PROXY" = "YES" ]; then 2 | have=NGX_HTTP_PROXY . auto/have 3 | fi 4 | 5 | if [ "$HTTP_FASTCGI" = "YES" ]; then 6 | have=NGX_HTTP_FASTCGI . auto/have 7 | fi 8 | 9 | if [ "$HTTP_SCGI" = "YES" ]; then 10 | have=NGX_HTTP_SCGI . auto/have 11 | fi 12 | 13 | if [ "$HTTP_UWSGI" = "YES" ]; then 14 | have=NGX_HTTP_UWSGI . auto/have 15 | fi 16 | 17 | ngx_addon_name=ngx_selective_cache_purge_module 18 | ngx_feature_libs="-lhiredis -lredis_nginx_adapter" 19 | HTTP_MODULES="$HTTP_MODULES ${ngx_addon_name}" 20 | CORE_INCS="$CORE_INCS ${ngx_addon_dir}/src ${ngx_addon_dir}/include" 21 | NGX_ADDON_SRCS="$NGX_ADDON_SRCS ${ngx_addon_dir}/src/${ngx_addon_name}.c" 22 | CORE_LIBS="$CORE_LIBS $ngx_feature_libs" 23 | -------------------------------------------------------------------------------- /include/ngx_selective_cache_purge_module.h: -------------------------------------------------------------------------------- 1 | #ifndef _NGX_SELECTIVE_CACHE_PURGE_MODULE_H_ 2 | #define _NGX_SELECTIVE_CACHE_PURGE_MODULE_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | typedef struct { 14 | void *connection; 15 | void *data; 16 | void (*callback) (void *); 17 | void (*err_callback) (void *); 18 | ngx_str_t purge_query; 19 | ngx_queue_t entries; 20 | ngx_pool_t *pool; 21 | ngx_uint_t purging:1; 22 | } ngx_selective_cache_purge_db_ctx_t; 23 | 24 | typedef struct { 25 | ngx_flag_t enabled; 26 | ngx_str_t redis_socket_path; 27 | ngx_str_t redis_host; 28 | ngx_uint_t redis_port; 29 | ngx_uint_t redis_database; 30 | ngx_str_t redis_password; 31 | } ngx_selective_cache_purge_main_conf_t; 32 | 33 | typedef struct { 34 | ngx_http_complex_value_t *purge_query; 35 | } ngx_selective_cache_purge_loc_conf_t; 36 | 37 | typedef struct { 38 | ngx_rbtree_node_t node; 39 | ngx_queue_t queue; 40 | ngx_str_t *zone; 41 | ngx_str_t *type; 42 | ngx_str_t *cache_key; 43 | ngx_str_t *filename; 44 | ngx_str_t *path; 45 | ngx_flag_t removed; 46 | u_char key[NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t)]; 47 | u_char key_dumped[2 * NGX_HTTP_CACHE_KEY_LEN]; 48 | time_t expire; 49 | } ngx_selective_cache_purge_cache_item_t; 50 | 51 | typedef struct { 52 | ngx_flag_t remove_any_entry; 53 | ngx_queue_t queue; 54 | ngx_queue_t *last; 55 | ngx_event_t *purging_files_event; 56 | ngx_event_t *print_result_event; 57 | ngx_selective_cache_purge_db_ctx_t *db_ctx; 58 | } ngx_selective_cache_purge_request_ctx_t; 59 | 60 | typedef struct { 61 | ngx_rbtree_node_t node; 62 | ngx_str_t *name; 63 | ngx_str_t *type; 64 | ngx_shm_zone_t *cache; 65 | ngx_event_t *sync_database_event; 66 | ngx_rbtree_t files_info_tree; 67 | ngx_rbtree_node_t files_info_sentinel; 68 | ngx_queue_t files_info_queue; 69 | ngx_flag_t read_memory; 70 | ngx_uint_t count; 71 | ngx_selective_cache_purge_db_ctx_t *db_ctx; 72 | ngx_http_file_cache_node_t *last; 73 | } ngx_selective_cache_purge_zone_t; 74 | 75 | typedef struct { 76 | ngx_rbtree_t zones_tree; 77 | } ngx_selective_cache_purge_worker_data_t; 78 | 79 | // shared memory 80 | typedef struct { 81 | ngx_atomic_t syncing; 82 | ngx_int_t syncing_slot; 83 | ngx_pid_t syncing_pid; 84 | ngx_int_t syncing_pipe_fd; 85 | ngx_uint_t zones; 86 | ngx_uint_t zones_to_sync; 87 | ngx_queue_t files_info_to_renew_queue; 88 | } ngx_selective_cache_purge_shm_data_t; 89 | 90 | ngx_int_t ngx_selective_cache_purge_indexer_handler(ngx_http_request_t *r); 91 | ngx_int_t ngx_selective_cache_purge_handler(ngx_http_request_t *r); 92 | 93 | ngx_http_output_header_filter_pt ngx_selective_cache_purge_next_header_filter; 94 | 95 | ngx_shm_zone_t *ngx_selective_cache_purge_shm_zone = NULL; 96 | 97 | ngx_selective_cache_purge_worker_data_t *ngx_selective_cache_purge_worker_data = NULL; 98 | 99 | static ngx_str_t ngx_selective_cache_purge_shm_name = ngx_string("selective_cache_purge_module"); 100 | 101 | ngx_selective_cache_purge_db_ctx_t *db_ctxs[NGX_MAX_PROCESSES]; 102 | ngx_queue_t *purge_requests_queue; 103 | 104 | ngx_int_t ngx_selective_cache_purge_sync_memory_to_database(void); 105 | 106 | ngx_int_t ngx_selective_cache_purge_fork_sync_process(void); 107 | ngx_int_t ngx_selective_cache_purge_remove_cache_entry(ngx_http_request_t *r, ngx_selective_cache_purge_cache_item_t *entry, ngx_selective_cache_purge_db_ctx_t *db_ctx); 108 | 109 | static void ngx_selective_cache_purge_cleanup_request_context(ngx_http_request_t *r); 110 | 111 | static ngx_str_t CONTENT_TYPE = ngx_string("text/plain"); 112 | 113 | #define NGX_HTTP_FILE_CACHE_KEY_LEN 6 114 | 115 | #if NGX_HTTP_FASTCGI 116 | extern ngx_module_t ngx_http_fastcgi_module; 117 | static ngx_str_t NGX_SELECTIVE_CACHE_PURGE_FASTCGI_TYPE = ngx_string("fastcgi"); 118 | #endif /* NGX_HTTP_FASTCGI */ 119 | 120 | #if NGX_HTTP_PROXY 121 | extern ngx_module_t ngx_http_proxy_module; 122 | static ngx_str_t NGX_SELECTIVE_CACHE_PURGE_PROXY_TYPE = ngx_string("proxy"); 123 | #endif /* NGX_HTTP_PROXY */ 124 | 125 | #if NGX_HTTP_SCGI 126 | extern ngx_module_t ngx_http_scgi_module; 127 | static ngx_str_t NGX_SELECTIVE_CACHE_PURGE_SCGI_TYPE = ngx_string("scgi"); 128 | #endif /* NGX_HTTP_SCGI */ 129 | 130 | #if NGX_HTTP_UWSGI 131 | extern ngx_module_t ngx_http_uwsgi_module; 132 | static ngx_str_t NGX_SELECTIVE_CACHE_PURGE_UWSGI_TYPE = ngx_string("uwsgi"); 133 | #endif /* NGX_HTTP_UWSGI */ 134 | 135 | 136 | #endif /* _NGX_SELECTIVE_CACHE_PURGE_MODULE_H_ */ 137 | -------------------------------------------------------------------------------- /include/ngx_selective_cache_purge_module_db.h: -------------------------------------------------------------------------------- 1 | #ifndef _NGX_SELECTIVE_CACHE_PURGE_DB_H_ 2 | #define _NGX_SELECTIVE_CACHE_PURGE_DB_H_ 3 | 4 | #include 5 | 6 | ngx_int_t ngx_selective_cache_purge_init_db(ngx_cycle_t *cycle); 7 | ngx_int_t ngx_selective_cache_purge_finish_db(ngx_cycle_t *cycle); 8 | 9 | ngx_int_t ngx_selective_cache_purge_store(ngx_str_t *zone, ngx_str_t *type, ngx_str_t *cache_key, ngx_str_t *filename, time_t expires, ngx_selective_cache_purge_db_ctx_t *db_ctx); 10 | ngx_int_t ngx_selective_cache_purge_remove(ngx_str_t *zone, ngx_str_t *type, ngx_str_t *cache_key, ngx_str_t *filename, ngx_selective_cache_purge_db_ctx_t *db_ctx); 11 | ngx_int_t ngx_selective_cache_purge_barrier_execution(ngx_selective_cache_purge_db_ctx_t *db_ctx); 12 | void ngx_selective_cache_purge_read_all_entires(ngx_selective_cache_purge_db_ctx_t *db_ctx); 13 | void ngx_selective_cache_purge_select_by_cache_key(ngx_selective_cache_purge_db_ctx_t *db_ctx); 14 | 15 | ngx_selective_cache_purge_db_ctx_t *ngx_selective_cache_purge_init_db_context(void); 16 | void ngx_selective_cache_purge_destroy_db_context(ngx_selective_cache_purge_db_ctx_t **db_ctx); 17 | 18 | #endif /* _NGX_SELECTIVE_CACHE_PURGE_DB_H_ */ 19 | -------------------------------------------------------------------------------- /include/ngx_selective_cache_purge_module_utils.h: -------------------------------------------------------------------------------- 1 | #ifndef _NGX_SELECTIVE_CACHE_PURGE_UTILS_H_ 2 | #define _NGX_SELECTIVE_CACHE_PURGE_UTILS_H_ 3 | 4 | #include 5 | 6 | ngx_str_t *ngx_selective_cache_purge_alloc_str(ngx_pool_t *pool, uint len); 7 | static ngx_int_t ngx_selective_cache_purge_send_response_text(ngx_http_request_t *r, const u_char *text, uint len, ngx_flag_t last_buffer); 8 | ngx_int_t ngx_selective_cache_purge_send_response(ngx_http_request_t *r, u_char *data, size_t len, ngx_uint_t status, ngx_str_t *content_type); 9 | ngx_str_t *ngx_selective_cache_purge_get_module_type_by_tag(void *tag); 10 | 11 | ngx_selective_cache_purge_cache_item_t *ngx_selective_cache_purge_file_info_lookup(ngx_rbtree_t *tree, ngx_http_file_cache_node_t *fcn); 12 | 13 | ngx_http_file_cache_node_t *ngx_selective_cache_purge_file_cache_lookup(ngx_http_file_cache_t *cache, u_char *key); 14 | ngx_int_t ngx_selective_cache_purge_file_cache_lookup_on_disk(ngx_http_request_t *r, ngx_http_file_cache_t *cache, ngx_str_t *cache_key, u_char *key); 15 | 16 | void ngx_selective_cache_purge_rbtree_zones_insert(ngx_rbtree_node_t *temp, ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel); 17 | void ngx_selective_cache_purge_rbtree_file_info_insert(ngx_rbtree_node_t *temp, ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel); 18 | 19 | static void ngx_selective_cache_purge_timer_reset(ngx_msec_t timer_interval, ngx_event_t *timer_event); 20 | void ngx_selective_cache_purge_timer_set(ngx_msec_t timer_interval, ngx_event_t *event, ngx_event_handler_pt event_handler, ngx_flag_t start_timer); 21 | 22 | static void ngx_selective_cache_purge_rbtree_walker(ngx_rbtree_t *tree, ngx_rbtree_node_t *node, void *data, ngx_int_t (*apply) (ngx_rbtree_node_t *node, void *data)); 23 | 24 | #endif /* _NGX_SELECTIVE_CACHE_PURGE_UTILS_H_ */ 25 | -------------------------------------------------------------------------------- /nginx.conf: -------------------------------------------------------------------------------- 1 | pid logs/nginx.pid; 2 | error_log logs/nginx-main_error.log debug; 3 | 4 | # Development Mode 5 | master_process off; 6 | daemon off; 7 | worker_processes 1; 8 | worker_rlimit_core 500M; 9 | working_directory /tmp; 10 | debug_points abort; 11 | 12 | events { 13 | worker_connections 1024; 14 | #use kqueue; # MacOS 15 | use epoll; # Linux 16 | } 17 | 18 | http { 19 | default_type application/octet-stream; 20 | 21 | log_format main '["$time_local"] $remote_addr ($status) $body_bytes_sent $request_time $host ' 22 | '$upstream_cache_status $upstream_response_time ' 23 | '$http_x_forwarded_for ["$request"] ["$http_user_agent"] ["$http_referer"] '; 24 | 25 | access_log logs/nginx-http_access.log main; 26 | error_log logs/nginx-http_error.log; 27 | 28 | proxy_cache_path /tmp/cache_zone levels=1:2 keys_zone=zone:10m inactive=1m max_size=100m; 29 | proxy_cache_path /tmp/cache_other_zone levels=1:2 keys_zone=other_zone:1m inactive=10m max_size=10m; 30 | 31 | #selective_cache_purge_redis_unix_socket "/tmp/redis.sock"; 32 | selective_cache_purge_redis_host localhost; 33 | selective_cache_purge_redis_port 6379; 34 | selective_cache_purge_redis_database 4; 35 | 36 | server { 37 | listen 8080; 38 | server_name localhost; 39 | 40 | location ~ /purge(.*) { 41 | selective_cache_purge_query "$1*"; 42 | } 43 | 44 | location / { 45 | add_header "x-cache-status" $upstream_cache_status; 46 | 47 | proxy_pass http://localhost:8081; 48 | 49 | proxy_cache zone; 50 | proxy_cache_key "$uri"; 51 | proxy_cache_valid 200 2m; 52 | proxy_cache_use_stale error timeout invalid_header updating http_500; 53 | } 54 | } 55 | 56 | server { 57 | listen 8090; 58 | server_name localhost; 59 | 60 | location ~ /purge/.*\.(.*)$ { 61 | selective_cache_purge_query "*$1"; 62 | } 63 | 64 | location / { 65 | add_header "x-cache-status" $upstream_cache_status; 66 | 67 | proxy_pass http://localhost:8081; 68 | 69 | proxy_cache other_zone; 70 | proxy_cache_key "$uri"; 71 | proxy_cache_valid 200 2m; 72 | } 73 | } 74 | 75 | server { 76 | listen 8081; 77 | server_name localhost; 78 | 79 | location / { 80 | if ($arg_x = 1) { 81 | return 500 "requested url: $uri\n"; 82 | } 83 | return 200 "requested url: $uri\n"; 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/ngx_selective_cache_purge_module.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | ngx_str_t *ngx_selective_cache_purge_get_cache_key(ngx_http_request_t *r); 7 | void ngx_selective_cache_purge_register_cache_entry(ngx_http_request_t *r, ngx_str_t *cache_key); 8 | ngx_int_t ngx_selective_cache_purge_remove_cache_entry(ngx_http_request_t *r, ngx_selective_cache_purge_cache_item_t *entry, ngx_selective_cache_purge_db_ctx_t *db_ctx); 9 | void ngx_selective_cache_purge_entries_handler(ngx_http_request_t *r); 10 | void ngx_selective_cache_purge_print_result_handler(ngx_http_request_t *r); 11 | void ngx_selective_cache_purge_finalize_request_with_error(ngx_http_request_t *r); 12 | void ngx_selective_cache_purge_send_purge_response(void *d); 13 | static void ngx_selective_cache_purge_force_remove(ngx_http_request_t *r); 14 | ngx_int_t ngx_selective_cache_purge_create_cache_item_for_zone(ngx_rbtree_node_t *v_node, void *data); 15 | static void ngx_selective_cache_purge_deleting_files_timer_handler(ngx_event_t *ev); 16 | static void ngx_selective_cache_purge_print_result_timer_handler(ngx_event_t *ev); 17 | 18 | static ngx_str_t NOT_FOUND_MESSAGE = ngx_string("Could not found any entry that match the expression: %V\n"); 19 | static ngx_str_t OK_MESSAGE = ngx_string("The following entries were purged matched by the expression: %V\n"); 20 | static ngx_str_t CACHE_KEY_FILENAME_SEPARATOR = ngx_string(" -> "); 21 | static ngx_str_t LF_SEPARATOR = ngx_string("\n"); 22 | static ngx_str_t SYNC = ngx_string("sync"); 23 | static ngx_str_t CACHE_KEY = ngx_string("cache_key"); 24 | static ngx_str_t SYNC_OPERATION_START_MESSAGE = ngx_string("Sync operation will be started, wait ...\n"); 25 | static ngx_str_t SYNC_OPERATION_PROGRESS_MESSAGE = ngx_string("Sync operation in progress, wait ...\n"); 26 | static ngx_str_t SYNC_OPERATION_NOT_START_MESSAGE = ngx_string("Sync will NOT be started, check logs.\n"); 27 | static ngx_str_t NOTHING_TO_DO_MESSAGE = ngx_string("Nothing to be done.\n"); 28 | 29 | ngx_int_t 30 | ngx_selective_cache_purge_indexer_handler(ngx_http_request_t *r) 31 | { 32 | ngx_selective_cache_purge_request_ctx_t *ctx = ngx_http_get_module_ctx(r, ngx_selective_cache_purge_module); 33 | 34 | if (ctx == NULL) { 35 | ngx_str_t *cache_key = ngx_selective_cache_purge_get_cache_key(r); 36 | if (cache_key != NULL) { 37 | ngx_selective_cache_purge_register_cache_entry(r, cache_key); 38 | } 39 | } 40 | 41 | return NGX_DECLINED; 42 | } 43 | 44 | 45 | ngx_int_t 46 | ngx_selective_cache_purge_handler(ngx_http_request_t *r) 47 | { 48 | ngx_selective_cache_purge_request_ctx_t *ctx = NULL; 49 | ngx_selective_cache_purge_loc_conf_t *conf = ngx_http_get_module_loc_conf(r, ngx_selective_cache_purge_module); 50 | ngx_str_t vv_purge_query = ngx_null_string, vv_sync = ngx_null_string, vv_cache_key = ngx_null_string, *message; 51 | ngx_pool_cleanup_t *cln; 52 | 53 | if (ngx_http_arg(r, SYNC.data, SYNC.len, &vv_sync) == NGX_OK) { 54 | message = &NOTHING_TO_DO_MESSAGE; 55 | if (ngx_atoi(vv_sync.data, vv_sync.len) == 1) { 56 | switch (ngx_selective_cache_purge_sync_memory_to_database()) { 57 | case NGX_ERROR: 58 | message = &SYNC_OPERATION_NOT_START_MESSAGE; 59 | break; 60 | case NGX_DECLINED: 61 | message = &SYNC_OPERATION_PROGRESS_MESSAGE; 62 | break; 63 | default: 64 | message = &SYNC_OPERATION_START_MESSAGE; 65 | break; 66 | } 67 | } 68 | return ngx_selective_cache_purge_send_response(r, message->data, message->len, NGX_HTTP_OK, &CONTENT_TYPE); 69 | } 70 | 71 | ngx_http_arg(r, CACHE_KEY.data, CACHE_KEY.len, &vv_cache_key); 72 | ngx_http_complex_value(r, conf->purge_query, &vv_purge_query); 73 | if ((vv_purge_query.len == 0) && (vv_cache_key.len == 0)) { 74 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "ngx_selective_cache_purge: purge_query is empty"); 75 | return ngx_selective_cache_purge_send_response(r, NULL, 0, NGX_HTTP_BAD_REQUEST, &CONTENT_TYPE); 76 | } 77 | 78 | if (ngx_http_discard_request_body(r) != NGX_OK) { 79 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "ngx_selective_cache_purge: could not discard body"); 80 | return ngx_selective_cache_purge_send_response(r, NULL, 0, NGX_HTTP_INTERNAL_SERVER_ERROR, &CONTENT_TYPE); 81 | } 82 | 83 | if ((ctx = ngx_pcalloc(r->pool, sizeof(ngx_selective_cache_purge_request_ctx_t))) == NULL) { 84 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "ngx_selective_cache_purge: could not allocate memory to request context"); 85 | return ngx_selective_cache_purge_send_response(r, NULL, 0, NGX_HTTP_INTERNAL_SERVER_ERROR, &CONTENT_TYPE); 86 | } 87 | 88 | if ((ctx->purging_files_event = ngx_pcalloc(r->pool, sizeof(ngx_event_t))) == NULL) { 89 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "ngx_selective_cache_purge: could not allocate memory to purge event"); 90 | return ngx_selective_cache_purge_send_response(r, NULL, 0, NGX_HTTP_INTERNAL_SERVER_ERROR, &CONTENT_TYPE); 91 | } 92 | 93 | if ((ctx->print_result_event = ngx_pcalloc(r->pool, sizeof(ngx_event_t))) == NULL) { 94 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "ngx_selective_cache_purge: could not allocate memory to print result event"); 95 | return ngx_selective_cache_purge_send_response(r, NULL, 0, NGX_HTTP_INTERNAL_SERVER_ERROR, &CONTENT_TYPE); 96 | } 97 | 98 | if ((cln = ngx_pool_cleanup_add(r->pool, 0)) == NULL) { 99 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "ngx_selective_cache_purge: unable to allocate memory for cleanup"); 100 | return ngx_selective_cache_purge_send_response(r, NULL, 0, NGX_HTTP_INTERNAL_SERVER_ERROR, &CONTENT_TYPE); 101 | } 102 | 103 | // set a cleaner to request 104 | cln->handler = (ngx_pool_cleanup_pt) ngx_selective_cache_purge_cleanup_request_context; 105 | cln->data = r; 106 | 107 | ctx->remove_any_entry = 0; 108 | ctx->purging_files_event->data = r; 109 | ctx->print_result_event->data = r; 110 | ngx_queue_init(&ctx->queue); 111 | 112 | ngx_http_set_ctx(r, ctx, ngx_selective_cache_purge_module); 113 | 114 | if ((ctx->db_ctx = ngx_selective_cache_purge_init_db_context()) == NULL) { 115 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "ngx_selective_cache_purge: unable to initialize a db context"); 116 | return ngx_selective_cache_purge_send_response(r, NULL, 0, NGX_HTTP_INTERNAL_SERVER_ERROR, &CONTENT_TYPE); 117 | } 118 | 119 | ctx->db_ctx->data = r; 120 | 121 | r->main->count++; 122 | r->read_event_handler = ngx_http_test_reading; 123 | 124 | if (vv_cache_key.len > 0) { 125 | ctx->db_ctx->purge_query = vv_cache_key; 126 | ngx_selective_cache_purge_force_remove(r); 127 | } else { 128 | ngx_queue_insert_tail(purge_requests_queue, &ctx->queue); 129 | ctx->db_ctx->purge_query = vv_purge_query; 130 | ctx->db_ctx->callback = (void *) ngx_selective_cache_purge_entries_handler; 131 | ctx->db_ctx->err_callback = (void *) ngx_selective_cache_purge_finalize_request_with_error; 132 | if (ngx_queue_head(purge_requests_queue) == &ctx->queue) { 133 | ngx_selective_cache_purge_select_by_cache_key(ctx->db_ctx); 134 | } 135 | } 136 | 137 | return NGX_DONE; 138 | } 139 | 140 | 141 | void 142 | ngx_selective_cache_purge_entries_handler(ngx_http_request_t *r) 143 | { 144 | ngx_selective_cache_purge_request_ctx_t *ctx = ngx_http_get_module_ctx(r, ngx_selective_cache_purge_module); 145 | ngx_selective_cache_purge_cache_item_t *entry; 146 | ngx_queue_t *cur; 147 | ngx_int_t rc; 148 | ngx_int_t processed = 0; 149 | 150 | ctx->db_ctx->callback = NULL; 151 | 152 | # if (NGX_HAVE_FILE_AIO) 153 | if (r->aio) { 154 | return; 155 | } 156 | # endif 157 | 158 | if (!ngx_queue_empty(&ctx->db_ctx->entries)) { 159 | for (cur = (ctx->last == NULL) ? ngx_queue_head(&ctx->db_ctx->entries) : ctx->last; cur != ngx_queue_sentinel(&ctx->db_ctx->entries); cur = ngx_queue_next(cur), processed++) { 160 | entry = ngx_queue_data(cur, ngx_selective_cache_purge_cache_item_t, queue); 161 | if (!entry->removed) { 162 | rc = ngx_selective_cache_purge_remove_cache_entry(r, entry, ctx->db_ctx); 163 | 164 | switch (rc) { 165 | case NGX_OK: 166 | ctx->remove_any_entry = 1; 167 | break; 168 | case NGX_DECLINED: 169 | if (processed >= 50) { 170 | ctx->last = ngx_queue_next(cur); 171 | ngx_selective_cache_purge_timer_set(100, ctx->purging_files_event, ngx_selective_cache_purge_deleting_files_timer_handler, 1); 172 | return; 173 | } 174 | break; 175 | # if (NGX_HAVE_FILE_AIO) 176 | case NGX_AGAIN: 177 | r->write_event_handler = ngx_selective_cache_purge_entries_handler; 178 | return; 179 | # endif 180 | default: 181 | ngx_http_finalize_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR); 182 | return; 183 | } 184 | } 185 | } 186 | } 187 | 188 | ctx->db_ctx->callback = ngx_selective_cache_purge_send_purge_response; 189 | ngx_selective_cache_purge_barrier_execution(ctx->db_ctx); 190 | } 191 | 192 | 193 | void 194 | ngx_selective_cache_purge_send_purge_response(void *d) 195 | { 196 | ngx_http_request_t *r = d; 197 | ngx_selective_cache_purge_request_ctx_t *ctx = ngx_http_get_module_ctx(r, ngx_selective_cache_purge_module); 198 | ngx_str_t *response; 199 | ngx_int_t rc; 200 | 201 | r->write_event_handler = ngx_http_request_empty_handler; 202 | ctx->db_ctx->callback = NULL; 203 | 204 | if (ctx->remove_any_entry) { 205 | if (r->method == NGX_HTTP_HEAD) { 206 | rc = ngx_selective_cache_purge_send_response(r, NULL, 0, NGX_HTTP_OK, &CONTENT_TYPE); 207 | ngx_http_finalize_request(r, rc); 208 | return; 209 | } 210 | 211 | r->headers_out.status = NGX_HTTP_OK; 212 | r->headers_out.content_length_n = -1; 213 | r->headers_out.content_type.data = CONTENT_TYPE.data; 214 | r->headers_out.content_type.len = CONTENT_TYPE.len; 215 | r->headers_out.content_type_len = CONTENT_TYPE.len; 216 | rc = ngx_http_send_header(r); 217 | if (rc == NGX_ERROR || rc > NGX_OK || r->header_only) { 218 | ngx_http_finalize_request(r, rc); 219 | return; 220 | } 221 | 222 | response = ngx_selective_cache_purge_alloc_str(r->pool, ctx->db_ctx->purge_query.len + OK_MESSAGE.len - 2); // -2 for the %V format 223 | ngx_sprintf(response->data, (char *) OK_MESSAGE.data, &ctx->db_ctx->purge_query); 224 | ngx_selective_cache_purge_send_response_text(r, response->data, response->len, 0); 225 | 226 | ngx_selective_cache_purge_print_result_handler(r); 227 | return; 228 | } 229 | 230 | // No entries were found 231 | response = ngx_selective_cache_purge_alloc_str(r->pool, ctx->db_ctx->purge_query.len + NOT_FOUND_MESSAGE.len - 2); // -2 for the %V format 232 | ngx_sprintf(response->data, (char *) NOT_FOUND_MESSAGE.data, &ctx->db_ctx->purge_query); 233 | rc = ngx_selective_cache_purge_send_response(r, response->data, response->len, NGX_HTTP_NOT_FOUND, &CONTENT_TYPE); 234 | ngx_http_finalize_request(r, rc); 235 | } 236 | 237 | 238 | void 239 | ngx_selective_cache_purge_print_result_handler(ngx_http_request_t *r) 240 | { 241 | ngx_selective_cache_purge_request_ctx_t *ctx = ngx_http_get_module_ctx(r, ngx_selective_cache_purge_module); 242 | ngx_selective_cache_purge_cache_item_t *entry; 243 | ngx_queue_t *cur; 244 | ngx_int_t rc = NGX_OK; 245 | ngx_int_t count = 0; 246 | 247 | while (!ngx_queue_empty(&ctx->db_ctx->entries) && (rc == NGX_OK) && (count++ < 500)) { 248 | cur = ngx_queue_head(&ctx->db_ctx->entries); 249 | ngx_queue_remove(cur); 250 | entry = ngx_queue_data(cur, ngx_selective_cache_purge_cache_item_t, queue); 251 | if (entry->removed) { 252 | ngx_selective_cache_purge_send_response_text(r, entry->cache_key->data, entry->cache_key->len, 0); 253 | ngx_selective_cache_purge_send_response_text(r, CACHE_KEY_FILENAME_SEPARATOR.data, CACHE_KEY_FILENAME_SEPARATOR.len, 0); 254 | ngx_selective_cache_purge_send_response_text(r, entry->path->data, entry->path->len, 0); 255 | ngx_selective_cache_purge_send_response_text(r, entry->filename->data, entry->filename->len, 0); 256 | rc = ngx_selective_cache_purge_send_response_text(r, LF_SEPARATOR.data, LF_SEPARATOR.len, 0); 257 | } 258 | } 259 | 260 | if (ngx_queue_empty(&ctx->db_ctx->entries)) { 261 | rc = ngx_selective_cache_purge_send_response_text(r, LF_SEPARATOR.data, LF_SEPARATOR.len, 1); 262 | ngx_http_finalize_request(r, rc); 263 | } else { 264 | ngx_selective_cache_purge_timer_set(50, ctx->print_result_event, ngx_selective_cache_purge_print_result_timer_handler, 1); 265 | } 266 | } 267 | 268 | 269 | void 270 | ngx_selective_cache_purge_finalize_request_with_error(ngx_http_request_t *r) 271 | { 272 | ngx_selective_cache_purge_send_response(r, NULL, 0, NGX_HTTP_INTERNAL_SERVER_ERROR, &CONTENT_TYPE); 273 | } 274 | 275 | 276 | ngx_str_t * 277 | ngx_selective_cache_purge_get_cache_key(ngx_http_request_t *r) 278 | { 279 | ngx_str_t *cache_key = NULL; 280 | 281 | #if NGX_HTTP_CACHE 282 | ngx_uint_t i; 283 | size_t len = 0; 284 | u_char *p = NULL; 285 | ngx_str_t *key = NULL; 286 | 287 | if (r->upstream && (r->upstream->cache_status >= NGX_HTTP_CACHE_MISS) && (r->upstream->cache_status < NGX_HTTP_CACHE_HIT) && 288 | r->cache && (r->cache->node != NULL) && (r->cache->file.name.len > 0)) { 289 | 290 | key = r->cache->keys.elts; 291 | for (i = 0; i < r->cache->keys.nelts; i++) { 292 | len += key[i].len; 293 | } 294 | 295 | if ((cache_key = ngx_selective_cache_purge_alloc_str(r->pool, len)) == NULL) { 296 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "ngx_selective_cache_purge: could not alloc memory to write the cache_key"); 297 | return NULL; 298 | } 299 | 300 | key = r->cache->keys.elts; 301 | p = cache_key->data; 302 | for (i = 0; i < r->cache->keys.nelts; i++) { 303 | p = ngx_copy(p, key[i].data, key[i].len); 304 | } 305 | } 306 | #endif 307 | 308 | return cache_key; 309 | } 310 | 311 | 312 | void 313 | ngx_selective_cache_purge_register_cache_entry(ngx_http_request_t *r, ngx_str_t *cache_key) 314 | { 315 | #if NGX_HTTP_CACHE 316 | ngx_str_t *zone = &r->cache->file_cache->shm_zone->shm.name; 317 | time_t expires = ngx_max(r->cache->node->expire, r->cache->valid_sec); 318 | ngx_str_t *type = ngx_selective_cache_purge_get_module_type_by_tag(r->cache->file_cache->shm_zone->tag); 319 | ngx_str_t *filename = ngx_selective_cache_purge_alloc_str(r->pool, r->cache->file.name.len - r->cache->file_cache->path->name.len); 320 | if ((type != NULL) && (filename != NULL)) { 321 | ngx_memcpy(filename->data, r->cache->file.name.data + r->cache->file_cache->path->name.len, filename->len); 322 | ngx_selective_cache_purge_store(zone, type, cache_key, filename, expires, db_ctxs[ngx_process_slot]); 323 | } 324 | #endif 325 | } 326 | 327 | 328 | ngx_int_t 329 | ngx_selective_cache_purge_remove_cache_entry(ngx_http_request_t *r, ngx_selective_cache_purge_cache_item_t *entry, ngx_selective_cache_purge_db_ctx_t *db_ctx) 330 | { 331 | ngx_selective_cache_purge_zone_t *cache_zone = NULL; 332 | ngx_http_file_cache_t *cache = NULL; 333 | ngx_http_file_cache_node_t *fcn; 334 | u_char key[NGX_HTTP_CACHE_KEY_LEN]; 335 | size_t len = 2 * NGX_HTTP_CACHE_KEY_LEN; 336 | ngx_int_t err = 0; 337 | 338 | /* get cache by zone/type */ 339 | if ((entry->filename == NULL) || 340 | ((cache_zone = ngx_selective_cache_purge_find_zone(entry->zone, entry->type)) == NULL) || 341 | ((cache = (ngx_http_file_cache_t *) cache_zone->cache->data) == NULL)) { 342 | return NGX_DECLINED; 343 | } 344 | 345 | entry->path = &cache->path->name; 346 | 347 | /* restore cache key md5 */ 348 | ngx_selective_cache_purge_hex_read(key, entry->filename->data + entry->filename->len - len, len); 349 | 350 | /* search file cache reference */ 351 | ngx_shmtx_lock(&cache->shpool->mutex); 352 | fcn = ngx_selective_cache_purge_file_cache_lookup(cache, key); 353 | ngx_shmtx_unlock(&cache->shpool->mutex); 354 | 355 | /* try to get the file cache reference forcing the read from disk */ 356 | if ((fcn == NULL) && (r != NULL)) { 357 | if (ngx_selective_cache_purge_file_cache_lookup_on_disk(r, cache, entry->cache_key, key) != NGX_OK) { 358 | if (ngx_errno == NGX_ENOENT) { 359 | ngx_selective_cache_purge_remove(entry->zone, entry->type, entry->cache_key, entry->filename, db_ctx); 360 | } 361 | return NGX_DECLINED; 362 | } 363 | #if NGX_HTTP_CACHE 364 | fcn = r->cache->node; 365 | #endif 366 | } 367 | 368 | if (fcn != NULL) { 369 | ngx_shmtx_lock(&cache->shpool->mutex); 370 | 371 | if (!fcn->exists) { 372 | /* race between concurrent purges, backoff */ 373 | ngx_shmtx_unlock(&cache->shpool->mutex); 374 | if (!fcn->deleting) { 375 | ngx_selective_cache_purge_remove(entry->zone, entry->type, entry->cache_key, entry->filename, db_ctx); 376 | } 377 | return NGX_DECLINED; 378 | } 379 | 380 | cache->sh->size -= fcn->fs_size; 381 | fcn->fs_size = 0; 382 | fcn->exists = 0; 383 | fcn->updating = 0; 384 | fcn->deleting = 1; 385 | 386 | u_char filename_data[entry->path->len + entry->filename->len + 1]; 387 | 388 | ngx_memcpy(filename_data, entry->path->data, entry->path->len); 389 | ngx_memcpy(filename_data + entry->path->len, entry->filename->data, entry->filename->len); 390 | filename_data[entry->path->len + entry->filename->len] = '\0'; 391 | 392 | ngx_shmtx_unlock(&cache->shpool->mutex); 393 | 394 | if (ngx_delete_file(filename_data) == NGX_FILE_ERROR) { 395 | /* entry in error log is enough, don't notice client */ 396 | ngx_log_error(NGX_LOG_CRIT, ngx_cycle->log, ngx_errno, "ngx_selective_cache_purge: "ngx_delete_file_n " \"%s\" failed", filename_data); 397 | err = ngx_errno; 398 | } 399 | 400 | if ((err == 0) || (err == NGX_ENOENT)) { 401 | if (ngx_selective_cache_purge_remove(entry->zone, entry->type, entry->cache_key, entry->filename, db_ctx) == NGX_OK) { 402 | if (err == 0) { 403 | entry->removed = 1; 404 | } 405 | } 406 | } 407 | 408 | ngx_shmtx_lock(&cache->shpool->mutex); 409 | fcn->deleting = 0; 410 | ngx_shmtx_unlock(&cache->shpool->mutex); 411 | 412 | return NGX_OK; 413 | } 414 | 415 | return NGX_DECLINED; 416 | } 417 | 418 | 419 | ngx_int_t 420 | ngx_selective_cache_purge_sync_memory_to_database(void) 421 | { 422 | if (ngx_process == NGX_PROCESS_SINGLE) { 423 | ngx_log_error(NGX_LOG_CRIT, ngx_cycle->log, 0, "ngx_selective_cache_purge: sync process can not be done when running without the loader process"); 424 | return NGX_ERROR; 425 | } 426 | 427 | ngx_selective_cache_purge_shm_data_t *data = (ngx_selective_cache_purge_shm_data_t *) ngx_selective_cache_purge_shm_zone->data; 428 | if (ngx_trylock(&data->syncing)) { 429 | return ngx_selective_cache_purge_fork_sync_process(); 430 | } 431 | return NGX_DECLINED; 432 | } 433 | 434 | 435 | ngx_int_t 436 | ngx_selective_cache_purge_create_cache_item_for_zone(ngx_rbtree_node_t *v_node, void *data) 437 | { 438 | ngx_selective_cache_purge_zone_t *node = (ngx_selective_cache_purge_zone_t *) v_node; 439 | ngx_http_file_cache_t *cache = (ngx_http_file_cache_t *) node->cache->data; 440 | ngx_http_request_t *r = data; 441 | u_char *p; 442 | size_t len = cache->path->name.len + 1 + cache->path->len + 2 * NGX_HTTP_CACHE_KEY_LEN; 443 | u_char filename_data[len + 1]; 444 | ngx_md5_t md5; 445 | u_char key[NGX_HTTP_CACHE_KEY_LEN]; 446 | 447 | ngx_selective_cache_purge_request_ctx_t *ctx = ngx_http_get_module_ctx(r, ngx_selective_cache_purge_module); 448 | ngx_selective_cache_purge_cache_item_t *cur = NULL; 449 | 450 | ngx_md5_init(&md5); 451 | ngx_md5_update(&md5, ctx->db_ctx->purge_query.data, ctx->db_ctx->purge_query.len); 452 | ngx_md5_final(key, &md5); 453 | 454 | ngx_memcpy(filename_data, cache->path->name.data, cache->path->name.len); 455 | p = filename_data + cache->path->name.len + 1 + cache->path->len; 456 | p = ngx_hex_dump(p, key, NGX_HTTP_CACHE_KEY_LEN); 457 | filename_data[len] = '\0'; 458 | 459 | ngx_create_hashed_filename(cache->path, filename_data, len); 460 | 461 | if ((cur = (ngx_selective_cache_purge_cache_item_t *) ngx_palloc(r->pool, sizeof(ngx_selective_cache_purge_cache_item_t))) == NULL) { 462 | ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, 0, "ngx_selective_cache_purge: could not allocate memory to result list"); 463 | return NGX_ERROR; 464 | } 465 | 466 | if ((cur->filename = ngx_selective_cache_purge_alloc_str(r->pool, len - cache->path->name.len)) == NULL) { 467 | ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, 0, "ngx_selective_cache_purge: unable to allocate memory for file info"); 468 | return NGX_ERROR; 469 | } 470 | 471 | ngx_memcpy(cur->filename->data, filename_data + cache->path->name.len, cur->filename->len); 472 | 473 | cur->cache_key = &ctx->db_ctx->purge_query; 474 | cur->zone = node->name; 475 | cur->type = node->type; 476 | cur->path = NULL; 477 | cur->removed = 0; 478 | ngx_queue_insert_tail(&ctx->db_ctx->entries, &cur->queue); 479 | 480 | return NGX_OK; 481 | } 482 | 483 | 484 | static void 485 | ngx_selective_cache_purge_cleanup_request_context(ngx_http_request_t *r) 486 | { 487 | ngx_selective_cache_purge_request_ctx_t *ctx = ngx_http_get_module_ctx(r, ngx_selective_cache_purge_module); 488 | ngx_selective_cache_purge_request_ctx_t *cur; 489 | ngx_queue_t *q; 490 | 491 | if (ctx != NULL) { 492 | ngx_queue_remove(&ctx->queue); 493 | if ((ctx->purging_files_event != NULL) && ctx->purging_files_event->timer_set) { 494 | ngx_del_timer(ctx->purging_files_event); 495 | } 496 | ctx->purging_files_event = NULL; 497 | 498 | if ((ctx->print_result_event != NULL) && ctx->print_result_event->timer_set) { 499 | ngx_del_timer(ctx->print_result_event); 500 | } 501 | ctx->print_result_event = NULL; 502 | 503 | if (ctx->db_ctx->purging && !ngx_queue_empty(purge_requests_queue)) { 504 | q = ngx_queue_head(purge_requests_queue); 505 | cur = ngx_queue_data(q, ngx_selective_cache_purge_request_ctx_t, queue); 506 | ngx_selective_cache_purge_select_by_cache_key(cur->db_ctx); 507 | } 508 | 509 | if (ctx->db_ctx != NULL) { 510 | ctx->db_ctx->data = NULL; 511 | if (ctx->db_ctx->callback == NULL) { 512 | ngx_selective_cache_purge_destroy_db_context(&ctx->db_ctx); 513 | } 514 | } 515 | 516 | ngx_http_set_ctx(r, NULL, ngx_selective_cache_purge_module); 517 | } 518 | } 519 | 520 | 521 | static void 522 | ngx_selective_cache_purge_force_remove(ngx_http_request_t *r) 523 | { 524 | ngx_selective_cache_purge_worker_data_t *data = ngx_selective_cache_purge_worker_data; 525 | 526 | ngx_selective_cache_purge_rbtree_walker(&data->zones_tree, data->zones_tree.root, (void *) r, ngx_selective_cache_purge_create_cache_item_for_zone); 527 | 528 | ngx_selective_cache_purge_entries_handler(r); 529 | } 530 | 531 | 532 | static void 533 | ngx_selective_cache_purge_deleting_files_timer_handler(ngx_event_t *ev) 534 | { 535 | ngx_selective_cache_purge_entries_handler(ev->data); 536 | } 537 | 538 | 539 | static void 540 | ngx_selective_cache_purge_print_result_timer_handler(ngx_event_t *ev) 541 | { 542 | ngx_selective_cache_purge_print_result_handler(ev->data); 543 | } 544 | -------------------------------------------------------------------------------- /src/ngx_selective_cache_purge_module_redis.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | redisAsyncContext *open_context(redisAsyncContext **context); 5 | void scan_callback(redisAsyncContext *c, void *rep, void *privdata); 6 | void scan_by_cache_key_callback(redisAsyncContext *c, void *rep, void *privdata); 7 | ngx_int_t parse_redis_key_to_cache_item(u_char *key, ngx_queue_t *entries, ngx_pool_t *pool); 8 | void select_by_cache_key(ngx_selective_cache_purge_db_ctx_t *db_ctx, char *cursor); 9 | 10 | 11 | #define SCAN_DATABASE_COMMAND "SCAN %s COUNT 100" 12 | #define SCAN_BY_CACHE_KEY_DATABASE_COMMAND "SCAN %s MATCH %b:*:*:* COUNT 100" 13 | #define SET_DATABASE_COMMAND "SETEX %b:%b:%b:%b %d 1" 14 | #define DEL_DATABASE_COMMAND "DEL %b:%b:%b:%b" 15 | #define PING_DATABASE_COMMAND "PING" 16 | 17 | static ngx_str_t REDIS_KEY_PATTERN = ngx_string("^(.*):(.*):(.*):(.*)$"); 18 | static ngx_regex_t *redis_key_regex; 19 | 20 | ngx_int_t 21 | ngx_selective_cache_purge_init_db(ngx_cycle_t *cycle) 22 | { 23 | u_char errstr[NGX_MAX_CONF_ERRSTR]; 24 | ngx_regex_compile_t *rc = NULL; 25 | if ((rc = ngx_pcalloc(cycle->pool, sizeof(ngx_regex_compile_t))) == NULL) { 26 | ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "ngx_selective_cache_purge: unable to allocate memory to compile redis key pattern"); 27 | return NGX_ERROR; 28 | } 29 | 30 | rc->pattern = REDIS_KEY_PATTERN; 31 | rc->pool = cycle->pool; 32 | rc->err.len = NGX_MAX_CONF_ERRSTR; 33 | rc->err.data = errstr; 34 | 35 | if (ngx_regex_compile(rc) != NGX_OK) { 36 | ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "ngx_selective_cache_purge: unable to compile redis key pattern %V", &REDIS_KEY_PATTERN); 37 | return NGX_ERROR; 38 | } 39 | redis_key_regex = rc->regex; 40 | 41 | 42 | if ((db_ctxs[ngx_process_slot] = ngx_calloc(sizeof(ngx_selective_cache_purge_db_ctx_t), cycle->log)) == NULL) { 43 | ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "ngx_selective_cache_purge: unable to allocate memory to db_ctx"); 44 | return NGX_ERROR; 45 | } 46 | 47 | redis_nginx_init(); 48 | 49 | return NGX_OK; 50 | } 51 | 52 | 53 | ngx_int_t 54 | ngx_selective_cache_purge_finish_db(ngx_cycle_t *cycle) 55 | { 56 | ngx_selective_cache_purge_destroy_db_context(&db_ctxs[ngx_process_slot]); 57 | 58 | return NGX_OK; 59 | } 60 | 61 | 62 | void 63 | stub_callback(redisAsyncContext *c, void *rep, void *privdata) 64 | { 65 | ngx_selective_cache_purge_db_ctx_t *db_ctx = privdata; 66 | 67 | if (db_ctx->callback != NULL) { 68 | if (db_ctx->data == NULL) { 69 | ngx_selective_cache_purge_destroy_db_context(&db_ctx); 70 | return; 71 | } 72 | 73 | db_ctx->callback(db_ctx->data); 74 | } 75 | } 76 | 77 | 78 | ngx_int_t 79 | ngx_selective_cache_purge_barrier_execution(ngx_selective_cache_purge_db_ctx_t *db_ctx) 80 | { 81 | redisAsyncContext *c = open_context((redisAsyncContext **) &db_ctx->connection); 82 | if (c == NULL) { 83 | return NGX_ERROR; 84 | } 85 | 86 | redisAsyncCommand(c, stub_callback, db_ctx, PING_DATABASE_COMMAND); 87 | 88 | return NGX_OK; 89 | } 90 | 91 | 92 | ngx_int_t 93 | ngx_selective_cache_purge_store(ngx_str_t *zone, ngx_str_t *type, ngx_str_t *cache_key, ngx_str_t *filename, time_t expires, ngx_selective_cache_purge_db_ctx_t *db_ctx) 94 | { 95 | redisAsyncContext *c = open_context((redisAsyncContext **) &db_ctx->connection); 96 | if (c == NULL) { 97 | return NGX_ERROR; 98 | } 99 | 100 | redisAsyncCommand(c, NULL, NULL, SET_DATABASE_COMMAND, cache_key->data, cache_key->len, zone->data, zone->len, type->data, type->len, filename->data, filename->len, expires - ngx_time()); 101 | 102 | return NGX_OK; 103 | } 104 | 105 | 106 | ngx_int_t 107 | ngx_selective_cache_purge_remove(ngx_str_t *zone, ngx_str_t *type, ngx_str_t *cache_key, ngx_str_t *filename, ngx_selective_cache_purge_db_ctx_t *db_ctx) 108 | { 109 | redisAsyncContext *c = open_context((redisAsyncContext **) &db_ctx->connection); 110 | if (c == NULL) { 111 | return NGX_ERROR; 112 | } 113 | 114 | redisAsyncCommand(c, NULL, NULL, DEL_DATABASE_COMMAND, cache_key->data, cache_key->len, zone->data, zone->len, type->data, type->len, filename->data, filename->len); 115 | 116 | return NGX_OK; 117 | } 118 | 119 | 120 | void 121 | ngx_selective_cache_purge_read_all_entires(ngx_selective_cache_purge_db_ctx_t *db_ctx) 122 | { 123 | redisAsyncContext *c = open_context((redisAsyncContext **) &db_ctx->connection); 124 | if (c == NULL) { 125 | db_ctx->callback(db_ctx->data); 126 | return; 127 | } 128 | 129 | redisAsyncCommand(c, scan_callback, db_ctx, SCAN_DATABASE_COMMAND, "0"); 130 | } 131 | 132 | 133 | void 134 | ngx_selective_cache_purge_select_by_cache_key(ngx_selective_cache_purge_db_ctx_t *db_ctx) 135 | { 136 | redisAsyncContext *c = open_context((redisAsyncContext **) &db_ctx->connection); 137 | if (c == NULL) { 138 | return; 139 | } 140 | 141 | db_ctx->purging = 1; 142 | 143 | redisAsyncCommand(c, scan_by_cache_key_callback, db_ctx, SCAN_BY_CACHE_KEY_DATABASE_COMMAND, "0", db_ctx->purge_query.data, db_ctx->purge_query.len); 144 | } 145 | 146 | 147 | redisAsyncContext * 148 | open_context(redisAsyncContext **context) 149 | { 150 | ngx_selective_cache_purge_main_conf_t *conf = ngx_http_cycle_get_module_main_conf(ngx_cycle, ngx_selective_cache_purge_module); 151 | 152 | if (conf->redis_host.data != NULL) { 153 | return redis_nginx_open_context((const char *) conf->redis_host.data, conf->redis_port, conf->redis_database, (const char *) conf->redis_password.data, context); 154 | } else { 155 | return redis_nginx_open_context_unix((const char *) conf->redis_socket_path.data, conf->redis_database, (const char *) conf->redis_password.data, context); 156 | } 157 | } 158 | 159 | 160 | void 161 | scan_callback(redisAsyncContext *c, void *rep, void *privdata) 162 | { 163 | ngx_selective_cache_purge_db_ctx_t *db_ctx = privdata; 164 | ngx_uint_t i; 165 | 166 | redisReply *reply = rep; 167 | if ((reply == NULL) || (reply->element == NULL)) { 168 | ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, 0, "ngx_selective_cache_purge: empty reply from redis on scan_callback"); 169 | db_ctx->err_callback(db_ctx->data); 170 | return; 171 | } 172 | 173 | for (i = 0; i < reply->element[1]->elements; i++) { 174 | if (parse_redis_key_to_cache_item((u_char *) reply->element[1]->element[i]->str, &db_ctx->entries, db_ctx->pool) != NGX_OK) { 175 | db_ctx->err_callback(db_ctx->data); 176 | return; 177 | } 178 | } 179 | 180 | if (strncmp(reply->element[0]->str, "0", 1) == 0) { 181 | db_ctx->callback(db_ctx->data); 182 | } else { 183 | redisAsyncCommand(c, scan_callback, db_ctx, SCAN_DATABASE_COMMAND, reply->element[0]->str); 184 | } 185 | 186 | } 187 | 188 | 189 | void 190 | scan_by_cache_key_callback(redisAsyncContext *c, void *rep, void *privdata) 191 | { 192 | ngx_selective_cache_purge_db_ctx_t *db_ctx = privdata; 193 | ngx_uint_t i; 194 | redisReply *reply = rep; 195 | 196 | if (db_ctx->data == NULL) { 197 | ngx_selective_cache_purge_destroy_db_context(&db_ctx); 198 | return; 199 | } 200 | 201 | if ((reply == NULL) || (reply->element == NULL)) { 202 | ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, 0, "ngx_selective_cache_purge: empty reply from redis on scan_by_cache_key_callback"); 203 | db_ctx->err_callback(db_ctx->data); 204 | return; 205 | } 206 | 207 | for (i = 0; i < reply->element[1]->elements; i++) { 208 | if (parse_redis_key_to_cache_item((u_char *) reply->element[1]->element[i]->str, &db_ctx->entries, db_ctx->pool) != NGX_OK) { 209 | db_ctx->err_callback(db_ctx->data); 210 | return; 211 | } 212 | } 213 | 214 | if (strncmp(reply->element[0]->str, "0", 1) == 0) { 215 | db_ctx->callback(db_ctx->data); 216 | } else { 217 | redisAsyncCommand(c, scan_by_cache_key_callback, db_ctx, SCAN_BY_CACHE_KEY_DATABASE_COMMAND, reply->element[0]->str, db_ctx->purge_query.data, db_ctx->purge_query.len); 218 | } 219 | } 220 | 221 | 222 | ngx_int_t 223 | parse_redis_key_to_cache_item(u_char *key, ngx_queue_t *entries, ngx_pool_t *pool) 224 | { 225 | ngx_str_t redis_key = ngx_null_string; 226 | int captures[15]; 227 | ngx_selective_cache_purge_cache_item_t *cur = NULL; 228 | 229 | redis_key.data = key; 230 | redis_key.len = ngx_strlen(redis_key.data); 231 | if (ngx_regex_exec(redis_key_regex, &redis_key, captures, 15) != NGX_REGEX_NO_MATCHED) { 232 | if ((cur = (ngx_selective_cache_purge_cache_item_t *) ngx_palloc(pool, sizeof(ngx_selective_cache_purge_cache_item_t))) == NULL) { 233 | ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, 0, "ngx_selective_cache_purge: could not allocate memory to result list"); 234 | return NGX_ERROR; 235 | } 236 | 237 | cur->cache_key = ngx_selective_cache_purge_alloc_str(pool, captures[3]); 238 | cur->zone = ngx_selective_cache_purge_alloc_str(pool, captures[5] - captures[4]); 239 | cur->type = ngx_selective_cache_purge_alloc_str(pool, captures[7] - captures[6]); 240 | cur->filename = ngx_selective_cache_purge_alloc_str(pool, captures[9] - captures[8]); 241 | if ((cur->zone != NULL) && (cur->type != NULL) && (cur->cache_key != NULL) && (cur->filename != NULL)) { 242 | ngx_memcpy(cur->cache_key->data, redis_key.data, cur->cache_key->len); 243 | ngx_memcpy(cur->zone->data, redis_key.data + captures[4], cur->zone->len); 244 | ngx_memcpy(cur->type->data, redis_key.data + captures[6], cur->type->len); 245 | ngx_memcpy(cur->filename->data, redis_key.data + captures[8], cur->filename->len); 246 | cur->path = NULL; 247 | cur->removed = 0; 248 | ngx_queue_insert_tail(entries, &cur->queue); 249 | } else { 250 | ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, 0, "ngx_selective_cache_purge: could not allocate memory to keep a selected item"); 251 | return NGX_ERROR; 252 | } 253 | } 254 | 255 | return NGX_OK; 256 | } 257 | 258 | 259 | ngx_selective_cache_purge_db_ctx_t * 260 | ngx_selective_cache_purge_init_db_context(void) 261 | { 262 | ngx_selective_cache_purge_db_ctx_t *db_ctx; 263 | 264 | if ((db_ctx = ngx_calloc(sizeof(ngx_selective_cache_purge_db_ctx_t), ngx_cycle->log)) != NULL) { 265 | db_ctx->callback = NULL; 266 | db_ctx->err_callback = NULL; 267 | db_ctx->data = NULL; 268 | db_ctx->connection = NULL; 269 | db_ctx->purging = 0; 270 | ngx_str_null(&db_ctx->purge_query); 271 | ngx_queue_init(&db_ctx->entries); 272 | 273 | if ((db_ctx->pool = ngx_create_pool(4096, ngx_cycle->log)) == NULL) { 274 | ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, 0, "ngx_selective_cache_purge: could not allocate memory to db context pool"); 275 | return NULL; 276 | } 277 | } 278 | 279 | return db_ctx; 280 | } 281 | 282 | 283 | void 284 | ngx_selective_cache_purge_destroy_db_context(ngx_selective_cache_purge_db_ctx_t **db_ctx) 285 | { 286 | if (db_ctx && *db_ctx) { 287 | redis_nginx_force_close_context((redisAsyncContext **) &(*db_ctx)->connection); 288 | if ((*db_ctx)->pool) { 289 | ngx_destroy_pool((*db_ctx)->pool); 290 | } 291 | ngx_free(*db_ctx); 292 | *db_ctx = NULL; 293 | } 294 | } 295 | -------------------------------------------------------------------------------- /src/ngx_selective_cache_purge_module_setup.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | static char *ngx_selective_cache_purge(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); 6 | 7 | static ngx_int_t ngx_selective_cache_purge_postconfig(ngx_conf_t *cf); 8 | static void *ngx_selective_cache_purge_create_main_conf(ngx_conf_t *cf); 9 | static char *ngx_selective_cache_purge_init_main_conf(ngx_conf_t *cf, void *parent); 10 | static ngx_int_t ngx_selective_cache_purge_init_worker(ngx_cycle_t *cycle); 11 | static void ngx_selective_cache_purge_exit_worker(ngx_cycle_t *cycle); 12 | static void *ngx_selective_cache_purge_create_loc_conf(ngx_conf_t *cf); 13 | static char *ngx_selective_cache_purge_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child); 14 | 15 | static ngx_int_t ngx_selective_cache_purge_set_up_shm(ngx_conf_t *cf); 16 | static ngx_int_t ngx_selective_cache_purge_init_shm_zone(ngx_shm_zone_t *shm_zone, void *data); 17 | 18 | static ngx_str_t SERVER_IS_RESTARTING_MESSAGE = ngx_string("Server is restarting, try again ...\n"); 19 | 20 | static ngx_command_t ngx_selective_cache_purge_commands[] = { 21 | { ngx_string("selective_cache_purge_redis_unix_socket"), 22 | NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, 23 | ngx_conf_set_str_slot, 24 | NGX_HTTP_MAIN_CONF_OFFSET, 25 | offsetof(ngx_selective_cache_purge_main_conf_t, redis_socket_path), 26 | NULL }, 27 | { ngx_string("selective_cache_purge_redis_host"), 28 | NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, 29 | ngx_conf_set_str_slot, 30 | NGX_HTTP_MAIN_CONF_OFFSET, 31 | offsetof(ngx_selective_cache_purge_main_conf_t, redis_host), 32 | NULL }, 33 | { ngx_string("selective_cache_purge_redis_port"), 34 | NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, 35 | ngx_conf_set_num_slot, 36 | NGX_HTTP_MAIN_CONF_OFFSET, 37 | offsetof(ngx_selective_cache_purge_main_conf_t, redis_port), 38 | NULL }, 39 | { ngx_string("selective_cache_purge_redis_database"), 40 | NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, 41 | ngx_conf_set_num_slot, 42 | NGX_HTTP_MAIN_CONF_OFFSET, 43 | offsetof(ngx_selective_cache_purge_main_conf_t, redis_database), 44 | NULL }, 45 | { ngx_string("selective_cache_purge_redis_password"), 46 | NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, 47 | ngx_conf_set_str_slot, 48 | NGX_HTTP_MAIN_CONF_OFFSET, 49 | offsetof(ngx_selective_cache_purge_main_conf_t, redis_password), 50 | NULL }, 51 | { ngx_string("selective_cache_purge_query"), 52 | NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, 53 | ngx_selective_cache_purge, 54 | NGX_HTTP_LOC_CONF_OFFSET, 55 | offsetof(ngx_selective_cache_purge_loc_conf_t, purge_query), 56 | NULL }, 57 | ngx_null_command 58 | }; 59 | 60 | static ngx_http_module_t ngx_selective_cache_purge_module_ctx = { 61 | NULL, /* preconfiguration */ 62 | ngx_selective_cache_purge_postconfig, /* postconfiguration */ 63 | 64 | ngx_selective_cache_purge_create_main_conf, /* create main configuration */ 65 | ngx_selective_cache_purge_init_main_conf, /* init main configuration */ 66 | 67 | NULL, /* create server configuration */ 68 | NULL, /* merge server configuration */ 69 | 70 | ngx_selective_cache_purge_create_loc_conf, /* create location configuration */ 71 | ngx_selective_cache_purge_merge_loc_conf /* merge location configuration */ 72 | }; 73 | 74 | ngx_module_t ngx_selective_cache_purge_module = { 75 | NGX_MODULE_V1, 76 | &ngx_selective_cache_purge_module_ctx, /* module context */ 77 | ngx_selective_cache_purge_commands, /* module directives */ 78 | NGX_HTTP_MODULE, /* module type */ 79 | NULL, /* init master */ 80 | NULL, /* init module */ 81 | ngx_selective_cache_purge_init_worker, /* init process */ 82 | NULL, /* init thread */ 83 | NULL, /* exit thread */ 84 | ngx_selective_cache_purge_exit_worker, /* exit process */ 85 | NULL, /* exit master */ 86 | NGX_MODULE_V1_PADDING 87 | }; 88 | 89 | 90 | // main config 91 | static void * 92 | ngx_selective_cache_purge_create_main_conf(ngx_conf_t *cf) 93 | { 94 | ngx_selective_cache_purge_main_conf_t *conf = ngx_pcalloc(cf->pool, sizeof(ngx_selective_cache_purge_main_conf_t)); 95 | 96 | if (conf == NULL) { 97 | return NGX_CONF_ERROR; 98 | } 99 | 100 | conf->enabled = 0; 101 | conf->redis_socket_path.data = NULL; 102 | conf->redis_host.data = NULL; 103 | conf->redis_port = NGX_CONF_UNSET_UINT; 104 | conf->redis_database = NGX_CONF_UNSET_UINT; 105 | conf->redis_password.data = NULL; 106 | 107 | return conf; 108 | } 109 | 110 | 111 | static char * 112 | ngx_selective_cache_purge_init_main_conf(ngx_conf_t *cf, void *parent) 113 | { 114 | #ifdef NGX_HTTP_CACHE 115 | ngx_selective_cache_purge_main_conf_t *conf = parent; 116 | 117 | if (conf->redis_host.data != NULL) { 118 | ngx_str_t *redis_host = ngx_selective_cache_purge_alloc_str(cf->pool, conf->redis_host.len); 119 | ngx_snprintf(redis_host->data, conf->redis_host.len, "%V", &conf->redis_host); 120 | conf->redis_host.data = redis_host->data; 121 | 122 | conf->enabled = 1; 123 | } 124 | 125 | if (conf->redis_socket_path.data != NULL) { 126 | ngx_str_t *redis_socket_path = ngx_selective_cache_purge_alloc_str(cf->pool, conf->redis_socket_path.len); 127 | ngx_snprintf(redis_socket_path->data, conf->redis_socket_path.len, "%V", &conf->redis_socket_path); 128 | conf->redis_socket_path.data = redis_socket_path->data; 129 | 130 | conf->enabled = 1; 131 | } 132 | 133 | if (conf->redis_password.data != NULL) { 134 | ngx_str_t *redis_password = ngx_selective_cache_purge_alloc_str(cf->pool, conf->redis_password.len); 135 | ngx_snprintf(redis_password->data, conf->redis_password.len, "%V", &conf->redis_password); 136 | conf->redis_password.data = redis_password->data; 137 | } 138 | 139 | ngx_conf_merge_uint_value(conf->redis_port, conf->redis_port, 6379); 140 | ngx_conf_merge_uint_value(conf->redis_database, conf->redis_database, 0); 141 | #endif 142 | 143 | return NGX_CONF_OK; 144 | } 145 | 146 | 147 | static ngx_int_t 148 | ngx_selective_cache_purge_init_worker(ngx_cycle_t *cycle) 149 | { 150 | ngx_selective_cache_purge_main_conf_t *conf = ngx_http_cycle_get_module_main_conf(cycle, ngx_selective_cache_purge_module); 151 | ngx_rbtree_node_t *sentinel; 152 | ngx_uint_t i, qtd_zones = 0; 153 | size_t shm_size = 0; 154 | ngx_shm_zone_t *shm_zones; 155 | ngx_list_part_t *part; 156 | ngx_selective_cache_purge_zone_t *zone; 157 | 158 | if (!conf->enabled) { 159 | return NGX_OK; 160 | } 161 | 162 | part = (ngx_list_part_t *) &cycle->shared_memory.part; 163 | shm_zones = part->elts; 164 | for (i = 0; /* void */ ; i++) { 165 | if (i >= part->nelts) { 166 | if (part->next == NULL) { 167 | break; 168 | } 169 | part = part->next; 170 | shm_zones = part->elts; 171 | i = 0; 172 | } 173 | 174 | if ((shm_zones[i].tag != NULL) && (ngx_selective_cache_purge_get_module_type_by_tag(shm_zones[i].tag) != NULL)) { 175 | qtd_zones++; 176 | } 177 | } 178 | 179 | shm_size = ngx_align(sizeof(ngx_selective_cache_purge_worker_data_t) + (qtd_zones * ngx_align(sizeof(ngx_selective_cache_purge_zone_t), 256)), ngx_pagesize); 180 | 181 | if ((ngx_selective_cache_purge_worker_data = ngx_pcalloc(cycle->pool, shm_size)) == NULL) { 182 | return NGX_ERROR; 183 | } 184 | 185 | if ((sentinel = ngx_pcalloc(cycle->pool, sizeof(*sentinel))) == NULL) { 186 | return NGX_ERROR; 187 | } 188 | 189 | ngx_rbtree_init(&ngx_selective_cache_purge_worker_data->zones_tree, sentinel, ngx_selective_cache_purge_rbtree_zones_insert); 190 | 191 | part = (ngx_list_part_t *) &cycle->shared_memory.part; 192 | shm_zones = part->elts; 193 | for (i = 0; /* void */ ; i++) { 194 | if (i >= part->nelts) { 195 | if (part->next == NULL) { 196 | break; 197 | } 198 | part = part->next; 199 | shm_zones = part->elts; 200 | i = 0; 201 | } 202 | 203 | if (shm_zones[i].tag != NULL) { 204 | ngx_str_t *type = ngx_selective_cache_purge_get_module_type_by_tag(shm_zones[i].tag); 205 | if (type != NULL) { 206 | if ((zone = ngx_pcalloc(cycle->pool, sizeof(*zone))) == NULL) { 207 | return NGX_ERROR; 208 | } 209 | 210 | zone->sync_database_event = NULL; 211 | zone->cache = &shm_zones[i]; 212 | zone->name = &shm_zones[i].shm.name; 213 | zone->type = type; 214 | zone->node.key = ngx_crc32_short(zone->name->data, zone->name->len); 215 | 216 | ngx_rbtree_insert(&ngx_selective_cache_purge_worker_data->zones_tree, &zone->node); 217 | } 218 | } 219 | } 220 | 221 | if ((ngx_process != NGX_PROCESS_SINGLE) && (ngx_process != NGX_PROCESS_WORKER)) { 222 | return NGX_OK; 223 | } 224 | 225 | if (ngx_selective_cache_purge_init_db(cycle) != NGX_OK) { 226 | return NGX_ERROR; 227 | } 228 | 229 | if ((purge_requests_queue = ngx_pcalloc(cycle->pool, sizeof(ngx_queue_t))) == NULL) { 230 | ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "ngx_selective_cache_purge: could not alloc memory to purge requests queue"); 231 | return NGX_ERROR; 232 | } 233 | ngx_queue_init(purge_requests_queue); 234 | 235 | ngx_selective_cache_purge_sync_memory_to_database(); 236 | 237 | return NGX_OK; 238 | } 239 | 240 | 241 | static void 242 | ngx_selective_cache_purge_exit_worker(ngx_cycle_t *cycle) 243 | { 244 | ngx_selective_cache_purge_main_conf_t *conf = ngx_http_cycle_get_module_main_conf(cycle, ngx_selective_cache_purge_module); 245 | ngx_selective_cache_purge_shm_data_t *data = NULL; 246 | 247 | if (!conf->enabled) { 248 | return; 249 | } 250 | 251 | if ((ngx_process != NGX_PROCESS_SINGLE) && (ngx_process != NGX_PROCESS_WORKER)) { 252 | return; 253 | } 254 | 255 | ngx_queue_t *q; 256 | while (!ngx_queue_empty(purge_requests_queue) && (q = ngx_queue_last(purge_requests_queue))) { 257 | ngx_selective_cache_purge_request_ctx_t *ctx = ngx_queue_data(q, ngx_selective_cache_purge_request_ctx_t, queue); 258 | 259 | ngx_selective_cache_purge_send_response(ctx->db_ctx->data, SERVER_IS_RESTARTING_MESSAGE.data, SERVER_IS_RESTARTING_MESSAGE.len, NGX_HTTP_PRECONDITION_FAILED, &CONTENT_TYPE); 260 | ngx_selective_cache_purge_destroy_db_context(&ctx->db_ctx); 261 | } 262 | 263 | ngx_selective_cache_purge_finish_db(cycle); 264 | 265 | data = (ngx_selective_cache_purge_shm_data_t *) ngx_selective_cache_purge_shm_zone->data; 266 | if ((data->syncing_slot == ngx_process_slot) && (data->syncing_pid != -1)) { 267 | kill(data->syncing_pid, SIGTERM); 268 | } 269 | } 270 | 271 | 272 | static void * 273 | ngx_selective_cache_purge_create_loc_conf(ngx_conf_t *cf) 274 | { 275 | ngx_selective_cache_purge_loc_conf_t *conf; 276 | 277 | conf = ngx_pcalloc(cf->pool, sizeof(ngx_selective_cache_purge_loc_conf_t)); 278 | if (conf == NULL) { 279 | return NGX_CONF_ERROR; 280 | } 281 | 282 | conf->purge_query = NULL; 283 | 284 | return conf; 285 | } 286 | 287 | 288 | static char * 289 | ngx_selective_cache_purge_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) 290 | { 291 | ngx_selective_cache_purge_main_conf_t *mcf = ngx_http_conf_get_module_main_conf(cf, ngx_selective_cache_purge_module); 292 | ngx_selective_cache_purge_loc_conf_t *prev = parent; 293 | ngx_selective_cache_purge_loc_conf_t *conf = child; 294 | 295 | if (conf->purge_query == NULL) { 296 | conf->purge_query = prev->purge_query; 297 | } 298 | 299 | if (!mcf->enabled && (conf->purge_query != NULL)) { 300 | ngx_conf_log_error(NGX_LOG_ERR, cf, 0, "ngx_selective_cache_purge: could not use this module without set a database or compile Nginx with cache support"); 301 | return NGX_CONF_ERROR; 302 | } 303 | 304 | return NGX_CONF_OK; 305 | } 306 | 307 | 308 | static ngx_int_t 309 | ngx_selective_cache_purge_postconfig(ngx_conf_t *cf) 310 | { 311 | ngx_http_handler_pt *h; 312 | ngx_http_core_main_conf_t *cmcf; 313 | 314 | ngx_selective_cache_purge_main_conf_t *conf = ngx_http_conf_get_module_main_conf(cf, ngx_selective_cache_purge_module); 315 | 316 | if (!conf->enabled) { 317 | return NGX_OK; 318 | } 319 | 320 | cmcf = ngx_http_conf_get_module_main_conf(cf, ngx_http_core_module); 321 | 322 | h = ngx_array_push(&cmcf->phases[NGX_HTTP_LOG_PHASE].handlers); 323 | if (h == NULL) { 324 | return NGX_ERROR; 325 | } 326 | 327 | *h = ngx_selective_cache_purge_indexer_handler; 328 | 329 | return ngx_selective_cache_purge_set_up_shm(cf); 330 | } 331 | 332 | 333 | static ngx_int_t 334 | ngx_selective_cache_purge_set_up_shm(ngx_conf_t *cf) 335 | { 336 | ngx_selective_cache_purge_shm_zone = ngx_shared_memory_add(cf, &ngx_selective_cache_purge_shm_name, ngx_align(3 * ngx_pagesize, ngx_pagesize), &ngx_selective_cache_purge_module); 337 | 338 | if (ngx_selective_cache_purge_shm_zone == NULL) { 339 | return NGX_ERROR; 340 | } 341 | 342 | ngx_selective_cache_purge_shm_zone->init = ngx_selective_cache_purge_init_shm_zone; 343 | ngx_selective_cache_purge_shm_zone->data = (void *) 1; 344 | 345 | return NGX_OK; 346 | } 347 | 348 | 349 | static ngx_int_t 350 | ngx_selective_cache_purge_init_shm_zone(ngx_shm_zone_t *shm_zone, void *data) 351 | { 352 | ngx_slab_pool_t *shpool = (ngx_slab_pool_t *) shm_zone->shm.addr; 353 | ngx_selective_cache_purge_shm_data_t *d; 354 | 355 | if (data) { 356 | d = (ngx_selective_cache_purge_shm_data_t *) data; 357 | 358 | if (d->syncing_pid != -1) { 359 | kill(d->syncing_pid, SIGTERM); 360 | } 361 | } else { 362 | if ((d = (ngx_selective_cache_purge_shm_data_t *) ngx_slab_alloc(shpool, sizeof(*d))) == NULL) { 363 | return NGX_ERROR; 364 | } 365 | 366 | d->syncing = 0; 367 | d->syncing_slot = 0; 368 | d->syncing_pid = -1; 369 | d->syncing_pipe_fd = -1; 370 | } 371 | 372 | shm_zone->data = d; 373 | 374 | return NGX_OK; 375 | } 376 | 377 | 378 | static char * 379 | ngx_selective_cache_purge(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) 380 | { 381 | ngx_http_core_loc_conf_t *clcf = ngx_http_conf_get_module_loc_conf(cf, ngx_http_core_module); 382 | char *ret; 383 | 384 | if ((ret = ngx_http_set_complex_value_slot(cf, cmd, conf)) != NGX_CONF_OK) { 385 | return ret; 386 | } 387 | 388 | clcf->handler = ngx_selective_cache_purge_handler; 389 | 390 | return NGX_CONF_OK; 391 | } 392 | -------------------------------------------------------------------------------- /src/ngx_selective_cache_purge_module_sync.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | void ngx_selective_cache_purge_run_sync(void); 5 | void ngx_selective_cache_purge_end_sync(ngx_event_t *ev); 6 | void ngx_selective_cache_purge_sig_handler(int signo); 7 | void ngx_selective_cache_purge_cleanup_sync(ngx_selective_cache_purge_shm_data_t *data); 8 | 9 | ngx_int_t ngx_selective_cache_purge_zone_init(ngx_rbtree_node_t *v_node, void *data); 10 | ngx_int_t ngx_selective_cache_purge_zone_finish(ngx_rbtree_node_t *v_node, void *data); 11 | void ngx_selective_cache_purge_organize_entries(ngx_selective_cache_purge_shm_data_t *data); 12 | void ngx_selective_cache_purge_store_new_entries(void *d); 13 | void ngx_selective_cache_purge_remove_old_entries(void *d); 14 | void ngx_selective_cache_purge_renew_entries(void *d); 15 | 16 | ngx_selective_cache_purge_db_ctx_t *db_ctx = NULL; 17 | 18 | ngx_int_t 19 | ngx_selective_cache_purge_fork_sync_process(void) 20 | { 21 | ngx_selective_cache_purge_shm_data_t *shm_data = (ngx_selective_cache_purge_shm_data_t *) ngx_selective_cache_purge_shm_zone->data; 22 | int pipefd[2]; 23 | int ret; 24 | ngx_pid_t pid; 25 | ngx_event_t *rev; 26 | ngx_connection_t *conn; 27 | 28 | pipefd[0] = -1; 29 | pipefd[1] = -1; 30 | 31 | if (pipe(pipefd) == -1) { 32 | ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, ngx_errno, "ngx_selective_cache_purge: unable to initialize a pipe"); 33 | return NGX_ERROR; 34 | } 35 | 36 | /* make pipe write end survive through exec */ 37 | 38 | ret = fcntl(pipefd[1], F_GETFD); 39 | 40 | if (ret != -1) { 41 | ret &= ~FD_CLOEXEC; 42 | ret = fcntl(pipefd[1], F_SETFD, ret); 43 | } 44 | 45 | if (ret == -1) { 46 | close(pipefd[0]); 47 | close(pipefd[1]); 48 | 49 | ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, ngx_errno, "ngx_selective_cache_purge: unable to make pipe write end live longer"); 50 | return NGX_ERROR; 51 | } 52 | 53 | /* ignore the signal when the child dies */ 54 | signal(SIGCHLD, SIG_IGN); 55 | 56 | pid = fork(); 57 | 58 | switch (pid) { 59 | 60 | case -1: 61 | /* failure */ 62 | if (pipefd[0] != -1) { 63 | close(pipefd[0]); 64 | } 65 | 66 | if (pipefd[1] != -1) { 67 | close(pipefd[1]); 68 | } 69 | 70 | ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, ngx_errno, "ngx_selective_cache_purge: unable to fork the process"); 71 | return NGX_ERROR; 72 | break; 73 | 74 | case 0: 75 | /* child */ 76 | 77 | #if (NGX_LINUX) 78 | prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0); 79 | #endif 80 | if (pipefd[0] != -1) { 81 | close(pipefd[0]); 82 | } 83 | 84 | shm_data->syncing_pipe_fd = pipefd[1]; 85 | ngx_pid = ngx_getpid(); 86 | ngx_setproctitle("cache synchronizer"); 87 | ngx_selective_cache_purge_run_sync(); 88 | break; 89 | 90 | default: 91 | /* parent */ 92 | if (pipefd[1] != -1) { 93 | close(pipefd[1]); 94 | } 95 | 96 | if (pipefd[0] != -1) { 97 | conn = ngx_get_connection(pipefd[0], ngx_cycle->log); 98 | if (conn == NULL) { 99 | ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, ngx_errno, "ngx_selective_cache_purge: failed to add child control event"); 100 | return NGX_ERROR; 101 | } 102 | 103 | conn->data = shm_data; 104 | 105 | rev = conn->read; 106 | rev->handler = ngx_selective_cache_purge_end_sync; 107 | rev->log = ngx_cycle->log; 108 | 109 | if (ngx_add_event(rev, NGX_READ_EVENT, 0) != NGX_OK) { 110 | ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, ngx_errno, "ngx_selective_cache_purge: failed to add child control event"); 111 | } 112 | } 113 | break; 114 | } 115 | 116 | return NGX_OK; 117 | } 118 | 119 | 120 | void 121 | ngx_selective_cache_purge_run_sync(void) 122 | { 123 | ngx_selective_cache_purge_worker_data_t *worker_data = ngx_selective_cache_purge_worker_data; 124 | ngx_selective_cache_purge_shm_data_t *data = (ngx_selective_cache_purge_shm_data_t *) ngx_selective_cache_purge_shm_zone->data; 125 | ngx_uint_t i; 126 | ngx_cycle_t *cycle; 127 | ngx_log_t *log; 128 | ngx_pool_t *pool; 129 | 130 | ngx_done_events((ngx_cycle_t *) ngx_cycle); 131 | 132 | if (signal(SIGTERM, ngx_selective_cache_purge_sig_handler) == SIG_ERR) { 133 | ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, ngx_errno, "ngx_selective_cache_purge: could not set the catch signal for SIGTERM"); 134 | } 135 | 136 | log = ngx_cycle->log; 137 | 138 | pool = ngx_create_pool(NGX_CYCLE_POOL_SIZE, log); 139 | if (pool == NULL) { 140 | exit(1); 141 | } 142 | pool->log = log; 143 | 144 | cycle = ngx_pcalloc(pool, sizeof(ngx_cycle_t)); 145 | if (cycle == NULL) { 146 | ngx_destroy_pool(pool); 147 | exit(1); 148 | } 149 | 150 | cycle->pool = pool; 151 | cycle->log = log; 152 | cycle->new_log.log_level = NGX_LOG_ERR; 153 | cycle->old_cycle = (ngx_cycle_t *) ngx_cycle; 154 | cycle->conf_ctx = ngx_cycle->conf_ctx; 155 | cycle->conf_file = ngx_cycle->conf_file; 156 | cycle->conf_param = ngx_cycle->conf_param; 157 | cycle->conf_prefix = ngx_cycle->conf_prefix; 158 | 159 | cycle->connection_n = 512; 160 | #if (nginx_version >= 1009011) 161 | cycle->modules = ngx_modules; 162 | #endif 163 | 164 | 165 | ngx_process = NGX_PROCESS_HELPER; 166 | 167 | for (i = 0; ngx_modules[i]; i++) { 168 | if ((ngx_modules[i]->type == NGX_EVENT_MODULE) && ngx_modules[i]->init_process) { 169 | if (ngx_modules[i]->init_process(cycle) == NGX_ERROR) { 170 | /* fatal */ 171 | exit(2); 172 | } 173 | } 174 | } 175 | 176 | ngx_close_listening_sockets(cycle); 177 | 178 | ngx_cycle = cycle; 179 | 180 | ngx_log_error(NGX_LOG_NOTICE, ngx_cycle->log, 0, "ngx_selective_cache_purge: sync process started"); 181 | 182 | if ((db_ctx = ngx_selective_cache_purge_init_db_context()) == NULL) { 183 | ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, 0, "ngx_selective_cache_purge: unable to allocate memory to sync db_ctx"); 184 | exit(1); 185 | } 186 | data->zones = 0; 187 | data->zones_to_sync = 0; 188 | data->syncing_slot = ngx_process_slot; 189 | data->syncing_pid = ngx_pid; 190 | ngx_queue_init(&data->files_info_to_renew_queue); 191 | 192 | ngx_selective_cache_purge_rbtree_walker(&worker_data->zones_tree, worker_data->zones_tree.root, data, ngx_selective_cache_purge_zone_init); 193 | 194 | db_ctx->data = data; 195 | db_ctx->callback = (void *) ngx_selective_cache_purge_organize_entries; 196 | ngx_selective_cache_purge_read_all_entires(db_ctx); 197 | 198 | for ( ;; ) { 199 | ngx_process_events_and_timers(cycle); 200 | } 201 | } 202 | 203 | 204 | void 205 | ngx_selective_cache_purge_end_sync(ngx_event_t *ev) 206 | { 207 | ngx_connection_t *c = ev->data ; 208 | ngx_selective_cache_purge_shm_data_t *data = c->data; 209 | 210 | data->syncing_pid = -1; 211 | ngx_unlock(&data->syncing); 212 | ngx_close_connection(c); 213 | } 214 | 215 | 216 | void 217 | ngx_selective_cache_purge_sig_handler(int signo) 218 | { 219 | ngx_selective_cache_purge_shm_data_t *data = (ngx_selective_cache_purge_shm_data_t *) ngx_selective_cache_purge_shm_zone->data; 220 | if (signo == SIGTERM) { 221 | ngx_selective_cache_purge_cleanup_sync(data); 222 | } 223 | } 224 | 225 | 226 | void 227 | ngx_selective_cache_purge_cleanup_sync(ngx_selective_cache_purge_shm_data_t *data) 228 | { 229 | ngx_uint_t i; 230 | ngx_connection_t *c; 231 | ngx_selective_cache_purge_worker_data_t *worker_data = ngx_selective_cache_purge_worker_data; 232 | 233 | if (data->syncing_pipe_fd != -1) { 234 | close(data->syncing_pipe_fd); 235 | data->syncing_pipe_fd = -1; 236 | } 237 | ngx_selective_cache_purge_rbtree_walker(&worker_data->zones_tree, worker_data->zones_tree.root, data, ngx_selective_cache_purge_zone_finish); 238 | ngx_selective_cache_purge_destroy_db_context(&db_ctx); 239 | c = ngx_cycle->connections; 240 | 241 | for (i = 0; i < ngx_cycle->connection_n; i++) { 242 | if (c[i].fd != -1) { 243 | ngx_close_connection(&c[i]); 244 | } 245 | } 246 | 247 | ngx_done_events((ngx_cycle_t *) ngx_cycle); 248 | exit(0); 249 | } 250 | 251 | 252 | ngx_int_t 253 | ngx_selective_cache_purge_zone_init(ngx_rbtree_node_t *v_node, void *data) 254 | { 255 | ngx_selective_cache_purge_shm_data_t *d = (ngx_selective_cache_purge_shm_data_t *) data; 256 | ngx_selective_cache_purge_zone_t *node = (ngx_selective_cache_purge_zone_t *) v_node; 257 | 258 | ngx_rbtree_init(&node->files_info_tree, &node->files_info_sentinel, ngx_selective_cache_purge_rbtree_file_info_insert); 259 | ngx_queue_init(&node->files_info_queue); 260 | 261 | d->zones++; 262 | d->zones_to_sync++; 263 | node->count = 0; 264 | node->read_memory = 1; 265 | 266 | if ((node->db_ctx = ngx_selective_cache_purge_init_db_context()) == NULL) { 267 | ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, 0, "ngx_selective_cache_purge: unable to allocate memory for sync db context"); 268 | return NGX_ERROR; 269 | } 270 | 271 | if ((node->sync_database_event = ngx_pcalloc(node->db_ctx->pool, sizeof(ngx_event_t))) == NULL) { 272 | ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, 0, "ngx_selective_cache_purge: unable to allocate memory for sync database event"); 273 | return NGX_ERROR; 274 | } 275 | node->sync_database_event->data = node; 276 | return NGX_OK; 277 | } 278 | 279 | 280 | ngx_int_t 281 | ngx_selective_cache_purge_zone_finish(ngx_rbtree_node_t *v_node, void *data) 282 | { 283 | ngx_selective_cache_purge_zone_t *node = (ngx_selective_cache_purge_zone_t *) v_node; 284 | 285 | ngx_rbtree_init(&node->files_info_tree, &node->files_info_sentinel, ngx_selective_cache_purge_rbtree_file_info_insert); 286 | ngx_queue_init(&node->files_info_queue); 287 | 288 | if ((node->sync_database_event != NULL) && node->sync_database_event->active) { 289 | ngx_del_timer(node->sync_database_event); 290 | } 291 | 292 | ngx_selective_cache_purge_destroy_db_context(&node->db_ctx); 293 | node->sync_database_event = NULL; 294 | 295 | return NGX_OK; 296 | } 297 | 298 | 299 | static void 300 | ngx_selective_cache_purge_sync_database_timer_wake_handler(ngx_event_t *ev) 301 | { 302 | ngx_selective_cache_purge_shm_data_t *data = (ngx_selective_cache_purge_shm_data_t *) ngx_selective_cache_purge_shm_zone->data; 303 | ngx_selective_cache_purge_zone_t *node = (ngx_selective_cache_purge_zone_t *) ev->data; 304 | ngx_http_file_cache_t *cache = (ngx_http_file_cache_t *) node->cache->data; 305 | ngx_http_file_cache_node_t *fcn; 306 | ngx_queue_t *q; 307 | u_char *p; 308 | ngx_flag_t loading = 0; 309 | ngx_uint_t count = 0; 310 | 311 | if (ngx_exiting || (data == NULL) || (cache == NULL)) { 312 | return; 313 | } 314 | 315 | ngx_log_error(NGX_LOG_DEBUG, ngx_cycle->log, 0, "ngx_selective_cache_purge: start a cycle of sync for zone %V", node->name); 316 | 317 | ngx_shmtx_lock(&cache->shpool->mutex); 318 | loading = cache->sh->cold || cache->sh->loading; 319 | for (q = ngx_queue_head(&cache->sh->queue); node->read_memory && (q != ngx_queue_sentinel(&cache->sh->queue)); q = ngx_queue_next(q)) { 320 | fcn = ngx_queue_data(q, ngx_http_file_cache_node_t, queue); 321 | 322 | if (loading && (node->last != NULL) && (node->last < fcn)) { 323 | continue; 324 | } 325 | 326 | node->last = fcn; 327 | if (loading && (count++ >= 10000)) { 328 | break; 329 | } 330 | 331 | ngx_selective_cache_purge_cache_item_t *ci = NULL; 332 | if ((ci = ngx_selective_cache_purge_file_info_lookup(&node->files_info_tree, fcn)) == NULL) { 333 | if ((ci = ngx_pcalloc(db_ctx->pool, sizeof(ngx_selective_cache_purge_cache_item_t))) == NULL) { 334 | ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, 0, "ngx_selective_cache_purge: unable to allocate memory for file info"); 335 | break; 336 | } 337 | 338 | ci->zone = node->name; 339 | ci->type = node->type; 340 | ci->filename = NULL; 341 | ci->cache_key = NULL; 342 | ci->expire = fcn->expire; 343 | p = ngx_hex_dump(ci->key_dumped, (u_char *) &fcn->node.key, sizeof(ngx_rbtree_key_t)); 344 | p = ngx_hex_dump(p, fcn->key, NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t)); 345 | ngx_queue_insert_tail(&node->files_info_queue, &ci->queue); 346 | 347 | ngx_memcpy(&ci->node.key, &fcn->node.key, sizeof(ngx_rbtree_key_t)); 348 | ngx_memcpy(&ci->key, &fcn->key, NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t)); 349 | ngx_rbtree_insert(&node->files_info_tree, &ci->node); 350 | node->count++; 351 | } else if (!loading && (ci->expire < 0)) { 352 | ci->expire = fcn->expire; 353 | ngx_rbtree_delete(&node->files_info_tree, &ci->node); 354 | ngx_queue_remove(&ci->queue); 355 | ngx_queue_insert_tail(&data->files_info_to_renew_queue, &ci->queue); 356 | } 357 | } 358 | node->read_memory = loading; 359 | ngx_shmtx_unlock(&cache->shpool->mutex); 360 | 361 | ngx_selective_cache_purge_store_new_entries(node); 362 | } 363 | 364 | 365 | ngx_int_t 366 | ngx_selective_cache_purge_start_sync_database_timer(ngx_rbtree_node_t *v_node, void *data) 367 | { 368 | ngx_selective_cache_purge_zone_t *node = (ngx_selective_cache_purge_zone_t *) v_node; 369 | ngx_http_file_cache_t *cache = (ngx_http_file_cache_t *) node->cache->data; 370 | 371 | ngx_selective_cache_purge_timer_set(cache->loader_sleep * 1.5, node->sync_database_event, ngx_selective_cache_purge_sync_database_timer_wake_handler, 1); 372 | return NGX_OK; 373 | } 374 | 375 | 376 | void 377 | ngx_selective_cache_purge_organize_entries(ngx_selective_cache_purge_shm_data_t *data) 378 | { 379 | ngx_selective_cache_purge_worker_data_t *worker_data = ngx_selective_cache_purge_worker_data; 380 | ngx_selective_cache_purge_zone_t *node = NULL; 381 | ngx_http_file_cache_t *cache = NULL; 382 | ngx_queue_t *q; 383 | ngx_md5_t md5; 384 | u_char key[NGX_HTTP_CACHE_KEY_LEN]; 385 | 386 | for (q = ngx_queue_last(&db_ctx->entries); q != ngx_queue_sentinel(&db_ctx->entries); q = ngx_queue_prev(q)) { 387 | ngx_selective_cache_purge_cache_item_t *ci = ngx_queue_data(q, ngx_selective_cache_purge_cache_item_t, queue); 388 | 389 | if ((node = ngx_selective_cache_purge_find_zone(ci->zone, ci->type)) != NULL) { 390 | cache = (ngx_http_file_cache_t *) node->cache->data; 391 | 392 | ci->expire = -1; 393 | ngx_memcpy(ci->key_dumped, ci->filename + cache->path->len + 1, 2 * NGX_HTTP_CACHE_KEY_LEN); 394 | 395 | ngx_md5_init(&md5); 396 | ngx_md5_update(&md5, ci->cache_key->data, ci->cache_key->len); 397 | ngx_md5_final(key, &md5); 398 | 399 | ngx_memcpy(&ci->node.key, &key, sizeof(ngx_rbtree_key_t)); 400 | ngx_memcpy(&ci->key, &key[sizeof(ngx_rbtree_key_t)], NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t)); 401 | ngx_rbtree_insert(&node->files_info_tree, &ci->node); 402 | } 403 | } 404 | 405 | ngx_selective_cache_purge_rbtree_walker(&worker_data->zones_tree, worker_data->zones_tree.root, NULL, ngx_selective_cache_purge_start_sync_database_timer); 406 | } 407 | 408 | 409 | void 410 | ngx_selective_cache_purge_store_new_entries(void *d) 411 | { 412 | ngx_selective_cache_purge_shm_data_t *data = (ngx_selective_cache_purge_shm_data_t *) ngx_selective_cache_purge_shm_zone->data; 413 | ngx_selective_cache_purge_zone_t *node = (ngx_selective_cache_purge_zone_t *) d; 414 | ngx_http_file_cache_t *cache = (ngx_http_file_cache_t *) node->cache->data; 415 | ngx_queue_t *q; 416 | u_char *p; 417 | ngx_uint_t loaded = 0; 418 | ngx_flag_t has_elements = 0; 419 | ngx_file_t file; 420 | ngx_err_t err; 421 | ngx_http_file_cache_header_t h; 422 | 423 | size_t len = cache->path->name.len + 1 + cache->path->len + 2 * NGX_HTTP_CACHE_KEY_LEN; 424 | u_char filename_data[len + 1]; 425 | 426 | ngx_log_error(NGX_LOG_DEBUG, ngx_cycle->log, 0, "ngx_selective_cache_purge: adding new entries"); 427 | 428 | ngx_memcpy(filename_data, cache->path->name.data, cache->path->name.len); 429 | filename_data[len] = '\0'; 430 | 431 | while (!ngx_queue_empty(&node->files_info_queue) && (q = ngx_queue_last(&node->files_info_queue))) { 432 | ngx_selective_cache_purge_cache_item_t *ci = ngx_queue_data(q, ngx_selective_cache_purge_cache_item_t, queue); 433 | 434 | 435 | p = filename_data + len - (2 * NGX_HTTP_CACHE_KEY_LEN); 436 | p = ngx_copy(p, ci->key_dumped, (2 * NGX_HTTP_CACHE_KEY_LEN)); 437 | 438 | ngx_create_hashed_filename(cache->path, filename_data, len); 439 | 440 | if ((ci->filename = ngx_selective_cache_purge_alloc_str(db_ctx->pool, len - cache->path->name.len)) == NULL) { 441 | ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, 0, "ngx_selective_cache_purge: unable to allocate memory for file info"); 442 | break; 443 | } 444 | 445 | ngx_memcpy(ci->filename->data, filename_data + cache->path->name.len, ci->filename->len); 446 | 447 | ngx_memzero(&file, sizeof(ngx_file_t)); 448 | file.name.data = filename_data; 449 | file.name.len = len; 450 | file.log = ngx_cycle->log; 451 | 452 | file.fd = ngx_open_file(filename_data, NGX_FILE_RDONLY, NGX_FILE_OPEN, 0); 453 | if (file.fd == NGX_INVALID_FILE) { 454 | node->count--; 455 | ngx_queue_remove(q); 456 | err = ngx_errno; 457 | if (err != NGX_ENOENT) { 458 | ngx_log_error(NGX_LOG_CRIT, ngx_cycle->log, err, "ngx_selective_cache_purge: "ngx_open_file_n " \"%V\" failed", &file.name); 459 | } 460 | continue; 461 | } 462 | 463 | if (ngx_read_file(&file, (u_char *) &h, sizeof(ngx_http_file_cache_header_t), 0) == NGX_ERROR) { 464 | ngx_log_error(NGX_LOG_CRIT, ngx_cycle->log, ngx_errno, "ngx_selective_cache_purge: "ngx_read_file_n " cache file %V failed", &file.name); 465 | ngx_close_file(file.fd); 466 | break; 467 | } 468 | 469 | #ifdef NGX_HTTP_CACHE_VERSION 470 | if (h.version != NGX_HTTP_CACHE_VERSION) { 471 | node->count--; 472 | ngx_queue_remove(q); 473 | ngx_log_error(NGX_LOG_INFO, ngx_cycle->log, 0, "ngx_selective_cache_purge: cache file \"%V\" version mismatch. expected: %d, cached: %d", &file.name, NGX_HTTP_CACHE_VERSION, h.version); 474 | ngx_close_file(file.fd); 475 | continue; 476 | } 477 | #endif 478 | 479 | if ((ci->cache_key = ngx_selective_cache_purge_alloc_str(db_ctx->pool, h.header_start - sizeof(ngx_http_file_cache_header_t) - NGX_HTTP_FILE_CACHE_KEY_LEN - 1)) == NULL) { 480 | ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, 0, "ngx_selective_cache_purge: unable to allocate memory for file info"); 481 | ngx_close_file(file.fd); 482 | break; 483 | } 484 | 485 | if (ngx_read_file(&file, ci->cache_key->data, ci->cache_key->len, sizeof(ngx_http_file_cache_header_t) + NGX_HTTP_FILE_CACHE_KEY_LEN) == NGX_ERROR) { 486 | ngx_log_error(NGX_LOG_CRIT, ngx_cycle->log, ngx_errno, "ngx_selective_cache_purge: "ngx_read_file_n " cache file %V failed", &file.name); 487 | ngx_close_file(file.fd); 488 | break; 489 | } 490 | 491 | if (ngx_close_file(file.fd) == NGX_FILE_ERROR) { 492 | ngx_log_error(NGX_LOG_CRIT, ngx_cycle->log, ngx_errno, "ngx_selective_cache_purge: "ngx_close_file_n " cache file %V failed", &file.name); 493 | break; 494 | } 495 | 496 | if (ngx_selective_cache_purge_store(node->name, node->type, ci->cache_key, ci->filename, ci->expire, node->db_ctx) != NGX_OK) { 497 | ngx_log_error(NGX_LOG_CRIT, ngx_cycle->log, ngx_errno, "ngx_selective_cache_purge: could not store entry"); 498 | break; 499 | } 500 | 501 | has_elements = 1; 502 | node->count--; 503 | ngx_queue_remove(q); 504 | 505 | loaded++; 506 | if ((loaded >= 50) || ngx_queue_empty(&node->files_info_queue)) { 507 | node->db_ctx->data = node; 508 | node->db_ctx->callback = ngx_selective_cache_purge_store_new_entries; 509 | if (ngx_selective_cache_purge_barrier_execution(node->db_ctx) != NGX_OK) { 510 | ngx_selective_cache_purge_store_new_entries(node); 511 | } 512 | return; 513 | } 514 | } 515 | 516 | if (has_elements || node->read_memory) { 517 | ngx_selective_cache_purge_timer_reset(node->read_memory ? 15000 : cache->loader_sleep, node->sync_database_event); 518 | ngx_log_error(NGX_LOG_DEBUG, ngx_cycle->log, 0, "ngx_selective_cache_purge: finish a cycle of sync for zone %V, scheduling one more to process >= %d files", node->name, node->count); 519 | } 520 | 521 | if (!node->read_memory && (node->count <= 0)) { 522 | data->zones_to_sync--; 523 | ngx_log_error(NGX_LOG_NOTICE, ngx_cycle->log, 0, "ngx_selective_cache_purge: sync for zone %V from memory to database finished", node->name); 524 | } 525 | 526 | if (data->zones_to_sync <= 0) { 527 | ngx_selective_cache_purge_remove_old_entries(data); 528 | } 529 | } 530 | 531 | 532 | void 533 | ngx_selective_cache_purge_remove_old_entries(void *d) 534 | { 535 | ngx_selective_cache_purge_shm_data_t *data = d; 536 | ngx_queue_t *q; 537 | ngx_uint_t count = 0; 538 | 539 | ngx_log_error(NGX_LOG_DEBUG, ngx_cycle->log, 0, "ngx_selective_cache_purge: removing old entries"); 540 | 541 | // remove keys from database not found on disk 542 | while (!ngx_queue_empty(&db_ctx->entries) && (q = ngx_queue_last(&db_ctx->entries))) { 543 | ngx_selective_cache_purge_cache_item_t *ci = ngx_queue_data(q, ngx_selective_cache_purge_cache_item_t, queue); 544 | ci->removed = 0; 545 | 546 | if (ngx_selective_cache_purge_remove_cache_entry(NULL, ci, db_ctx) != NGX_ERROR) { 547 | ngx_selective_cache_purge_remove(ci->zone, ci->type, ci->cache_key, ci->filename, db_ctx); 548 | } 549 | 550 | ngx_queue_remove(q); 551 | if ((count++ >= 50) || ngx_queue_empty(&db_ctx->entries)) { 552 | db_ctx->callback = ngx_selective_cache_purge_remove_old_entries; 553 | if (ngx_selective_cache_purge_barrier_execution(db_ctx) != NGX_OK) { 554 | ngx_selective_cache_purge_remove_old_entries(data); 555 | } 556 | return; 557 | } 558 | } 559 | 560 | if (ngx_queue_empty(&db_ctx->entries)) { 561 | ngx_selective_cache_purge_renew_entries(data); 562 | } 563 | } 564 | 565 | 566 | void 567 | ngx_selective_cache_purge_renew_entries(void *d) 568 | { 569 | ngx_selective_cache_purge_shm_data_t *data = d; 570 | ngx_queue_t *q; 571 | ngx_uint_t count = 0; 572 | 573 | ngx_log_error(NGX_LOG_DEBUG, ngx_cycle->log, 0, "ngx_selective_cache_purge: renew entries"); 574 | 575 | // renew expires of keys already on database 576 | count = 0; 577 | while (!ngx_queue_empty(&data->files_info_to_renew_queue) && (q = ngx_queue_last(&data->files_info_to_renew_queue))) { 578 | ngx_selective_cache_purge_cache_item_t *ci = ngx_queue_data(q, ngx_selective_cache_purge_cache_item_t, queue); 579 | 580 | if (ngx_selective_cache_purge_store(ci->zone, ci->type, ci->cache_key, ci->filename, ci->expire, db_ctx) != NGX_OK) { 581 | break; 582 | } 583 | 584 | ngx_queue_remove(q); 585 | if ((count++ >= 50) || ngx_queue_empty(&data->files_info_to_renew_queue)) { 586 | db_ctx->callback = ngx_selective_cache_purge_renew_entries; 587 | if (ngx_selective_cache_purge_barrier_execution(db_ctx) != NGX_OK) { 588 | ngx_selective_cache_purge_renew_entries(data); 589 | } 590 | return; 591 | } 592 | } 593 | 594 | ngx_log_error(NGX_LOG_NOTICE, ngx_cycle->log, 0, "ngx_selective_cache_purge: sync process finished"); 595 | 596 | ngx_selective_cache_purge_cleanup_sync(data); 597 | } 598 | -------------------------------------------------------------------------------- /src/ngx_selective_cache_purge_module_utils.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | 4 | ngx_str_t * 5 | ngx_selective_cache_purge_alloc_str(ngx_pool_t *pool, uint len) 6 | { 7 | ngx_str_t *aux = (ngx_str_t *) ngx_pcalloc(pool, sizeof(ngx_str_t) + len + 1); 8 | if (aux != NULL) { 9 | aux->data = (u_char *) (aux + 1); 10 | aux->len = len; 11 | ngx_memset(aux->data, '\0', len + 1); 12 | } 13 | return aux; 14 | } 15 | 16 | 17 | static ngx_int_t 18 | ngx_selective_cache_purge_send_response_text(ngx_http_request_t *r, const u_char *text, uint len, ngx_flag_t last_buffer) 19 | { 20 | ngx_buf_t *b; 21 | ngx_chain_t out; 22 | 23 | if ((text == NULL) || (r->connection->error)) { 24 | return NGX_ERROR; 25 | } 26 | 27 | b = ngx_create_temp_buf(r->pool, len); 28 | if (b == NULL) { 29 | return NGX_HTTP_INTERNAL_SERVER_ERROR; 30 | } 31 | 32 | b->last = ngx_copy(b->pos, text, len); 33 | b->memory = len ? 1 : 0; 34 | b->last_buf = (r == r->main) ? last_buffer : 0; 35 | b->last_in_chain = 1; 36 | b->flush = 1; 37 | 38 | out.buf = b; 39 | out.next = NULL; 40 | 41 | return ngx_http_output_filter(r, &out); 42 | } 43 | 44 | 45 | static ngx_int_t 46 | ngx_selective_cache_purge_send_header(ngx_http_request_t *r, size_t len, ngx_uint_t status, ngx_str_t *content_type) 47 | { 48 | r->headers_out.status = status; 49 | r->headers_out.content_length_n = len; 50 | r->header_only = len ? 0 : 1; 51 | r->keepalive = 0; 52 | 53 | r->headers_out.content_type.data = content_type->data; 54 | r->headers_out.content_type.len = content_type->len; 55 | r->headers_out.content_type_len = content_type->len; 56 | 57 | return ngx_http_send_header(r); 58 | } 59 | 60 | 61 | ngx_int_t 62 | ngx_selective_cache_purge_send_response(ngx_http_request_t *r, u_char *data, size_t len, ngx_uint_t status, ngx_str_t *content_type) 63 | { 64 | ngx_int_t rc; 65 | 66 | if (ngx_http_discard_request_body(r) != NGX_OK) { 67 | return ngx_selective_cache_purge_send_header(r, 0, NGX_HTTP_INTERNAL_SERVER_ERROR, content_type); 68 | } 69 | 70 | if ((r->method == NGX_HTTP_HEAD) || (len == 0)) { 71 | return ngx_selective_cache_purge_send_header(r, len, status, content_type); 72 | } 73 | 74 | rc = ngx_selective_cache_purge_send_header(r, len, status, content_type); 75 | 76 | if (rc == NGX_ERROR || rc > NGX_OK || r->header_only) { 77 | return rc; 78 | } 79 | 80 | return ngx_selective_cache_purge_send_response_text(r, data, len, 1); 81 | } 82 | 83 | 84 | ngx_str_t * 85 | ngx_selective_cache_purge_get_module_type_by_tag(void *tag) 86 | { 87 | ngx_str_t *type = NULL; 88 | 89 | #if NGX_HTTP_FASTCGI 90 | if (tag == &ngx_http_fastcgi_module) { 91 | type = &NGX_SELECTIVE_CACHE_PURGE_FASTCGI_TYPE; 92 | } 93 | #endif /* NGX_HTTP_FASTCGI */ 94 | 95 | #if NGX_HTTP_PROXY 96 | if (tag == &ngx_http_proxy_module) { 97 | type = &NGX_SELECTIVE_CACHE_PURGE_PROXY_TYPE; 98 | } 99 | #endif /* NGX_HTTP_PROXY */ 100 | 101 | #if NGX_HTTP_SCGI 102 | if (tag == &ngx_http_scgi_module) { 103 | type = &NGX_SELECTIVE_CACHE_PURGE_SCGI_TYPE; 104 | } 105 | #endif /* NGX_HTTP_SCGI */ 106 | 107 | #if NGX_HTTP_UWSGI 108 | if (tag == &ngx_http_uwsgi_module) { 109 | type = &NGX_SELECTIVE_CACHE_PURGE_UWSGI_TYPE; 110 | } 111 | #endif /* NGX_HTTP_UWSGI */ 112 | 113 | return type; 114 | } 115 | 116 | 117 | u_char 118 | ngx_selective_cache_purge_hex_char_to_byte(u_char c) 119 | { 120 | if(c >= '0' && c <= '9') { 121 | return (u_char)(c - '0'); 122 | } else if(c >= 'A' && c <= 'F') { 123 | return (u_char)(10 + c - 'A'); 124 | } else if(c >= 'a' && c <= 'f') { 125 | return (u_char)(10 + c - 'a'); 126 | } 127 | return 0; 128 | } 129 | 130 | 131 | u_char * 132 | ngx_selective_cache_purge_hex_read(u_char *dst, u_char *src, size_t len) 133 | { 134 | while (len > 0) { 135 | *dst = (ngx_selective_cache_purge_hex_char_to_byte(*src++) << 4); 136 | *dst |= (ngx_selective_cache_purge_hex_char_to_byte(*src++) & 0xf); 137 | 138 | len -= 2; 139 | dst++; 140 | } 141 | return dst; 142 | } 143 | 144 | 145 | static void * 146 | ngx_rbtree_generic_find(ngx_rbtree_t *tree, ngx_rbtree_key_t node_key, void *untie, int (*compare) (const ngx_rbtree_node_t *node, const void *untie)) 147 | { 148 | ngx_rbtree_node_t *node, *sentinel; 149 | ngx_int_t rc; 150 | 151 | node = tree->root; 152 | sentinel = tree->sentinel; 153 | 154 | while ((node != NULL) && (node != sentinel)) { 155 | if (node_key < node->key) { 156 | node = node->left; 157 | continue; 158 | } 159 | 160 | if (node_key > node->key) { 161 | node = node->right; 162 | continue; 163 | } 164 | 165 | /* node_key == node->key */ 166 | rc = compare(node, untie); 167 | if (rc == 0) { 168 | return node; 169 | } 170 | 171 | node = (rc < 0) ? node->left : node->right; 172 | } 173 | 174 | return NULL; 175 | } 176 | 177 | 178 | static void 179 | ngx_rbtree_generic_insert(ngx_rbtree_node_t *temp, ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel, int (*compare) (const ngx_rbtree_node_t *left, const ngx_rbtree_node_t *right)) 180 | { 181 | ngx_rbtree_node_t **p; 182 | 183 | for (;;) { 184 | if (node->key < temp->key) { 185 | p = &temp->left; 186 | } else if (node->key > temp->key) { 187 | p = &temp->right; 188 | } else { /* node->key == temp->key */ 189 | p = (compare(node, temp) < 0) ? &temp->left : &temp->right; 190 | } 191 | 192 | if (*p == sentinel) { 193 | break; 194 | } 195 | 196 | temp = *p; 197 | } 198 | 199 | *p = node; 200 | node->parent = temp; 201 | node->left = sentinel; 202 | node->right = sentinel; 203 | ngx_rbt_red(node); 204 | } 205 | 206 | 207 | static int 208 | ngx_selective_cache_purge_compare_rbtree_file_info_key(const ngx_rbtree_node_t *v_node, const void *v_key) 209 | { 210 | ngx_selective_cache_purge_cache_item_t *node = (ngx_selective_cache_purge_cache_item_t *) v_node; 211 | return ngx_memcmp(v_key, node->key, NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t)); 212 | } 213 | 214 | 215 | static int 216 | ngx_selective_cache_purge_compare_rbtree_file_info_nodes(const ngx_rbtree_node_t *v_left, const ngx_rbtree_node_t *v_right) 217 | { 218 | ngx_selective_cache_purge_cache_item_t *right = (ngx_selective_cache_purge_cache_item_t *) v_right; 219 | return ngx_selective_cache_purge_compare_rbtree_file_info_key(v_left, right->key); 220 | } 221 | 222 | 223 | static int 224 | ngx_selective_cache_purge_compare_rbtree_zones_node(const ngx_rbtree_node_t *v_left, const ngx_rbtree_node_t *v_right) 225 | { 226 | ngx_selective_cache_purge_zone_t *left = (ngx_selective_cache_purge_zone_t *) v_left, *right = (ngx_selective_cache_purge_zone_t *) v_right; 227 | int rc = ngx_memn2cmp(left->name->data, right->name->data, left->name->len, right->name->len); 228 | if (rc == 0) { 229 | rc = ngx_memn2cmp(left->type->data, right->type->data, left->type->len, right->type->len); 230 | } 231 | return rc; 232 | } 233 | 234 | 235 | static int 236 | ngx_selective_cache_purge_compare_rbtree_zone_type(const ngx_rbtree_node_t *v_node, const void *v_type) 237 | { 238 | ngx_selective_cache_purge_zone_t *node = (ngx_selective_cache_purge_zone_t *) v_node; 239 | ngx_str_t *type = (ngx_str_t *) v_type; 240 | return ngx_memn2cmp(node->type->data, type->data, node->type->len, type->len); 241 | } 242 | 243 | 244 | void 245 | ngx_selective_cache_purge_rbtree_zones_insert(ngx_rbtree_node_t *temp, ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel) 246 | { 247 | ngx_rbtree_generic_insert(temp, node, sentinel, ngx_selective_cache_purge_compare_rbtree_zones_node); 248 | } 249 | 250 | 251 | void 252 | ngx_selective_cache_purge_rbtree_file_info_insert(ngx_rbtree_node_t *temp, ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel) 253 | { 254 | ngx_rbtree_generic_insert(temp, node, sentinel, ngx_selective_cache_purge_compare_rbtree_file_info_nodes); 255 | } 256 | 257 | 258 | ngx_selective_cache_purge_zone_t * 259 | ngx_selective_cache_purge_find_zone(ngx_str_t *zone, ngx_str_t *type) 260 | { 261 | ngx_selective_cache_purge_worker_data_t *data = ngx_selective_cache_purge_worker_data; 262 | ngx_rbtree_key_t node_key = ngx_crc32_short(zone->data, zone->len); 263 | return (ngx_selective_cache_purge_zone_t *) ngx_rbtree_generic_find(&data->zones_tree, node_key, type, ngx_selective_cache_purge_compare_rbtree_zone_type); 264 | } 265 | 266 | 267 | static int 268 | ngx_selective_cache_purge_compare_rbtree_file_cache_key(const ngx_rbtree_node_t *v_node, const void *v_key) 269 | { 270 | ngx_http_file_cache_node_t *node = (ngx_http_file_cache_node_t *) v_node; 271 | u_char *key = (u_char *) v_key; 272 | return ngx_memcmp(&key[sizeof(ngx_rbtree_key_t)], node->key, NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t)); 273 | } 274 | 275 | 276 | ngx_selective_cache_purge_cache_item_t * 277 | ngx_selective_cache_purge_file_info_lookup(ngx_rbtree_t *tree, ngx_http_file_cache_node_t *fcn) 278 | { 279 | return (ngx_selective_cache_purge_cache_item_t *) ngx_rbtree_generic_find(tree, fcn->node.key, fcn->key, ngx_selective_cache_purge_compare_rbtree_file_info_key); 280 | } 281 | 282 | 283 | ngx_http_file_cache_node_t * 284 | ngx_selective_cache_purge_file_cache_lookup(ngx_http_file_cache_t *cache, u_char *key) 285 | { 286 | ngx_rbtree_key_t node_key; 287 | ngx_memcpy((u_char *) &node_key, key, sizeof(ngx_rbtree_key_t)); 288 | return (ngx_http_file_cache_node_t *) ngx_rbtree_generic_find(&cache->sh->rbtree, node_key, key, ngx_selective_cache_purge_compare_rbtree_file_cache_key); 289 | } 290 | 291 | 292 | ngx_int_t 293 | ngx_selective_cache_purge_file_cache_lookup_on_disk(ngx_http_request_t *r, ngx_http_file_cache_t *cache, ngx_str_t *cache_key, u_char *md5_key) 294 | { 295 | #if NGX_HTTP_CACHE 296 | ngx_http_cache_t *c; 297 | ngx_str_t *key; 298 | ngx_int_t rc; 299 | 300 | if ((c = ngx_pcalloc(r->pool, sizeof(ngx_http_cache_t))) == NULL) { 301 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "ngx_selective_cache_purge: could not alloc memory to ngx_http_cache_t structure"); 302 | return NGX_ERROR; 303 | } 304 | 305 | ngx_memzero(c, sizeof(ngx_http_cache_t)); 306 | 307 | rc = ngx_array_init(&c->keys, r->pool, 1, sizeof(ngx_str_t)); 308 | if (rc != NGX_OK) { 309 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "ngx_selective_cache_purge: could not alloc memory to keys array"); 310 | return NGX_ERROR; 311 | } 312 | 313 | key = ngx_array_push(&c->keys); 314 | if (key == NULL) { 315 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "ngx_selective_cache_purge: could not alloc memory to key item"); 316 | return NGX_ERROR; 317 | } 318 | 319 | key->data = cache_key->data; 320 | key->len = cache_key->len; 321 | 322 | r->cache = c; 323 | c->body_start = ngx_pagesize; 324 | c->file_cache = cache; 325 | c->file.log = r->connection->log; 326 | 327 | ngx_crc32_init(c->crc32); 328 | ngx_crc32_update(&c->crc32, cache_key->data, cache_key->len); 329 | ngx_crc32_final(c->crc32); 330 | 331 | c->header_start = sizeof(ngx_http_file_cache_header_t) + NGX_HTTP_FILE_CACHE_KEY_LEN + cache_key->len + 1; 332 | 333 | ngx_memcpy(c->key, md5_key, NGX_HTTP_CACHE_KEY_LEN); 334 | 335 | switch (ngx_http_file_cache_open(r)) { 336 | case NGX_OK: 337 | case NGX_HTTP_CACHE_STALE: 338 | case NGX_HTTP_CACHE_UPDATING: 339 | return NGX_OK; 340 | break; 341 | case NGX_DECLINED: 342 | return NGX_DECLINED; 343 | # if (NGX_HAVE_FILE_AIO) 344 | case NGX_AGAIN: 345 | return NGX_AGAIN; 346 | # endif 347 | default: 348 | return NGX_ERROR; 349 | } 350 | #else 351 | return NGX_OK; 352 | #endif 353 | } 354 | 355 | 356 | void 357 | ngx_selective_cache_purge_timer_set(ngx_msec_t timer_interval, ngx_event_t *event, ngx_event_handler_pt event_handler, ngx_flag_t start_timer) 358 | { 359 | if ((timer_interval != NGX_CONF_UNSET_MSEC) && start_timer) { 360 | if (event->data == NULL) { 361 | event->data = event; //set event as data to avoid error when running on debug mode (on log event) 362 | } 363 | event->handler = event_handler; 364 | event->log = ngx_cycle->log; 365 | ngx_selective_cache_purge_timer_reset(timer_interval, event); 366 | } 367 | } 368 | 369 | 370 | static void 371 | ngx_selective_cache_purge_timer_reset(ngx_msec_t timer_interval, ngx_event_t *timer_event) 372 | { 373 | if (!ngx_exiting && (timer_interval != NGX_CONF_UNSET_MSEC)) { 374 | if (timer_event->timedout) { 375 | ngx_time_update(); 376 | } 377 | ngx_add_timer(timer_event, timer_interval); 378 | } 379 | } 380 | 381 | 382 | static void 383 | ngx_selective_cache_purge_rbtree_walker(ngx_rbtree_t *tree, ngx_rbtree_node_t *node, void *data, ngx_int_t (*apply) (ngx_rbtree_node_t *node, void *data)) 384 | { 385 | ngx_rbtree_node_t *sentinel = tree->sentinel; 386 | 387 | if ((node != NULL) && (node != sentinel)) { 388 | apply(node, data); 389 | if (node->left != NULL) { 390 | ngx_selective_cache_purge_rbtree_walker(tree, node->left, data, apply); 391 | } 392 | if (node->right != NULL) { 393 | ngx_selective_cache_purge_rbtree_walker(tree, node->right, data, apply); 394 | } 395 | } 396 | } 397 | -------------------------------------------------------------------------------- /test/Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | ruby '3.3.5' 4 | 5 | gem 'rake' 6 | 7 | group :test do 8 | gem 'rspec' 9 | gem 'nginx_test_helper' 10 | gem 'em-http-request' 11 | gem 'byebug' 12 | gem 'rubyzip', require: 'zip' 13 | gem 'hiredis-client' 14 | gem 'redis' 15 | gem 'ostruct' 16 | gem 'base64' 17 | end 18 | -------------------------------------------------------------------------------- /test/Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: https://rubygems.org/ 3 | specs: 4 | Platform (0.4.2) 5 | addressable (2.8.7) 6 | public_suffix (>= 2.0.2, < 7.0) 7 | base64 (0.2.0) 8 | byebug (11.1.3) 9 | connection_pool (2.4.1) 10 | cookiejar (0.3.4) 11 | diff-lcs (1.5.1) 12 | em-http-request (1.1.7) 13 | addressable (>= 2.3.4) 14 | cookiejar (!= 0.3.1) 15 | em-socksify (>= 0.3) 16 | eventmachine (>= 1.0.3) 17 | http_parser.rb (>= 0.6.0) 18 | em-socksify (0.3.3) 19 | base64 20 | eventmachine (>= 1.0.0.beta.4) 21 | eventmachine (1.2.7) 22 | hiredis-client (0.22.2) 23 | redis-client (= 0.22.2) 24 | http_parser.rb (0.8.0) 25 | nginx_test_helper (0.4.2) 26 | popen4 27 | open4 (1.3.4) 28 | ostruct (0.6.0) 29 | popen4 (0.1.2) 30 | Platform (>= 0.4.0) 31 | open4 (>= 0.4.0) 32 | public_suffix (6.0.1) 33 | rake (13.2.1) 34 | redis (5.3.0) 35 | redis-client (>= 0.22.0) 36 | redis-client (0.22.2) 37 | connection_pool 38 | rspec (3.13.0) 39 | rspec-core (~> 3.13.0) 40 | rspec-expectations (~> 3.13.0) 41 | rspec-mocks (~> 3.13.0) 42 | rspec-core (3.13.2) 43 | rspec-support (~> 3.13.0) 44 | rspec-expectations (3.13.3) 45 | diff-lcs (>= 1.2.0, < 2.0) 46 | rspec-support (~> 3.13.0) 47 | rspec-mocks (3.13.2) 48 | diff-lcs (>= 1.2.0, < 2.0) 49 | rspec-support (~> 3.13.0) 50 | rspec-support (3.13.1) 51 | rubyzip (2.3.2) 52 | 53 | PLATFORMS 54 | ruby 55 | 56 | DEPENDENCIES 57 | base64 58 | byebug 59 | em-http-request 60 | hiredis-client 61 | nginx_test_helper 62 | ostruct 63 | rake 64 | redis 65 | rspec 66 | rubyzip 67 | 68 | RUBY VERSION 69 | ruby 3.3.5p100 70 | 71 | BUNDLED WITH 72 | 2.5.22 73 | -------------------------------------------------------------------------------- /test/assets/cache.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wandenberg/nginx-selective-cache-purge-module/0997b45207c4b9e2fa148243fd17bea28e55c985/test/assets/cache.zip -------------------------------------------------------------------------------- /test/assets/cache_2.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wandenberg/nginx-selective-cache-purge-module/0997b45207c4b9e2fa148243fd17bea28e55c985/test/assets/cache_2.zip -------------------------------------------------------------------------------- /test/assets/cache_3.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wandenberg/nginx-selective-cache-purge-module/0997b45207c4b9e2fa148243fd17bea28e55c985/test/assets/cache_3.zip -------------------------------------------------------------------------------- /test/assets/cache_4.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wandenberg/nginx-selective-cache-purge-module/0997b45207c4b9e2fa148243fd17bea28e55c985/test/assets/cache_4.zip -------------------------------------------------------------------------------- /test/assets/nginx-test.conf: -------------------------------------------------------------------------------- 1 | pid <%= pid_file %>; 2 | error_log <%= error_log %> error; 3 | 4 | worker_processes <%= worker_processes %>; 5 | worker_rlimit_core 500M; 6 | working_directory <%= nginx_tests_core_dir(config_id) %>; 7 | debug_points abort; 8 | 9 | events { 10 | worker_connections 4096; 11 | use <%= nginx_event_type %>; 12 | } 13 | 14 | http { 15 | default_type application/octet-stream; 16 | 17 | log_format main '$remote_addr - [$time_local] $host $request ($status) $request_time s ' 18 | '$body_bytes_sent b $http_referer $http_x_forwarded_for - ' 19 | '[$upstream_cache_status] $upstream_response_time s - $http_user_agent '; 20 | 21 | access_log <%= access_log %> main; 22 | 23 | <%= additional_config %> 24 | proxy_cache_path <%= proxy_cache_path %> levels=1:2 keys_zone=zone:<%= keys_zone %> inactive=<%= inactive %> max_size=<%= max_size %> loader_files=100 loader_sleep=1; 25 | 26 | error_page 404 /error_pages/404.html; 27 | 28 | server { 29 | listen <%= nginx_port %>; 30 | server_name <%= nginx_host %>; 31 | 32 | <%= write_directive("selective_cache_purge_redis_unix_socket", redis_unix_socket) %> 33 | <%= write_directive("selective_cache_purge_redis_host", redis_host) %> 34 | <%= write_directive("selective_cache_purge_redis_port", redis_port) %> 35 | <%= write_directive("selective_cache_purge_redis_password", redis_password) %> 36 | <%= write_directive("selective_cache_purge_redis_database", redis_database) %> 37 | 38 | location ~ /purge(.*) { 39 | <%= write_directive("selective_cache_purge_query", purge_query) %> 40 | } 41 | 42 | location /error_pages { 43 | internal; 44 | return 404 $uri; 45 | } 46 | 47 | location /no-cache { 48 | proxy_pass http://unix:/tmp/nginx_tests/nginx.socket; 49 | } 50 | 51 | location /unavailable { 52 | add_header "x-cache-status" $upstream_cache_status; 53 | 54 | proxy_pass http://unix:/tmp/nginx_tests/test_unavailable.socket; 55 | 56 | proxy_cache zone; 57 | proxy_cache_key "$uri"; 58 | proxy_cache_valid 200 1m; 59 | proxy_cache_valid any 30s; 60 | proxy_cache_use_stale timeout updating http_500; 61 | } 62 | 63 | location / { 64 | add_header "x-cache-status" $upstream_cache_status; 65 | 66 | proxy_pass http://unix:/tmp/nginx_tests/nginx.socket; 67 | 68 | proxy_no_cache $arg_nocache; 69 | proxy_cache_bypass $arg_nocache; 70 | 71 | proxy_cache zone; 72 | proxy_cache_key "$uri"; 73 | proxy_cache_valid 200 1m; 74 | proxy_cache_valid any 30s; 75 | proxy_cache_use_stale timeout updating http_500; 76 | } 77 | } 78 | 79 | server { 80 | listen unix:/tmp/nginx_tests/nginx.socket; 81 | 82 | location /cookie { 83 | add_header "Set-Cookie" "some=value"; 84 | return 200; 85 | } 86 | 87 | location /not-found { 88 | return 404; 89 | } 90 | 91 | location /big-cache { 92 | expires 30d; 93 | return 200; 94 | } 95 | 96 | location /small-cache { 97 | expires 5d; 98 | return 200; 99 | } 100 | 101 | location /conditional { 102 | expires 10s; 103 | 104 | if ($arg_error = 1) { 105 | return 500; 106 | } 107 | 108 | return 200; 109 | } 110 | 111 | location / { 112 | return 200; 113 | } 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /test/cache_full_spec.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path("./spec_helper", File.dirname(__FILE__)) 2 | 3 | describe "Selective Cache Purge Module Cache Full" do 4 | let!(:config) do 5 | { 6 | worker_processes: 1, 7 | max_size: "1m", 8 | keys_zone: "1m" 9 | } 10 | end 11 | 12 | def cached_files 13 | Dir["#{proxy_cache_path}/**/**"].select{|path| File.file?(path)} 14 | end 15 | 16 | def rotate_cache(start=1, expected_cached_files=nil) 17 | initial = get_database_entries_for("*").count 18 | number_of_requests = expected_cached_files.nil? ? 1000 : 2 * expected_cached_files 19 | expected_cached_files ||= 256 20 | number_of_requests.times do |i| 21 | response_for("http://#{nginx_host}:#{nginx_port}/to/set/cache/full#{start + i}.html") 22 | end 23 | expect(cached_files.count).to be > expected_cached_files # max_size / page_size 24 | sleep(0.5) if NginxTestHelper.nginx_executable.include?("valgrind") 25 | final = get_database_entries_for("*").count 26 | expect(final).to eql(initial + number_of_requests) 27 | count = 0 28 | while (total = cached_files.count) > expected_cached_files # max_size / page_size 29 | sleep 1 30 | count += 1 31 | raise "Cache still over limit. Has #{total} files" if count > 10 32 | end 33 | end 34 | 35 | shared_examples_for "not found entries to purge" do 36 | it "should return not found" do 37 | nginx_run_server(config, timeout: 60) do |conf| 38 | rotate_cache 39 | 40 | expect(response_for("http://#{nginx_host}:#{nginx_port}/purge#{url_to_purge}").code).to eql('404') 41 | 42 | rotate_cache(1001) 43 | end 44 | end 45 | end 46 | 47 | shared_examples_for "keep redis organized" do 48 | it "should not have entries on redis for purged pattern" do 49 | nginx_run_server(config, timeout: 6000) do |conf| 50 | rotate_cache 51 | 52 | response_for("http://#{nginx_host}:#{nginx_port}/purge#{url_to_purge}") 53 | expect(get_database_entries_for(url_to_purge).count).to eql(0) 54 | 55 | rotate_cache(1001) 56 | end 57 | end 58 | end 59 | 60 | shared_examples_for "keep control over cache size" do 61 | it "should return the cache to its limit" do 62 | nginx_run_server(config, timeout: 60) do |conf| 63 | rotate_cache 64 | 65 | response_for("http://#{nginx_host}:#{nginx_port}/purge#{url_to_purge}") 66 | 67 | expect{ rotate_cache(1001) }.to_not raise_error 68 | expect(cached_files.count).to be < 256 69 | end 70 | end 71 | end 72 | 73 | context "when purging only one entry" do 74 | 75 | context "and it was in cache" do 76 | let(:url_to_purge) { "/to/set/cache/full1000.html" } 77 | 78 | it "should return excluded files" do 79 | nginx_run_server(config, timeout: 60) do |conf| 80 | rotate_cache 81 | 82 | resp = response_for("http://#{nginx_host}:#{nginx_port}/purge#{url_to_purge}") 83 | expect(resp.code).to eql('200') 84 | expect(resp.body).to have_purged_urls([url_to_purge]) 85 | 86 | rotate_cache(1001) 87 | end 88 | end 89 | 90 | it_should_behave_like "keep redis organized" 91 | it_should_behave_like "keep control over cache size" 92 | end 93 | 94 | context "and it was not in cache" do 95 | let(:url_to_purge) { "/to/set/cache/full1.html" } 96 | 97 | it_should_behave_like "not found entries to purge" 98 | it_should_behave_like "keep redis organized" 99 | it_should_behave_like "keep control over cache size" 100 | end 101 | 102 | context "and it never was in cache" do 103 | let(:url_to_purge) { "/file_never_cached.html" } 104 | 105 | it_should_behave_like "not found entries to purge" 106 | it_should_behave_like "keep redis organized" 107 | it_should_behave_like "keep control over cache size" 108 | end 109 | 110 | context "reference count for cached itens" do 111 | it "should not keep references controlling the cache size inside its limits" do 112 | # to force the tries limit on function ngx_http_file_cache_forced_expire, actually in 20, we purge all files one each time 113 | nginx_run_server(config, timeout: 60) do |conf| 114 | rotate_cache 115 | 116 | entries = get_database_entries_for("*") 117 | entries.each do |cache_key, zone, type, filename| 118 | response_for("http://#{nginx_host}:#{nginx_port}/purge/#{cache_key}") 119 | end 120 | 121 | expect{ rotate_cache(1001) }.to_not raise_error 122 | expect(cached_files.count).to be < 256 123 | end 124 | end 125 | end 126 | end 127 | 128 | context "when purging multiple entries" do 129 | context "and they were in cache" do 130 | let(:url_to_purge) { "/to/set/cache/full99*.html" } 131 | 132 | it "should return excluded files" do 133 | nginx_run_server(config, timeout: 60) do |conf| 134 | rotate_cache 135 | 136 | resp = response_for("http://#{nginx_host}:#{nginx_port}/purge#{url_to_purge}") 137 | expect(resp.code).to eql('200') 138 | expect(resp.body).to have_purged_urls([ 139 | "/to/set/cache/full990.html", 140 | "/to/set/cache/full991.html", 141 | "/to/set/cache/full992.html", 142 | "/to/set/cache/full993.html", 143 | "/to/set/cache/full994.html", 144 | "/to/set/cache/full995.html", 145 | "/to/set/cache/full996.html", 146 | "/to/set/cache/full997.html", 147 | "/to/set/cache/full998.html", 148 | "/to/set/cache/full999.html", 149 | ]) 150 | 151 | rotate_cache(1001) 152 | end 153 | end 154 | 155 | it_should_behave_like "keep redis organized" 156 | it_should_behave_like "keep control over cache size" 157 | end 158 | 159 | context "and they were not in cache" do 160 | let(:url_to_purge) { "/to/set/cache/full20*.html" } 161 | 162 | it_should_behave_like "not found entries to purge" 163 | it_should_behave_like "keep redis organized" 164 | it_should_behave_like "keep control over cache size" 165 | end 166 | 167 | context "and some were in cache and others not" do 168 | let(:url_to_purge) { "/to/set/cache/full74*.html" } 169 | 170 | it "should return excluded files" do 171 | nginx_run_server(config, timeout: 60000) do |conf| 172 | rotate_cache 173 | 174 | resp = response_for("http://#{nginx_host}:#{nginx_port}/purge#{url_to_purge}") 175 | expect(resp.code).to eql('200') 176 | expect(resp.body).to have_purged_urls([ 177 | "/to/set/cache/full746.html", 178 | "/to/set/cache/full747.html", 179 | "/to/set/cache/full748.html", 180 | "/to/set/cache/full749.html", 181 | ]) 182 | 183 | rotate_cache(1001) 184 | end 185 | end 186 | 187 | it_should_behave_like "keep redis organized" 188 | it_should_behave_like "keep control over cache size" 189 | end 190 | 191 | context "and they never were in cache" do 192 | let(:url_to_purge) { "/files_never_cached*" } 193 | 194 | it_should_behave_like "not found entries to purge" 195 | it_should_behave_like "keep redis organized" 196 | it_should_behave_like "keep control over cache size" 197 | end 198 | 199 | context "reference count for cached itens" do 200 | it "should not keep references controlling the cache size inside its limits" do 201 | # to force the tries limit on function ngx_http_file_cache_forced_expire, actually in 20, we purge all files at once 202 | nginx_run_server(config, timeout: 60) do |conf| 203 | rotate_cache 204 | 205 | response_for("http://#{nginx_host}:#{nginx_port}/purge/*") 206 | 207 | expect{ rotate_cache(1001) }.to_not raise_error 208 | expect(cached_files.count).to be < 256 209 | end 210 | end 211 | end 212 | 213 | context "and there are other requests comming" do 214 | it "should send response to all requests" do 215 | nginx_run_server(config, timeout: 60) do |conf| 216 | rotate_cache 217 | 218 | EventMachine.run do 219 | request_sent = 0 220 | request_received = 0 221 | 222 | timer = EventMachine::PeriodicTimer.new(0.05) do 223 | request_sent += 1 224 | req = EventMachine::HttpRequest.new("http://#{nginx_host}:#{nginx_port}/index.html", connect_timeout: 10, inactivity_timeout: 15).get 225 | req.callback do 226 | fail("Request failed with error #{req.response_header.status}") if req.response_header.status != 200 227 | request_received += 1 228 | end 229 | end 230 | 231 | req = EventMachine::HttpRequest.new("http://#{nginx_host}:#{nginx_port}/purge/*", connect_timeout: 10, inactivity_timeout: 60).get 232 | req.callback do 233 | timer.cancel 234 | sleep 1.5 235 | expect(cached_files.count).to be_within(5).of(0) 236 | expect(request_sent).to be > 10 237 | expect(request_received).to be_within(5).of(request_sent) 238 | EventMachine.stop 239 | end 240 | end 241 | end 242 | end 243 | end 244 | 245 | context "and the purge request is canceled" do 246 | context "during the scan on redis step" do 247 | it "should stop purging" do 248 | nginx_run_server(config.merge(max_size: "4m"), timeout: 60) do |conf| 249 | rotate_cache(1, 1024) 250 | initial_size = cached_files.count 251 | expect(initial_size).to eq 1023 252 | 253 | log_pre = File.readlines(conf.access_log) 254 | 255 | EventMachine.run do 256 | req = EventMachine::HttpRequest.new("http://#{nginx_host}:#{nginx_port}/purge/*").get 257 | req.callback do 258 | fail("The request was not canceled") 259 | EventMachine.stop 260 | end 261 | req.errback do 262 | expect(cached_files.count).to eql initial_size 263 | 264 | EM.next_tick do 265 | EventMachine.stop 266 | EM.run do 267 | EM.add_timer(1) do 268 | expect(cached_files.count).to eql initial_size 269 | log_pos = File.readlines(conf.access_log) 270 | expect((log_pos - log_pre).join).to include("GET /purge/* HTTP/1.1 (499)") 271 | EventMachine.stop 272 | end 273 | end 274 | end 275 | end 276 | EM.next_tick do 277 | req.close 278 | end 279 | end 280 | end 281 | end 282 | end 283 | 284 | context "during the purge files step" do 285 | it "should stop purging" do 286 | nginx_run_server(config.merge(max_size: "4m"), timeout: 60) do |conf| 287 | rotate_cache(1, 1024) 288 | initial_size = cached_files.count 289 | expect(initial_size).to eq 1023 290 | 291 | EventMachine.run do 292 | req = EventMachine::HttpRequest.new("http://#{nginx_host}:#{nginx_port}/purge/*", inactivity_timeout: 2).get 293 | req.callback do 294 | fail("The request was not canceled") 295 | EventMachine.stop 296 | end 297 | req.errback do 298 | final_size = cached_files.count 299 | expect(final_size).to be > 0 300 | expect(final_size).to be < initial_size 301 | 302 | EM.add_timer(1) do 303 | expect(cached_files.count).to eq final_size 304 | EventMachine.stop 305 | end 306 | end 307 | end 308 | end 309 | end 310 | end 311 | end 312 | end 313 | end 314 | 315 | -------------------------------------------------------------------------------- /test/database_lock_spec.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path("./spec_helper", File.dirname(__FILE__)) 2 | 3 | describe "Selective Cache Purge Module Database Lock" do 4 | let!(:config) do 5 | { } 6 | end 7 | 8 | def run_concurrent_requests_check(number_of_requests, path = "", &block) 9 | requests_sent = 0 10 | requests_success = 0 11 | EventMachine.run do 12 | cached_requests_timer = EventMachine::PeriodicTimer.new(0.001) do 13 | if requests_sent >= number_of_requests 14 | if requests_success >= requests_sent 15 | cached_requests_timer.cancel 16 | sleep 1.5 17 | block.call unless block.nil? 18 | EventMachine.stop 19 | end 20 | else 21 | requests_sent += 1 22 | req = EventMachine::HttpRequest.new("http://#{nginx_host}:#{nginx_port}#{path}/#{requests_sent}/index.html", connect_timeout: 100, inactivity_timeout: 150).get 23 | req.callback do 24 | fail("Request failed with error #{req.response_header.status}") if req.response_header.status != 200 25 | requests_success += 1 26 | end 27 | end 28 | end 29 | end 30 | end 31 | 32 | context "serializing database writes" do 33 | it "should not lose requests when inserting cache entries into database" do 34 | nginx_run_server(config, timeout: 200) do 35 | number_of_requests = 200 36 | run_concurrent_requests_check(number_of_requests) do 37 | expect(get_database_entries_for('*').count).to eql(number_of_requests) 38 | end 39 | end 40 | end 41 | 42 | it "should not lose requests when deleting cache entries from database" do 43 | nginx_run_server(config, timeout: 200) do 44 | number_of_requests = 200 45 | run_concurrent_requests_check(number_of_requests) do 46 | run_concurrent_requests_check(number_of_requests, "/purge") do 47 | expect(get_database_entries_for('*').count).to eql(0) 48 | end 49 | end 50 | end 51 | end 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /test/module_spec.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path("./spec_helper", File.dirname(__FILE__)) 2 | 3 | describe "Selective Cache Purge Module" do 4 | let!(:config) do 5 | { } 6 | end 7 | 8 | context "when caching" do 9 | it "should return 200 for a existing url" do 10 | nginx_run_server(config) do 11 | expect(response_for("http://#{nginx_host}:#{nginx_port}/index.html").code).to eq '200' 12 | end 13 | end 14 | 15 | it "should not save entries for locations without cache enabled" do 16 | path = "/no-cache/index.html" 17 | nginx_run_server(config) do 18 | expect(response_for("http://#{nginx_host}:#{nginx_port}#{path}").code).to eq '200' 19 | end 20 | expect(get_database_entries_for(path)).to be_empty 21 | end 22 | 23 | it "should save an entry after caching" do 24 | path = "/index.html" 25 | nginx_run_server(config) do 26 | expect(response_for("http://#{nginx_host}:#{nginx_port}#{path}").code).to eq '200' 27 | end 28 | expect(get_database_entries_for(path)).not_to be_empty 29 | end 30 | 31 | it "should save an entry for status codes other than 200" do 32 | path = "/not-found/index.html" 33 | nginx_run_server(config, timeout: 60) do |conf| 34 | expect(log_changes_for(conf.access_log) do 35 | expect(response_for("http://#{nginx_host}:#{nginx_port}#{path}").code).to eq '404' 36 | end).to include("[MISS]") 37 | 38 | sleep 15 39 | 40 | expect(log_changes_for(conf.access_log) do 41 | expect(response_for("http://#{nginx_host}:#{nginx_port}#{path}").code).to eq '404' 42 | end).to include("[HIT]") 43 | 44 | sleep 20 45 | 46 | expect(log_changes_for(conf.access_log) do 47 | expect(response_for("http://#{nginx_host}:#{nginx_port}#{path}").code).to eq '404' 48 | end).to include("[EXPIRED]") 49 | end 50 | expect(get_database_entries_for(path)).not_to be_empty 51 | end 52 | 53 | it "should save an entry when backend is unavailable" do 54 | path = "/unavailable" 55 | nginx_run_server(config, timeout: 60) do |conf| 56 | expect(log_changes_for(conf.access_log) do 57 | expect(response_for("http://#{nginx_host}:#{nginx_port}#{path}").code).to eq '502' 58 | end).to include("[MISS]") 59 | 60 | sleep 15 61 | 62 | expect(log_changes_for(conf.access_log) do 63 | expect(response_for("http://#{nginx_host}:#{nginx_port}#{path}").code).to eq '502' 64 | end).to include("[HIT]") 65 | 66 | sleep 20 67 | 68 | expect(log_changes_for(conf.access_log) do 69 | expect(response_for("http://#{nginx_host}:#{nginx_port}#{path}").code).to eq '502' 70 | end).to include("[MISS]") 71 | end 72 | expect(get_database_entries_for(path)).not_to be_empty 73 | end 74 | 75 | it "should ignore when response is not cacheable" do 76 | path = "/cookie/index.html" 77 | nginx_run_server(config) do 78 | expect(response_for("http://#{nginx_host}:#{nginx_port}#{path}").code).to eq '200' 79 | end 80 | expect(get_database_entries_for(path)).to be_empty 81 | expect(Dir["#{proxy_cache_path}/*"]).to be_empty 82 | end 83 | 84 | it "should ignore when request match cache bypass" do 85 | path = "/index.html" 86 | nginx_run_server(config) do 87 | expect(response_for("http://#{nginx_host}:#{nginx_port}#{path}?nocache=1").code).to eq '200' 88 | end 89 | expect(get_database_entries_for(path)).to be_empty 90 | expect(Dir["#{proxy_cache_path}/*"]).to be_empty 91 | end 92 | 93 | it "should save using an unix socket" do 94 | path = "/index.html" 95 | nginx_run_server(config.merge(redis_host: nil, redis_unix_socket: redis_unix_socket)) do 96 | expect(response_for("http://#{nginx_host}:#{nginx_port}#{path}").code).to eq '200' 97 | end 98 | expect(get_database_entries_for(path)).not_to be_empty 99 | end 100 | 101 | it "should save using a password protected redis" do 102 | path = "/index.html" 103 | redis.config(:set, :requirepass, 'some_password') 104 | redis.auth 'some_password' 105 | begin 106 | nginx_run_server(config.merge(redis_password: 'some_password')) do 107 | expect(response_for("http://#{nginx_host}:#{nginx_port}#{path}").code).to eq '200' 108 | end 109 | expect(get_database_entries_for(path)).not_to be_empty 110 | ensure 111 | redis.config(:set, :requirepass, '') 112 | end 113 | end 114 | 115 | it "should not save with a wrong password" do 116 | path = "/index.html" 117 | redis.config(:set, :requirepass, 'some_password') 118 | redis.auth 'some_password' 119 | begin 120 | nginx_run_server(config.merge(redis_password: 'another_password')) do 121 | expect(response_for("http://#{nginx_host}:#{nginx_port}#{path}").code).to eq '200' 122 | end 123 | expect(get_database_entries_for(path)).to be_empty 124 | ensure 125 | redis.config(:set, :requirepass, '') 126 | end 127 | end 128 | 129 | it "should use the cache time or the inactive time as expires which is bigger" do 130 | nginx_run_server(config) do 131 | expect(response_for("http://#{nginx_host}:#{nginx_port}/index.html").code).to eq '200' 132 | expect(response_for("http://#{nginx_host}:#{nginx_port}/big-cache").code).to eq '200' 133 | expect(response_for("http://#{nginx_host}:#{nginx_port}/small-cache").code).to eq '200' 134 | end 135 | expect(ttl_database_entries_for("/index.html").first).to be_within(5).of(10 * 24 * 60 * 60) 136 | expect(ttl_database_entries_for("/big-cache").first).to be_within(5).of(30 * 24 * 60 * 60) 137 | expect(ttl_database_entries_for("/small-cache").first).to be_within(5).of(10 * 24 * 60 * 60) 138 | end 139 | 140 | it "should update the entry each time the cache is not HIT, including STALE" do 141 | path = "/conditional/index.html" 142 | nginx_run_server(config.merge(inactive: "20s"), timeout: 60) do |conf| 143 | expect((resp = response_for("http://#{nginx_host}:#{nginx_port}#{path}")).code).to eq '200' 144 | expect(resp['x-cache-status']).to include("MISS") 145 | sleep 1 146 | expect(ttl_database_entries_for(path).first).to be_within(2).of(20) 147 | 148 | sleep 4 149 | 150 | expect((resp = response_for("http://#{nginx_host}:#{nginx_port}#{path}")).code).to eq '200' 151 | expect(resp['x-cache-status']).to include("HIT") 152 | sleep 1 153 | expect(ttl_database_entries_for(path).first).to be_within(2).of(15) 154 | 155 | sleep 7 156 | 157 | expect((resp = response_for("http://#{nginx_host}:#{nginx_port}#{path}?error=1")).code).to eq '200' 158 | expect(resp['x-cache-status']).to include("STALE") 159 | sleep 1 160 | expect(ttl_database_entries_for(path).first).to be_within(2).of(20) 161 | 162 | sleep 1 163 | 164 | expect((resp = response_for("http://#{nginx_host}:#{nginx_port}#{path}")).code).to eq '200' 165 | expect(resp['x-cache-status']).to include("EXPIRED") 166 | sleep 1 167 | expect(ttl_database_entries_for(path).first).to be_within(2).of(20) 168 | 169 | sleep 9 170 | 171 | expect((resp = response_for("http://#{nginx_host}:#{nginx_port}#{path}")).code).to eq '200' 172 | expect(resp['x-cache-status']).to include("HIT") 173 | sleep 1 174 | expect(ttl_database_entries_for(path).first).to be_within(2).of(10) 175 | end 176 | end 177 | end 178 | 179 | context "when purging" do 180 | it "should return 400 when purging with an empty query" do 181 | nginx_run_server(config.merge(purge_query: "$1")) do 182 | expect(response_for("http://#{nginx_host}:#{nginx_port}/purge").code).to eq '400' 183 | end 184 | end 185 | 186 | it "should return 404 when purging with a query that doesn't match any cached entry" do 187 | nginx_run_server(config) do 188 | expect(response_for("http://#{nginx_host}:#{nginx_port}/purge/index.html").code).to eq '404' 189 | end 190 | end 191 | 192 | context "with cached entries" do 193 | let!(:cached_urls) do 194 | [ 195 | "/index.html", 196 | "/index2.html", 197 | "/resources.json", 198 | "/resources/r1.jpg", 199 | "/resources/r2.jpg", 200 | "/resources/r3.jpg", 201 | "/some/path/index.html", 202 | ] 203 | end 204 | 205 | def prepare_cache 206 | cached_urls.each do |url| 207 | response_for(File.join("http://#{nginx_host}:#{nginx_port}", url)) 208 | end 209 | sleep(0.5) if NginxTestHelper.nginx_executable.include?("valgrind") 210 | end 211 | 212 | it "should remove only matched entries" do 213 | purged_urls = ["/index.html","/index2.html"] 214 | 215 | nginx_run_server(config) do 216 | prepare_cache 217 | purged_files = get_database_entries_for('/index*').map{ |entry| entry[-1] } 218 | 219 | expect(purged_files.count).to eq 2 220 | purged_files.each do |f| 221 | expect(File.exist?("#{proxy_cache_path}#{f}")).to be_truthy 222 | end 223 | 224 | resp = response_for("http://#{nginx_host}:#{nginx_port}/purge/index") 225 | expect(resp.code).to eq '200' 226 | expect(resp.body).to have_purged_urls(purged_urls) 227 | 228 | expect(get_database_entries_for('/index*')).to be_empty 229 | remaining_keys = get_database_entries_for('*').map{ |entry| entry[0] }.sort 230 | expect(remaining_keys).to eql(cached_urls - purged_urls) 231 | 232 | purged_files.each do |f| 233 | expect(File.exist?("#{proxy_cache_path}#{f}")).to be_falsey 234 | end 235 | end 236 | end 237 | 238 | it "should remove only exact entry" do 239 | path = "/index.html" 240 | 241 | nginx_run_server(config) do 242 | prepare_cache 243 | purged_files = get_database_entries_for(path).map{ |entry| entry[-1] } 244 | 245 | expect(purged_files.count).to eq 1 246 | purged_files.each do |f| 247 | expect(File.exist?("#{proxy_cache_path}#{f}")).to be_truthy 248 | end 249 | 250 | resp = response_for("http://#{nginx_host}:#{nginx_port}/purge#{path}") 251 | expect(resp.code).to eq '200' 252 | expect(resp.body).to have_purged_urls([path]) 253 | 254 | expect(get_database_entries_for(path)).to be_empty 255 | remaining_keys = get_database_entries_for('*').map{ |entry| entry[0] }.sort 256 | expect(remaining_keys).to eql(cached_urls - [path]) 257 | 258 | purged_files.each do |f| 259 | expect(File.exist?("#{proxy_cache_path}#{f}")).to be_falsey 260 | end 261 | end 262 | end 263 | 264 | it "should return an empty list when the query does not match any entries" do 265 | nginx_run_server(config) do 266 | prepare_cache 267 | resp = response_for("http://#{nginx_host}:#{nginx_port}/purge/some/random/invalid/path") 268 | expect(resp.code).to eq '404' 269 | expect(resp.body).to eq "Could not found any entry that match the expression: /some/random/invalid/path*\n" 270 | end 271 | end 272 | 273 | it "should not cause md5 collision when nginx memory is empty" do 274 | nginx_run_server(config) do 275 | prepare_cache 276 | end 277 | 278 | nginx_run_server(config) do |conf| 279 | expect(log_changes_for(conf.error_log) do 280 | purged_urls = [ 281 | "/resources/r1.jpg", 282 | "/resources/r2.jpg", 283 | "/resources/r3.jpg" 284 | ] 285 | 286 | response = response_for("http://#{nginx_host}:#{nginx_port}/purge/resources/") 287 | expect(response.body).to have_purged_urls(purged_urls) 288 | expect(response.body).to have_not_purged_urls(cached_urls - purged_urls) 289 | 290 | end).not_to include("md5 collision") 291 | end 292 | end 293 | 294 | it "should remove using an unix socket" do 295 | purged_urls = ["/index.html","/index2.html"] 296 | 297 | nginx_run_server(config.merge(redis_host: nil, redis_unix_socket: redis_unix_socket)) do 298 | prepare_cache 299 | purged_files = get_database_entries_for('/index*').map{ |entry| entry[-1] } 300 | 301 | expect(purged_files.count).to eq 2 302 | purged_files.each do |f| 303 | expect(File.exist?("#{proxy_cache_path}#{f}")).to be_truthy 304 | end 305 | 306 | resp = response_for("http://#{nginx_host}:#{nginx_port}/purge/index") 307 | expect(resp.code).to eq '200' 308 | expect(resp.body).to have_purged_urls(purged_urls) 309 | 310 | expect(get_database_entries_for('/index*')).to be_empty 311 | remaining_keys = get_database_entries_for('*').map{ |entry| entry[0] }.sort 312 | expect(remaining_keys).to eql(cached_urls - purged_urls) 313 | 314 | purged_files.each do |f| 315 | expect(File.exist?("#{proxy_cache_path}#{f}")).to be_falsey 316 | end 317 | end 318 | end 319 | 320 | it "should remove using a password protected redis" do 321 | purged_urls = ["/index.html","/index2.html"] 322 | redis.config(:set, :requirepass, 'some_password') 323 | redis.auth 'some_password' 324 | begin 325 | nginx_run_server(config.merge(redis_password: 'some_password')) do 326 | prepare_cache 327 | purged_files = get_database_entries_for('/index*').map{ |entry| entry[-1] } 328 | 329 | expect(purged_files.count).to eq 2 330 | purged_files.each do |f| 331 | expect(File.exist?("#{proxy_cache_path}#{f}")).to be_truthy 332 | end 333 | 334 | resp = response_for("http://#{nginx_host}:#{nginx_port}/purge/index") 335 | expect(resp.code).to eq '200' 336 | expect(resp.body).to have_purged_urls(purged_urls) 337 | 338 | expect(get_database_entries_for('/index*')).to be_empty 339 | remaining_keys = get_database_entries_for('*').map{ |entry| entry[0] }.sort 340 | expect(remaining_keys).to eql(cached_urls - purged_urls) 341 | 342 | purged_files.each do |f| 343 | expect(File.exist?("#{proxy_cache_path}#{f}")).to be_falsey 344 | end 345 | end 346 | ensure 347 | redis.config(:set, :requirepass, '') 348 | end 349 | end 350 | 351 | context "and fail to remove file from the filesystem" do 352 | it "should ignore the missing entries but clear it from database" do 353 | nginx_run_server(config) do 354 | prepare_cache 355 | purged_files = get_database_entries_for('*') 356 | purged_files.each do |entry| 357 | expect(File.exist?("#{proxy_cache_path}#{entry[-1]}")).to be_truthy 358 | end 359 | 360 | # change directory of /index2.html to be read only 361 | FileUtils.chmod(0600, File.dirname("#{proxy_cache_path}/4/37")) 362 | 363 | # remove the file of /some/path/index.html from disk 364 | FileUtils.rm("#{proxy_cache_path}/6/93/9b06947cef9c730a57392e1221d3a936") 365 | expect(File.exist?("#{proxy_cache_path}/6/93/9b06947cef9c730a57392e1221d3a936")).to be_falsey 366 | 367 | resp = response_for("http://#{nginx_host}:#{nginx_port}/purge/*index*") 368 | expect(resp.code).to eq '200' 369 | expect(resp.body).to have_purged_urls(["/index.html"]) 370 | 371 | # change directory to original 372 | FileUtils.chmod(0700, File.dirname("#{proxy_cache_path}/4/37")) 373 | end 374 | 375 | # change directory of /resources/r2.jpg to be read only 376 | FileUtils.chmod(0600, File.dirname("#{proxy_cache_path}/9/e9")) 377 | 378 | # remove the file of /resources/r1.jpg from disk 379 | FileUtils.rm("#{proxy_cache_path}/5/32/e121c6da57be48c3f112adf6a8e54325") 380 | expect(File.exist?("#{proxy_cache_path}/5/32/e121c6da57be48c3f112adf6a8e54325")).to be_falsey 381 | 382 | nginx_run_server(config.merge(worker_processes: 1), timeout: 600) do 383 | resp = response_for("http://#{nginx_host}:#{nginx_port}/purge/resources*") 384 | expect(resp.code).to eq '200' 385 | expect(resp.body).to have_purged_urls(["/resources/r3.jpg", "/resources.json"]) 386 | end 387 | 388 | # change directory to original 389 | FileUtils.chmod(0700, File.dirname("#{proxy_cache_path}/9/e9")) 390 | 391 | remaining_keys = get_database_entries_for('*') 392 | expect(remaining_keys.map{|k| k[0]}.sort).to eq ["/index2.html", "/resources/r2.jpg"] 393 | 394 | remaining_files = Dir["#{proxy_cache_path}/**/**"].select{|f| File.file?(f)}.map{|f| f.gsub(proxy_cache_path, "") }.sort 395 | expect(remaining_files).to eq ["/4/37/893f012e35119c29787435670250b374", "/9/e9/2dd79c7d48e8dc92e4dfce4e3f638e99"] 396 | end 397 | end 398 | end 399 | end 400 | 401 | context "when doing a complex configuration" do 402 | it "should accept when a huge number of zones are used" do 403 | additional_config = 150.times.map{|i| "proxy_cache_path /tmp/cache_zone_#{i} levels=1:2 keys_zone=cache_zone_#{i}:10m loader_files=1000 loader_threshold=450 use_temp_path=off inactive=12h;"}.join("\n") 404 | expect(nginx_test_configuration(additional_config: additional_config)).to_not include("no memory") 405 | end 406 | end 407 | end 408 | 409 | -------------------------------------------------------------------------------- /test/nginx_configuration.rb: -------------------------------------------------------------------------------- 1 | module NginxConfiguration 2 | def self.default_configuration 3 | { 4 | disable_start_stop_server: false, 5 | master_process: 'off', 6 | daemon: 'off', 7 | unknown_value: nil, 8 | return_code: 404, 9 | additional_config: '', 10 | worker_processes: 4, 11 | proxy_cache_path: "/tmp/cache", 12 | redis_unix_socket: nil, 13 | redis_host: redis_host, 14 | redis_password: nil, 15 | redis_database: redis_database, 16 | purge_query: "$1*", 17 | max_size: "100m", 18 | keys_zone: "10m", 19 | inactive: "10d" 20 | } 21 | end 22 | 23 | def self.template_configuration 24 | File.open(File.expand_path('assets/nginx-test.conf', File.dirname(__FILE__))).read 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /test/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | # Set up gems listed in the Gemfile. 3 | ENV['BUNDLE_GEMFILE'] ||= File.expand_path('Gemfile', File.dirname(__FILE__)) 4 | require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE']) 5 | Bundler.require(:default, :test) if defined?(Bundler) 6 | 7 | require "net/http" 8 | require "uri" 9 | 10 | require File.expand_path('nginx_configuration', File.dirname(__FILE__)) 11 | 12 | def proxy_cache_path 13 | "/tmp/cache" 14 | end 15 | 16 | def redis_unix_socket 17 | File.join(NginxTestHelper::Config.nginx_tests_tmp_dir, "selective_cache_purge_redis_test.socket") 18 | end 19 | 20 | def redis_host 21 | 'localhost' 22 | end 23 | 24 | def redis_port 25 | 63790 26 | end 27 | 28 | def redis_database 29 | 4 30 | end 31 | 32 | def redis(host=redis_host, port=redis_port, database=redis_database) 33 | @redis ||= Redis.new(host: host, port: port, db: database, driver: :hiredis) 34 | end 35 | 36 | def clear_database 37 | redis.flushdb 38 | end 39 | 40 | def ttl_database_entries_for(cache_key) 41 | get_database_entries_for(cache_key).map do |entry| 42 | redis.ttl(entry.join(":")) 43 | end 44 | end 45 | 46 | def get_database_entries_for(cache_key) 47 | redis.scan_each(match: "#{cache_key}:*:*:*").map{ |key| key.split(":") } 48 | end 49 | 50 | def get_database_entries_for_zone(zone) 51 | redis.scan_each(match: "*:#{zone}:*:*").map{ |key| key.split(":") } 52 | end 53 | 54 | def insert_entry_on_database(zone, type, cache_key, filename, expires) 55 | redis.setex("#{cache_key}:#{zone}:#{type}:#{filename}", expires - Time.now.to_i + 1, 1) 56 | end 57 | 58 | def response_for(url) 59 | uri = URI.parse(url) 60 | Net::HTTP.get_response(uri) 61 | end 62 | 63 | def log_changes_for(log_file, &block) 64 | log_pre = File.readlines(log_file) 65 | block.call 66 | sleep(0.5) if NginxTestHelper.nginx_executable.include?("valgrind") 67 | log_pos = File.readlines(log_file) 68 | (log_pos - log_pre).join 69 | end 70 | 71 | RSpec::Matchers.define :have_purged_urls do |urls| 72 | match do |actual| 73 | text = actual.is_a?(Array) ? actual.map{|v| "\n#{v} ->"}.join : actual 74 | urls.all? do |url| 75 | text.match(/\n#{url} ->/) 76 | end 77 | end 78 | 79 | failure_message do |actual| 80 | "expected that #{actual} would #{description}" 81 | end 82 | 83 | failure_message_when_negated do |actual| 84 | "expected that #{actual} would not #{description}" 85 | end 86 | 87 | description do 88 | "have purged the urls: #{urls.join(", ")}" 89 | end 90 | end 91 | 92 | RSpec::Matchers.define :have_not_purged_urls do |urls| 93 | match do |actual| 94 | text = actual.is_a?(Array) ? actual.map{|v| "\n#{v} ->"}.join : actual 95 | urls.none? do |url| 96 | text.match(/\n#{url} ->/) 97 | end 98 | end 99 | 100 | failure_message do |actual| 101 | "expected that #{actual} would not #{description}" 102 | end 103 | 104 | failure_message_when_negated do |actual| 105 | "expected that #{actual} would #{description}" 106 | end 107 | 108 | description do 109 | "have purged none of the urls: #{urls.join(", ")}" 110 | end 111 | end 112 | 113 | RSpec.configure do |config| 114 | config.before(:suite) do 115 | FileUtils.mkdir_p NginxTestHelper.nginx_tests_tmp_dir 116 | system("redis-server --port #{redis_port} --unixsocket #{redis_unix_socket} --unixsocketperm 777 --daemonize yes --pidfile #{redis_unix_socket.gsub("socket", "pid")}") 117 | end 118 | 119 | config.after(:suite) do 120 | system("kill `cat #{redis_unix_socket.gsub("socket", "pid")}`") 121 | end 122 | 123 | config.before(:each) do 124 | clear_database 125 | FileUtils.chmod_R(0700, proxy_cache_path) if File.exist?(proxy_cache_path) 126 | FileUtils.rm_rf Dir["#{proxy_cache_path}/**"] 127 | FileUtils.mkdir_p proxy_cache_path 128 | end 129 | 130 | config.after(:each) do 131 | NginxTestHelper::Config.delete_config_and_log_files(config_id) if has_passed? 132 | redis.quit 133 | @redis = nil 134 | end 135 | config.order = "random" 136 | config.run_all_when_everything_filtered = true 137 | end 138 | 139 | -------------------------------------------------------------------------------- /test/sync_memory_spec.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path("./spec_helper", File.dirname(__FILE__)) 2 | 3 | describe "Selective Cache Purge Module Sync Memory" do 4 | let!(:config) do 5 | { } 6 | end 7 | 8 | let(:number_of_files_on_cache) { 500 } 9 | 10 | before :each do 11 | FileUtils.rm_rf Dir["#{proxy_cache_path}_2/**"] 12 | FileUtils.mkdir_p "#{proxy_cache_path}_2" 13 | 14 | nginx_version = Gem::Version.new(`#{NginxTestHelper.nginx_executable} -V 2>&1`.match(/nginx version\: nginx\/([\d\.]+)/)[1]) 15 | zip_name = "cache.zip" 16 | if nginx_version.between?(Gem::Version.new('1.7.3'), Gem::Version.new('1.7.7')) 17 | zip_name = "cache_2.zip" 18 | elsif nginx_version.between?(Gem::Version.new('1.7.8'), Gem::Version.new('1.11.9')) 19 | zip_name = "cache_3.zip" 20 | elsif nginx_version >= Gem::Version.new('1.11.10') 21 | zip_name = "cache_4.zip" 22 | end 23 | 24 | Zip::File.open(File.expand_path("./assets/#{zip_name}", File.dirname(__FILE__))) do |zipfile| 25 | zipfile.restore_permissions = true 26 | zipfile.each do |file| 27 | FileUtils.mkdir_p File.dirname("#{proxy_cache_path}/#{file}") 28 | zipfile.extract(file, "#{proxy_cache_path}/#{file}") 29 | end 30 | end 31 | end 32 | 33 | 34 | it "should be possible access the server during the load and sync process" do 35 | nginx_run_server(config, timeout: 200) do 36 | EventMachine.run do 37 | request_sent = 0 38 | request_received = 0 39 | timer = EventMachine::PeriodicTimer.new(0.05) do 40 | request_sent += 1 41 | req = EventMachine::HttpRequest.new("http://#{nginx_host}:#{nginx_port}/index.html", connect_timeout: 10, inactivity_timeout: 15).get 42 | req.callback do 43 | fail("Request failed with error #{req.response_header.status}") if req.response_header.status != 200 44 | request_received += 1 45 | end 46 | req.errback do 47 | fail("Request failed!!! #{req.error}") 48 | EventMachine.stop 49 | end 50 | end 51 | 52 | EventMachine::PeriodicTimer.new(0.5) do 53 | count = get_database_entries_for('*').count 54 | if count >= number_of_files_on_cache 55 | timer.cancel 56 | expect(request_received).to be_within(5).of(request_sent) 57 | EventMachine.stop 58 | end 59 | end 60 | end 61 | end 62 | end 63 | 64 | it "should sync all cache zones" do 65 | FileUtils.cp_r Dir["#{proxy_cache_path}/*"], "#{proxy_cache_path}_2" 66 | additional_config = "proxy_cache_path #{proxy_cache_path}_2 levels=1:2 keys_zone=zone2:10m inactive=10d max_size=100m loader_files=100 loader_sleep=1;" 67 | 68 | nginx_run_server(config.merge({additional_config: additional_config}), timeout: 200) do 69 | EventMachine.run do 70 | EventMachine::PeriodicTimer.new(0.5) do 71 | count = get_database_entries_for('*').count 72 | if count >= (2 * number_of_files_on_cache) 73 | EventMachine.stop 74 | end 75 | end 76 | end 77 | end 78 | end 79 | 80 | it "should clear old entries after sync" do 81 | insert_entry_on_database('unkown_zone', 'proxy', '/115/index.html', '/b/65/721a470787d1f40cdb6307c9108de65b', Time.now.to_i + 600) 82 | insert_entry_on_database('zone', 'proxy', '/old_file/index.html', '/f/26/079ed7046775b65ab8983b26750e426f', Time.now.to_i + 600) 83 | nginx_run_server(config, timeout: 100) do 84 | EventMachine.run do 85 | EventMachine::PeriodicTimer.new(0.5) do 86 | count = get_database_entries_for('*').count 87 | if count >= number_of_files_on_cache 88 | sleep 1.5 89 | expect(get_database_entries_for_zone('unkown_zone').count).to eql(0) 90 | expect(get_database_entries_for_zone('zone').count).to eql(number_of_files_on_cache) 91 | EventMachine.stop 92 | end 93 | end 94 | end 95 | end 96 | end 97 | end 98 | --------------------------------------------------------------------------------