├── .gitignore ├── CHANGES ├── INSTALL ├── LICENSE ├── Makefile ├── README ├── package.sh └── src ├── apache ├── jitify_apache_glue.c ├── jitify_apache_glue.h └── mod_jitify.c ├── core ├── jitify.h ├── jitify_array.c ├── jitify_css.c ├── jitify_css.h ├── jitify_css_lexer.rl ├── jitify_css_lexer_common.rl ├── jitify_html.c ├── jitify_html.h ├── jitify_html_lexer.rl ├── jitify_js.c ├── jitify_js.h ├── jitify_js_lexer.rl ├── jitify_lexer.c ├── jitify_lexer.h ├── jitify_lexer_common.rl ├── jitify_pool.c └── jitify_stream.c ├── nginx ├── config ├── config.make ├── jitify_nginx_glue.c ├── jitify_nginx_glue.h └── mod_jitify.c └── tools └── jitify.c /.gitignore: -------------------------------------------------------------------------------- 1 | .* 2 | *.o 3 | *.lo 4 | *.so 5 | *.la 6 | *.slo 7 | *.a 8 | !.gitignore 9 | jitify_*_lexer.c 10 | build 11 | -------------------------------------------------------------------------------- /CHANGES: -------------------------------------------------------------------------------- 1 | Changes with jitify-core 0.9.3 01 Jul 2011 2 | 3 | *) Fixes to the nginx configuration script to work on 4 | systems that only allow traditional Bourne shell syntax 5 | 6 | Changes with jitify-core 0.9.2 12 Apr 2011 7 | 8 | *) Minor fixes for compatibility with nginx-1.0 9 | 10 | Changes with jitify-core 0.9.1 08 Jul 2010 11 | 12 | *) Feature: mod_jitify for nginx 0.8.x 13 | 14 | *) Bugfix: regular expressions following ':' in JavaScript 15 | were not parsed properly. 16 | 17 | *) Change: refactored HTML lexical grammar to help support 18 | link rewriting in the future. 19 | 20 | Changes with jitify-core 0.9.0 12 Jun 2010 21 | 22 | *) First public version 23 | 24 | *) Feature: minification of CSS, HTML, and JS 25 | 26 | *) Feature: jitify command-line tool 27 | 28 | *) Feature: mod_jitify for Apache 2.2.x 29 | -------------------------------------------------------------------------------- /INSTALL: -------------------------------------------------------------------------------- 1 | Sorry, there isn't a configure script for jitify-core yet. 2 | 3 | There are three targets you can build from jitify-core: 4 | 1. Jitify Tools, the command-line application 5 | 2. Jitify for Apache, the module for Apache 2.2 6 | 3. Jitify for nginx, the module for nginx-1.0 7 | 8 | To build Jitify Tools: 9 | 1. cd to the root of the jitify-core source distribution 10 | 2. make tools 11 | 3. The Jitify Tools executable is now at build/jitify 12 | 4. cp build/jitify /wherever/you/want/to/put/it 13 | 14 | To build Jitify for Apache: 15 | 1. cd to the root of the jitify-core source distribution 16 | 2. Open the Makefile in your text editor of choice, and 17 | CHANGE the APACHE_HOME declaration at the top to point 18 | to the base directory of your Apache installation. The 19 | Makefile looks for Apache's apxs build tool under 20 | $APACHE_HOME/bin 21 | 3. make apache 22 | 4. The Jitify for Apache dynamically loadable module is 23 | now at build/mod_jitify.so 24 | 5. cp build/mod_jitify.so /wherever-your-apache-installation-is/modules 25 | 6. Add these lines to your Apache httpd.conf: 26 | LoadModule jitify_module modules/mod_jitify.so 27 | 28 | # Enable the minification transform 29 | Minify On 30 | 31 | 7. Restart Apache 32 | 33 | To build Jitify for nginx: 34 | 1. If you don't already the source code for nginx 1.0.x 35 | download and unpack it. 36 | 2. In your nginx-1.0.x source directory, run the configure script: 37 | CFLAGS="-O3 -g" ./configure \ 38 | --prefix=/directory/where/you/want/to/install/nginx \ 39 | --add-module=/dir/containing/jitify-core/jitify-core-version/src/nginx \ 40 | any additional nginx config options you need... 41 | The setting of those CFLAGS arguments is optional. "-O3" enables 42 | a relatively thorough level of optimization in the compiler, 43 | and "-g" adds debugging metadata to make the resulting nginx 44 | program easier to troubleshoot with gdb. 45 | 3. Still within the nginx-1.0.x source directory, 46 | make 47 | sudo make install 48 | 4. Edit your nginx.conf to add these lines inside the "http { ... }" block: 49 | minify on; 50 | 5. Restart nginx. 51 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2009-2010 Brian Pane 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Edit the definition of APACHE_HOME to point to the base of your Apache installation 2 | APACHE_HOME=$(HOME)/local/apache-2.2 3 | 4 | # Locations of programs needed during compilation 5 | APXS=$(APACHE_HOME)/bin/apxs 6 | RAGEL=ragel 7 | 8 | CFLAGS=-g -O3 -Werror -Wall 9 | APXS_CFLAGS=-Wc,"$(CFLAGS)" 10 | RAGELFLAGS=-G2 11 | 12 | build/%.o: src/%.c 13 | $(CC) $(CFLAGS) $(CPPFLAGS) -c -Isrc/core -o $@ $< 14 | 15 | CORE_SRCS= \ 16 | src/core/jitify_array.c \ 17 | src/core/jitify_css.c \ 18 | src/core/jitify_css_lexer.c \ 19 | src/core/jitify_html.c \ 20 | src/core/jitify_html_lexer.c \ 21 | src/core/jitify_js.c \ 22 | src/core/jitify_js_lexer.c \ 23 | src/core/jitify_lexer.c \ 24 | src/core/jitify_pool.c \ 25 | src/core/jitify_stream.c 26 | 27 | CORE_TMP_OBJS= $(CORE_SRCS:%.c=%.o) 28 | CORE_OBJS=$(CORE_TMP_OBJS:src/%=build/%) 29 | 30 | all: 31 | @echo 32 | @echo "Jitify build options:" 33 | @echo 34 | @echo " To build the Jitify command-line application," 35 | @echo " make tools" 36 | @echo 37 | @echo " To build mod_jitify for Apache," 38 | @echo " make apache" 39 | @#echo 40 | @#echo " To build mod_jitify for nginx," 41 | @#echo " make nginx" 42 | @echo 43 | @echo "Developer build options (useful only if you're modifying the Jitify core):" 44 | @echo 45 | @echo " To regenerate all C source files from the Ragel grammar files," 46 | @echo " make ragel" 47 | @echo 48 | 49 | build-prep: 50 | @mkdir -p build/core build/tools 51 | 52 | ragel: 53 | $(RAGEL) $(RAGELFLAGS) -o src/core/jitify_css_lexer.c src/core/jitify_css_lexer.rl 54 | $(RAGEL) $(RAGELFLAGS) -o src/core/jitify_html_lexer.c src/core/jitify_html_lexer.rl 55 | $(RAGEL) $(RAGELFLAGS) -o src/core/jitify_js_lexer.c src/core/jitify_js_lexer.rl 56 | 57 | TOOL_TARGETS=build/jitify 58 | TOOL_OBJS=$(CORE_OBJS) build/tools/jitify.o 59 | 60 | tools: $(TOOL_TARGETS) 61 | 62 | build/jitify: build-prep $(TOOL_OBJS) 63 | $(CC) -o $@ $(TOOL_OBJS) 64 | 65 | APACHE_TARGETS=build/mod_jitify.so 66 | APACHE_SRCS=$(CORE_SRCS) src/apache/mod_jitify.c src/apache/jitify_apache_glue.c 67 | apache: build-prep $(APACHE_TARGETS) 68 | 69 | build/mod_jitify.so: $(APACHE_SRCS) 70 | $(APXS) -c -Wc,-g -Wc,-O3 -Wc,-Werror -Wc,-Wall -o $@ -Isrc/core $(APACHE_SRCS) 71 | cp -f build/.libs/mod_jitify.so build 72 | 73 | clean: 74 | -rm -rf build/* 75 | -find src \( -name '*.o' -o -name '*.lo' -o -name '*.slo' \) -exec rm -f {} \; 76 | -find src -name '.libs' -exec rm -rf {} \; 77 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | jitify-core v0.9.3 2 | 3 | This distribution contains the source code for: 4 | Jitify Tools 5 | Jitify for Apache 6 | Jitify for nginx 7 | 8 | For compilation and installation instructions, please see the INSTALL file. 9 | 10 | Documentation is available online at http://www.jitify.com/ 11 | 12 | -------------------------------------------------------------------------------- /package.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Script to create a .tar.bz2 package of the jitify-core source 4 | # Usage: 5 | # 1. Check out the revision of jitify-core that you want to package 6 | # 2. make clean ragel 7 | # 3. package.sh version-number 8 | # e.g., package.sh 1.0.3 9 | 10 | if [ $# -ne 1 ] ;then 11 | echo "Usage: package.sh version-number" 12 | exit 1 13 | fi 14 | 15 | VERSION=$1 16 | STAGEDIR="jitify-core-$VERSION" 17 | mkdir -p $STAGEDIR 18 | rm -rf $STAGEDIR/* 19 | FILES="CHANGES INSTALL LICENSE README Makefile src" 20 | find $FILES |cpio -dump $STAGEDIR 21 | tar cf - $STAGEDIR | bzip2 > jitify-core-$VERSION.tar.bz2 22 | -------------------------------------------------------------------------------- /src/apache/jitify_apache_glue.c: -------------------------------------------------------------------------------- 1 | #define JITIFY_INTERNAL 2 | #include "jitify_apache_glue.h" 3 | 4 | static void *apache_malloc_wrapper(jitify_pool_t *pool, size_t len) 5 | { 6 | return apr_palloc(pool->state, len); 7 | } 8 | 9 | static void *apache_calloc_wrapper(jitify_pool_t *pool, size_t len) 10 | { 11 | return apr_pcalloc(pool->state, len); 12 | } 13 | 14 | static void apache_free_wrapper(jitify_pool_t *pool, void *block) 15 | { 16 | } 17 | 18 | jitify_pool_t *jitify_apache_pool_create(apr_pool_t *pool) 19 | { 20 | jitify_pool_t *jpool = apr_pcalloc(pool, sizeof(*jpool)); 21 | jpool->state = pool; 22 | jpool->malloc = apache_malloc_wrapper; 23 | jpool->calloc = apache_calloc_wrapper; 24 | jpool->free = apache_free_wrapper; 25 | return jpool; 26 | } 27 | 28 | #define JITIFY_OUTPUT_BUF_SIZE 8000 29 | 30 | static int apache_brigade_write(jitify_output_stream_t *stream, const void *data, size_t len) 31 | { 32 | apr_bucket_brigade *bb = stream->state; 33 | /* Case 1 of 3: the target brigade ends with a heap 34 | * buffer that has enough space left over to hold 35 | * this write 36 | */ 37 | if (!APR_BRIGADE_EMPTY(bb)) { 38 | apr_bucket *b; 39 | 40 | b = APR_BRIGADE_LAST(bb); 41 | if (APR_BUCKET_IS_HEAP(b)) { 42 | apr_bucket_heap *h; 43 | size_t avail; 44 | 45 | h = b->data; 46 | avail = h->alloc_len - (b->start + b->length); 47 | if (avail >= len) { 48 | memcpy(h->base + b->start + b->length, data, len); 49 | b->length += len; 50 | return len; 51 | } 52 | } 53 | } 54 | 55 | /* Case 2 of 3: this write is smaller than the normal 56 | * buffer size, so we allocate a full-sized buffer in 57 | * anticipation of more small writes to follow 58 | */ 59 | if (len <= JITIFY_OUTPUT_BUF_SIZE) { 60 | apr_bucket *b; 61 | char *heap_buf; 62 | 63 | heap_buf = apr_bucket_alloc(JITIFY_OUTPUT_BUF_SIZE, bb->bucket_alloc); 64 | b = apr_bucket_heap_create(heap_buf, JITIFY_OUTPUT_BUF_SIZE, apr_bucket_free, 65 | bb->bucket_alloc); 66 | memcpy(heap_buf, data, len); 67 | b->length = len; 68 | APR_BRIGADE_INSERT_TAIL(bb, b); 69 | } 70 | 71 | /* Case 3 of 3: this write is larger than the normal 72 | * buffer size, so we wrap it in a bucket as-is 73 | */ 74 | else { 75 | apr_bucket *b; 76 | 77 | b = apr_bucket_heap_create(data, len, NULL, bb->bucket_alloc); 78 | APR_BRIGADE_INSERT_TAIL(bb, b); 79 | } 80 | return len; 81 | } 82 | 83 | jitify_output_stream_t *jitify_apache_output_stream_create(jitify_pool_t *pool) 84 | { 85 | jitify_output_stream_t *stream = jitify_calloc(pool, sizeof(*stream)); 86 | stream->pool = pool; 87 | stream->write = apache_brigade_write; 88 | return stream; 89 | } 90 | -------------------------------------------------------------------------------- /src/apache/jitify_apache_glue.h: -------------------------------------------------------------------------------- 1 | #ifndef jitify_apache_glue_h 2 | #define jitify_apache_glue_h 3 | 4 | #include 5 | #include 6 | 7 | #include "jitify.h" 8 | 9 | extern jitify_pool_t *jitify_apache_pool_create(apr_pool_t *pool); 10 | 11 | extern jitify_output_stream_t *jitify_apache_output_stream_create(jitify_pool_t *pool); 12 | 13 | #endif /* !defined(jitify_apache_glue_h) */ 14 | -------------------------------------------------------------------------------- /src/apache/mod_jitify.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #define JITIFY_INTERNAL 10 | #include "jitify_apache_glue.h" 11 | 12 | module AP_MODULE_DECLARE_DATA jitify_module; 13 | 14 | #define JITIFY_FILTER_KEY "JITIFY" 15 | 16 | #define JITIFY_NOTE_KEY "jitify_ctx" 17 | 18 | typedef struct { 19 | int minify; /* 0 for false, >0 for true, <0 for unset */ 20 | } jitify_dir_conf_t; 21 | 22 | typedef struct { 23 | const char *orig_accept_encoding; 24 | } jitify_request_ctx_t; 25 | 26 | typedef struct { 27 | bool response_started; 28 | jitify_pool_t *pool; 29 | jitify_lexer_t *lexer; 30 | jitify_output_stream_t *out; 31 | } jitify_filter_ctx_t; 32 | 33 | static request_rec *main_request(request_rec *r) 34 | { 35 | if (r && r->main) { 36 | return r->main; 37 | } 38 | else { 39 | return r; 40 | } 41 | } 42 | 43 | static jitify_filter_ctx_t *jitify_filter_init(ap_filter_t *f) 44 | { 45 | jitify_pool_t *pool = jitify_apache_pool_create(f->r->pool); 46 | jitify_filter_ctx_t *ctx = jitify_calloc(pool, sizeof(*ctx)); 47 | jitify_dir_conf_t *jconf = ap_get_module_config(f->r->per_dir_config, &jitify_module); 48 | request_rec *r_main; 49 | jitify_request_ctx_t *jctx; 50 | ctx->pool = pool; 51 | if (jconf->minify > 0) { 52 | jitify_output_stream_t *out = jitify_apache_output_stream_create(pool); 53 | ctx->lexer = jitify_lexer_for_content_type(f->r->content_type, pool, out); 54 | if (ctx->lexer) { 55 | ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, f->r, "found lexer for content-type %s for %s", f->r->content_type, f->r->uri); 56 | jitify_lexer_set_minify_rules(ctx->lexer, 1, 1); 57 | } 58 | else { 59 | ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, f->r, "no lexer for content-type %s for %s", f->r->content_type, f->r->uri); 60 | } 61 | ctx->out = out; 62 | } 63 | else { 64 | ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, f->r, "no transforms enabled for %s, skipping lexer", f->r->uri); 65 | } 66 | r_main = main_request(f->r); 67 | jctx = (jitify_request_ctx_t *)apr_table_get(r_main->notes, JITIFY_NOTE_KEY); 68 | if (jctx && jctx->orig_accept_encoding) { 69 | ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r_main, "Restoring Accept-Encoding for %s", r_main->uri); 70 | apr_table_setn(r_main->headers_in, "Accept-Encoding", jctx->orig_accept_encoding); 71 | } 72 | else { 73 | ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, f->r, "Not restoring Accept-Encoding because %s for %s", 74 | (jctx == NULL) ? "there is no request context" : "original Accept-Encoding was NULL", r_main->uri); 75 | } 76 | return ctx; 77 | } 78 | 79 | #define DEFAULT_ERR_LEN 80 80 | 81 | static apr_status_t jitify_filter(ap_filter_t *f, apr_bucket_brigade *bb) 82 | { 83 | apr_status_t rv; 84 | jitify_filter_ctx_t *ctx; 85 | apr_bucket_brigade *out; 86 | ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, f->r, "in jitify filter for %s", f->r->uri); 87 | if (f->ctx == NULL) { 88 | f->ctx = jitify_filter_init(f); 89 | } 90 | ctx = f->ctx; 91 | if (!ctx || !ctx->lexer) { 92 | ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, f->r, "no lexer for %s, skipping", f->r->uri); 93 | return ap_pass_brigade(f->next, bb); 94 | } 95 | 96 | if (!ctx->response_started) { 97 | apr_table_unset(f->r->headers_out, "Content-Length"); 98 | apr_table_unset(f->r->headers_out, "Last-modified"); 99 | apr_table_unset(f->r->headers_out, "Accept-Ranges"); 100 | ctx->response_started = 1; 101 | } 102 | out = apr_brigade_create(f->r->pool, f->c->bucket_alloc); 103 | ctx->out->state = out; 104 | rv = APR_SUCCESS; 105 | while (!APR_BRIGADE_EMPTY(bb)) { 106 | apr_bucket *b; 107 | const char *data; 108 | apr_size_t len; 109 | b = APR_BRIGADE_FIRST(bb); 110 | APR_BUCKET_REMOVE(b); 111 | rv = apr_bucket_read(b, &data, &len, APR_BLOCK_READ); 112 | if (rv != APR_SUCCESS) { 113 | break; 114 | } 115 | if (len > 0) { 116 | const char *err; 117 | ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, f->r, "scanning %d bytes of %s", (int)len, f->r->uri); 118 | jitify_lexer_scan(ctx->lexer, data, len, 0); 119 | err = jitify_lexer_get_err(ctx->lexer); 120 | if (err) { 121 | char err_buf[DEFAULT_ERR_LEN + 1]; 122 | size_t err_len = DEFAULT_ERR_LEN; 123 | size_t max_err_len = (data + len) - err; 124 | if (err_len > max_err_len) { 125 | err_len = max_err_len; 126 | } 127 | memcpy(err_buf, err, err_len); 128 | err_buf[err_len] = 0; 129 | ap_log_rerror(APLOG_MARK, APLOG_WARNING, 0, f->r, "parse error in %s near '%s', entering failsafe mode", f->r->uri, err_buf); 130 | } 131 | } 132 | if (APR_BUCKET_IS_EOS(b)) { 133 | size_t processing_time_in_usec = jitify_lexer_get_processing_time(ctx->lexer); 134 | size_t bytes_in = jitify_lexer_get_bytes_in(ctx->lexer); 135 | size_t bytes_out = jitify_lexer_get_bytes_out(ctx->lexer); 136 | ap_log_rerror(APLOG_MARK, APLOG_INFO, 0, f->r, "Jitify stats: bytes_in=%lu bytes_out=%lu, nsec/byte=%lu for %s", 137 | (unsigned long)bytes_in, (unsigned long)bytes_out, 138 | (unsigned long)(bytes_in ? processing_time_in_usec * 1000 / bytes_in : 0), 139 | f->r->uri); 140 | jitify_lexer_scan(ctx->lexer, NULL, 0, 1); 141 | APR_BRIGADE_INSERT_TAIL(out, b); 142 | } 143 | else { 144 | apr_bucket_destroy(b); 145 | } 146 | } 147 | ctx->out->state = NULL; 148 | if (APR_BRIGADE_EMPTY(out)) { 149 | return APR_SUCCESS; 150 | } 151 | else { 152 | return ap_pass_brigade(f->next, out); 153 | } 154 | } 155 | 156 | static int jitify_translate_name(request_rec *r) 157 | { 158 | jitify_dir_conf_t *jconf; 159 | request_rec *r_main; 160 | ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, "In jitify_translate_name for %s", r->uri); 161 | r_main = main_request(r); 162 | jconf = ap_get_module_config(r_main->per_dir_config, &jitify_module); 163 | if (jconf && (jconf->minify > 0)) { 164 | jitify_request_ctx_t *jctx; 165 | ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, "Preparing request for Jitify processing"); 166 | jctx = (jitify_request_ctx_t *)apr_table_get(r_main->notes, JITIFY_NOTE_KEY); 167 | if (!jctx) { 168 | jctx = apr_pcalloc(r_main->pool, sizeof(*jctx)); 169 | apr_table_setn(r_main->notes, JITIFY_NOTE_KEY, (void *)jctx); 170 | jctx->orig_accept_encoding = apr_table_get(r_main->headers_in, "Accept-Encoding"); 171 | if (jctx->orig_accept_encoding) { 172 | ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, "Hiding Accept-Encoding until output filter for %s", r_main->uri); 173 | apr_table_unset(r_main->headers_in, "Accept-Encoding"); 174 | } 175 | } 176 | } 177 | return DECLINED; 178 | } 179 | 180 | static int jitify_fixup(request_rec *r) 181 | { 182 | ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, "in jitify fixup for %s", r->uri); 183 | ap_add_output_filter(JITIFY_FILTER_KEY, NULL, r, r->connection); 184 | return DECLINED; 185 | } 186 | 187 | static const command_rec jitify_cmds[] = 188 | { 189 | AP_INIT_FLAG("Minify", ap_set_flag_slot, APR_OFFSETOF(jitify_dir_conf_t, minify), 190 | RSRC_CONF|ACCESS_CONF, "Enable dynamic content minification"), 191 | /* 192 | AP_SOMETHING("CDNify", something, something, 193 | RSRC_CONF|ACCESS_CONF, "Rewrite links to use a new base URL"), 194 | */ 195 | {NULL} 196 | }; 197 | 198 | static void register_jitify_hooks(apr_pool_t *p) 199 | { 200 | ap_hook_translate_name(jitify_translate_name, NULL, NULL, APR_HOOK_REALLY_FIRST); 201 | ap_hook_fixups(jitify_fixup, NULL, NULL, APR_HOOK_REALLY_FIRST); 202 | ap_register_output_filter(JITIFY_FILTER_KEY, jitify_filter, NULL, AP_FTYPE_RESOURCE); 203 | } 204 | 205 | static void *create_jitify_dir_config(apr_pool_t *pool, char *unused) 206 | { 207 | jitify_dir_conf_t *conf = apr_pcalloc(pool, sizeof(*conf)); 208 | conf->minify = -1; 209 | return conf; 210 | } 211 | 212 | static void *merge_jitify_dir_config(apr_pool_t *pool, void *base_conf, void *add_conf) 213 | { 214 | jitify_dir_conf_t *merged = apr_pcalloc(pool, sizeof(*merged)); 215 | jitify_dir_conf_t *base = base_conf; 216 | jitify_dir_conf_t *add = add_conf; 217 | if (add->minify < 0) { 218 | merged->minify = base->minify; 219 | } 220 | else { 221 | merged->minify = add->minify; 222 | } 223 | return merged; 224 | } 225 | 226 | module AP_MODULE_DECLARE_DATA jitify_module = 227 | { 228 | STANDARD20_MODULE_STUFF, 229 | create_jitify_dir_config, /* create directory-level config */ 230 | merge_jitify_dir_config, /* merge directory-level config */ 231 | NULL, /* create server-level config */ 232 | NULL, /* merge server-level config */ 233 | jitify_cmds, /* configuration commands */ 234 | register_jitify_hooks /* register request lifecycle hooks */ 235 | }; 236 | -------------------------------------------------------------------------------- /src/core/jitify.h: -------------------------------------------------------------------------------- 1 | #ifndef jitify_h 2 | #define jitify_h 3 | 4 | #include 5 | #include 6 | 7 | /* Jitify Core external API */ 8 | 9 | /* Basic types */ 10 | 11 | typedef enum { JITIFY_OK, JITIFY_ERROR, JITIFY_AGAIN } jitify_status_t; 12 | 13 | typedef struct jitify_array_s jitify_array_t; 14 | 15 | /* Memory management */ 16 | 17 | typedef struct jitify_pool_s jitify_pool_t; 18 | 19 | extern void *jitify_malloc(jitify_pool_t *pool, size_t length); 20 | 21 | extern void *jitify_calloc(jitify_pool_t *pool, size_t length); 22 | 23 | extern void jitify_free(jitify_pool_t *pool, void *object); 24 | 25 | extern void jitify_pool_destroy(jitify_pool_t *pool); 26 | 27 | extern jitify_pool_t *jitify_malloc_pool_create(); 28 | 29 | /* Dynamically growing arrays */ 30 | 31 | extern jitify_array_t *jitify_array_create(jitify_pool_t *pool, size_t element_size); 32 | 33 | extern size_t jitify_array_length(const jitify_array_t *array); 34 | 35 | extern void *jitify_array_get(jitify_array_t *array, size_t index); 36 | 37 | extern void *jitify_array_push(jitify_array_t *array); 38 | 39 | extern void jitify_array_clear(jitify_array_t *array); 40 | 41 | extern void jitify_array_destroy(jitify_array_t *array); 42 | 43 | /* I/O */ 44 | 45 | typedef struct jitify_output_stream_s jitify_output_stream_t; 46 | 47 | extern void jitify_output_stream_destroy(jitify_output_stream_t *stream); 48 | 49 | extern jitify_output_stream_t *jitify_stdio_output_stream_create(jitify_pool_t *pool, FILE *out); 50 | 51 | /* Content parsing */ 52 | 53 | typedef struct jitify_lexer_s jitify_lexer_t; 54 | 55 | extern jitify_lexer_t *jitify_lexer_for_content_type(const char *content_type, jitify_pool_t *pool, jitify_output_stream_t *out); 56 | 57 | extern jitify_lexer_t *jitify_css_lexer_create(jitify_pool_t *pool, jitify_output_stream_t *out); 58 | 59 | extern jitify_lexer_t *jitify_html_lexer_create(jitify_pool_t *pool, jitify_output_stream_t *out); 60 | 61 | extern jitify_lexer_t *jitify_js_lexer_create(jitify_pool_t *pool, jitify_output_stream_t *out); 62 | 63 | extern void jitify_lexer_set_minify_rules(jitify_lexer_t *lexer, int remove_space, int remove_comments); 64 | 65 | extern void jitify_lexer_add_cdnify_rule(jitify_lexer_t *lexer, const char *prefix, const char *replacement); 66 | 67 | extern void jitify_lexer_set_max_setaside(jitify_lexer_t *lexer, size_t max); 68 | 69 | extern size_t jitify_lexer_get_bytes_in(jitify_lexer_t *lexer); 70 | 71 | extern size_t jitify_lexer_get_bytes_out(jitify_lexer_t *lexer); 72 | 73 | extern size_t jitify_lexer_get_processing_time(jitify_lexer_t *lexer); 74 | 75 | extern const char *jitify_lexer_get_err(jitify_lexer_t *lexer); 76 | 77 | extern int jitify_write(jitify_lexer_t *lexer, const void *data, size_t length); 78 | 79 | /** 80 | * @return number of bytes scanned, or a negative number if an unrecoverable error occurs 81 | */ 82 | extern int jitify_lexer_scan(jitify_lexer_t *lexer, const void *data, size_t len, int is_eof); 83 | 84 | extern void jitify_lexer_destroy(jitify_lexer_t *lexer); 85 | 86 | #ifdef JITIFY_INTERNAL 87 | 88 | struct jitify_pool_s { 89 | void *state; 90 | void *(*malloc)(jitify_pool_t *p, size_t size); 91 | void *(*calloc)(jitify_pool_t *p, size_t size); 92 | void (*free)(jitify_pool_t *p, void *object); 93 | void (*cleanup)(jitify_pool_t *p); 94 | }; 95 | 96 | struct jitify_output_stream_s { 97 | void *state; 98 | jitify_pool_t *pool; 99 | int (*write)(jitify_output_stream_t *stream, const void *data, size_t length); 100 | void (*cleanup)(jitify_output_stream_t *stream); 101 | }; 102 | 103 | extern void jitify_err_checkpoint(jitify_lexer_t *lexer); 104 | 105 | #endif /* JITIFY_INTERNAL */ 106 | 107 | #endif /* !defined(jitify_h) */ 108 | -------------------------------------------------------------------------------- /src/core/jitify_array.c: -------------------------------------------------------------------------------- 1 | #include 2 | #define JITIFY_INTERNAL 3 | #include "jitify.h" 4 | 5 | struct jitify_array_s { 6 | jitify_pool_t *pool; 7 | size_t len; 8 | size_t size; 9 | size_t element_size; 10 | char *data; 11 | }; 12 | 13 | #define INITIAL_ARRAY_SIZE 2 14 | 15 | jitify_array_t *jitify_array_create(jitify_pool_t *pool, size_t element_size) 16 | { 17 | jitify_array_t *array = jitify_malloc(pool, sizeof(*array)); 18 | array->pool = pool; 19 | array->len = 0; 20 | array->size = INITIAL_ARRAY_SIZE; 21 | array->element_size = element_size; 22 | array->data = jitify_malloc(pool, array->size * element_size); 23 | return array; 24 | } 25 | 26 | size_t jitify_array_length(const jitify_array_t *array) 27 | { 28 | return array->len; 29 | } 30 | 31 | void *jitify_array_get(jitify_array_t *array, size_t index) 32 | { 33 | if (index >= array->len) { 34 | return NULL; 35 | } 36 | else { 37 | return array->data + index * array->element_size; 38 | } 39 | } 40 | 41 | void *jitify_array_push(jitify_array_t *array) 42 | { 43 | if (array->len == array->size) { 44 | char *new_data; 45 | array->size *= 2; 46 | new_data = jitify_malloc(array->pool, array->size * array->element_size); 47 | memcpy(new_data, array->data, array->len * array->element_size); 48 | jitify_free(array->pool, array->data); 49 | array->data = new_data; 50 | } 51 | memset(array->data + array->len * array->element_size, 0, array->element_size); 52 | return array->data + array->len++ * array->element_size; 53 | } 54 | 55 | void jitify_array_clear(jitify_array_t *array) 56 | { 57 | array->len = 0; 58 | } 59 | 60 | void jitify_array_destroy(jitify_array_t *array) 61 | { 62 | jitify_free(array->pool, array->data); 63 | jitify_free(array->pool, array); 64 | } 65 | -------------------------------------------------------------------------------- /src/core/jitify_css.c: -------------------------------------------------------------------------------- 1 | #define JITIFY_INTERNAL 2 | #include "jitify_css.h" 3 | 4 | extern int jitify_css_scan(jitify_lexer_t *lexer, const void *data, size_t length, int is_eof); 5 | 6 | jitify_token_type_t jitify_type_css_selector = "CSS selector"; 7 | jitify_token_type_t jitify_type_css_term = "CSS term"; 8 | jitify_token_type_t jitify_type_css_comment = "CSS comment"; 9 | jitify_token_type_t jitify_type_css_optional_whitespace = "CSS optional space"; 10 | jitify_token_type_t jitify_type_css_required_whitespace = "CSS required space"; 11 | jitify_token_type_t jitify_type_css_url = "CSS URL"; 12 | 13 | 14 | static jitify_status_t css_transform(jitify_lexer_t *lexer, const void *data, size_t length, size_t offset) 15 | { 16 | jitify_css_state_t *state = lexer->state; 17 | if (lexer->token_type == jitify_type_css_optional_whitespace) { 18 | if (lexer->remove_space) { 19 | return JITIFY_OK; 20 | } 21 | } 22 | else if (lexer->token_type == jitify_type_css_comment) { 23 | if (lexer->remove_comments) { 24 | return JITIFY_OK; 25 | } 26 | } 27 | else if ((lexer->token_type == jitify_type_css_selector) || (lexer->token_type == jitify_type_css_term)) { 28 | if (lexer->remove_space && (state->last_token_type == lexer->token_type)) { 29 | /* The space we just skipped was actually necessary, so add a space back in */ 30 | if (jitify_write(lexer, " ", 1) < 0) { 31 | return JITIFY_ERROR; 32 | } 33 | } 34 | } 35 | state->last_token_type = lexer->token_type; 36 | if (jitify_write(lexer, data, length) < 0) { 37 | return JITIFY_ERROR; 38 | } 39 | else { 40 | return JITIFY_OK; 41 | } 42 | } 43 | 44 | static void css_cleanup(jitify_lexer_t *lexer) 45 | { 46 | jitify_free(lexer->pool, lexer->state); 47 | } 48 | 49 | jitify_lexer_t *jitify_css_lexer_create(jitify_pool_t *pool, jitify_output_stream_t *out) 50 | { 51 | jitify_lexer_t *lexer = jitify_lexer_create(pool, out); 52 | jitify_css_state_t *state = jitify_calloc(pool, sizeof(*state)); 53 | lexer->state = state; 54 | lexer->scan = jitify_css_scan; 55 | lexer->transform = css_transform; 56 | lexer->cleanup = css_cleanup; 57 | return lexer; 58 | } 59 | -------------------------------------------------------------------------------- /src/core/jitify_css.h: -------------------------------------------------------------------------------- 1 | #ifndef jitify_css_h 2 | #define jitify_css_h 3 | 4 | #include "jitify_lexer.h" 5 | 6 | #ifdef JITIFY_INTERNAL 7 | 8 | extern jitify_token_type_t jitify_type_css_selector; 9 | extern jitify_token_type_t jitify_type_css_term; 10 | extern jitify_token_type_t jitify_type_css_comment; 11 | extern jitify_token_type_t jitify_type_css_optional_whitespace; 12 | extern jitify_token_type_t jitify_type_css_required_whitespace; 13 | extern jitify_token_type_t jitify_type_css_url; 14 | 15 | typedef struct { 16 | jitify_token_type_t last_token_type; 17 | } jitify_css_state_t; 18 | 19 | #endif /* JITIFY_INTERNAL */ 20 | 21 | #endif /* !defined(jitify_css_h) */ 22 | -------------------------------------------------------------------------------- /src/core/jitify_css_lexer.rl: -------------------------------------------------------------------------------- 1 | #include 2 | #define JITIFY_INTERNAL 3 | #include "jitify_css.h" 4 | 5 | %%{ 6 | machine jitify_css; 7 | include jitify_common "jitify_lexer_common.rl"; 8 | include css_grammar "jitify_css_lexer_common.rl"; 9 | 10 | main := ( 11 | byte_order_mark? 12 | css_document 13 | ) $err(main_err); 14 | 15 | write data; 16 | }%% 17 | 18 | int jitify_css_scan(jitify_lexer_t *lexer, const void *data, size_t length, int is_eof) 19 | { 20 | const char *p = data; 21 | const char *pe = p + length; 22 | const char *eof = is_eof ? pe : NULL; 23 | jitify_css_state_t *state = lexer->state; 24 | if (!lexer->initialized) { 25 | %% write init; 26 | lexer->initialized = 1; 27 | } 28 | %% write exec; 29 | return p - (const char *)data; 30 | } 31 | -------------------------------------------------------------------------------- /src/core/jitify_css_lexer_common.rl: -------------------------------------------------------------------------------- 1 | /* CSS grammar based on W3C CSS 2.1 spec: 2 | * http://www.w3.org/TR/CSS2/grammar.html 3 | */ 4 | 5 | %%{ 6 | 7 | machine css_grammar; 8 | 9 | css_comment = ( 10 | '/*' ( any )* :>> '*/' 11 | ) >{ TOKEN_TYPE(jitify_type_css_comment); } %{ TOKEN_END; } ; 12 | 13 | optional_space = ( 14 | space+ 15 | ) >{ TOKEN_START(jitify_token_type_misc); } %{ TOKEN_TYPE(jitify_type_css_optional_whitespace); TOKEN_END; }; 16 | 17 | optional_space_or_comment = ( 18 | optional_space | 19 | css_comment 20 | )+; 21 | 22 | required_space = ( 23 | space+ 24 | ) >{ TOKEN_START(jitify_token_type_misc); } %{ TOKEN_TYPE(jitify_type_css_required_whitespace); TOKEN_END; }; 25 | 26 | combinator = ( 27 | '+' | '>' 28 | ) >{ TOKEN_START(jitify_token_type_misc); } %{ TOKEN_END; }; 29 | 30 | _nmchar = ( 31 | alnum | '_' | '-' 32 | ); 33 | 34 | _single_quoted = ( 35 | "'" ( [^'\\] | /\\./)* "'" 36 | ); 37 | 38 | _double_quoted = ( 39 | '"' ( [^"\\] | /\\./ )* '"' 40 | ); 41 | 42 | _ident = ( 43 | ( '-' | '*' | '!' )? ( alpha | '_') _nmchar* 44 | ); 45 | 46 | _hash = ( 47 | '#' _nmchar+ 48 | ); 49 | 50 | _css_class = ( 51 | '.' _ident 52 | ); 53 | 54 | _attrib = ( 55 | '[' space* 56 | _ident space* 57 | ( ( '=' | "~=" | "|=" ) space* (_ident | _single_quoted | _double_quoted) )? 58 | space* ']' 59 | ); 60 | 61 | _pseudo = ( 62 | ':'+ _ident ( '(' space* _ident space* ')' )? 63 | ); 64 | 65 | simple_selector = ( 66 | ( (_ident | '*') (_hash | _css_class | _attrib | _pseudo)* ) 67 | | 68 | (_hash | _css_class | _attrib | _pseudo)+ 69 | ) >{ TOKEN_START(jitify_token_type_misc); } %{ TOKEN_TYPE(jitify_type_css_selector); TOKEN_END; }; 70 | 71 | comma = ( 72 | ',' 73 | ) >{ TOKEN_START(jitify_token_type_misc); } %{ TOKEN_END; }; 74 | 75 | equals = ( 76 | '=' 77 | ) >{ TOKEN_START(jitify_token_type_misc); } %{ TOKEN_END; }; 78 | 79 | slash = ( 80 | '/' 81 | ) >{ TOKEN_START(jitify_token_type_misc); } %{ TOKEN_END; }; 82 | 83 | open_paren = ( 84 | '(' 85 | ) >{ TOKEN_START(jitify_token_type_misc); } %{ TOKEN_END; }; 86 | 87 | close_paren = ( 88 | ')' 89 | ) >{ TOKEN_START(jitify_token_type_misc); } %{ TOKEN_END; }; 90 | 91 | open_curly_brace = ( 92 | '{' 93 | ) >{ TOKEN_START(jitify_token_type_misc); } %{ TOKEN_END; }; 94 | 95 | close_curly_brace = ( 96 | '}' 97 | ) >{ TOKEN_START(jitify_token_type_misc); } %{ TOKEN_END; }; 98 | 99 | semicolon = ( 100 | ';' 101 | ) >{ TOKEN_START(jitify_token_type_misc); } %{ TOKEN_END; }; 102 | 103 | colon = ( 104 | ':' 105 | ) >{ TOKEN_START(jitify_token_type_misc); } %{ TOKEN_END; }; 106 | 107 | exclamation_point = ( 108 | '!' 109 | ) >{ TOKEN_START(jitify_token_type_misc); } %{ TOKEN_END; }; 110 | 111 | important = ( 112 | /important/i 113 | ) >{ TOKEN_START(jitify_token_type_misc); } %{ TOKEN_END; }; 114 | 115 | property = ( 116 | _ident 117 | ) >{ TOKEN_START(jitify_token_type_misc); } %{ TOKEN_END; }; 118 | 119 | priority = ( 120 | exclamation_point optional_space_or_comment? important 121 | ); 122 | 123 | _misc_term = ( any - ( space | '(' | ')' | ';' | '}' | '"' | "'" | ',' | '*' ) )+; 124 | 125 | _misc_function_arg = ( any - ( space | '(' | ')' | '}' | '"' | "'" | ',' | '*' ) )+; 126 | 127 | base_term = ( 128 | _single_quoted | _double_quoted | _misc_term 129 | ) >{ TOKEN_START(jitify_type_css_term); } %{ TOKEN_END; }; 130 | 131 | function_arg = ( _single_quoted | _double_quoted | _misc_function_arg ) 132 | >{ TOKEN_START(jitify_token_type_misc); } %{ TOKEN_END; } <: optional_space_or_comment?; 133 | 134 | arg_delimiter = ( ',' | '=' | '<' | '>' | '?' | ':' ) >{ TOKEN_START(jitify_token_type_misc); } %{ TOKEN_END; }; 135 | 136 | function_args = open_paren optional_space_or_comment? 137 | ( function_arg (arg_delimiter optional_space_or_comment? function_arg)* )? 138 | close_paren %{ state->last_token_type = jitify_type_css_term; }; 139 | 140 | term = base_term optional_space_or_comment? (function_args optional_space_or_comment?)?; 141 | 142 | declaration = ( 143 | property optional_space_or_comment? colon optional_space_or_comment? term <: ((comma optional_space_or_comment?)? term)** 144 | (priority optional_space_or_comment?)? 145 | ); 146 | 147 | ruleset = ( 148 | simple_selector <: optional_space_or_comment? ( (simple_selector|combinator|comma) optional_space_or_comment? )** 149 | open_curly_brace 150 | optional_space_or_comment? declaration? ( semicolon optional_space_or_comment? declaration? )* 151 | close_curly_brace 152 | ); 153 | 154 | html_open_comment = ( 155 | '' 160 | ) %{ TOKEN_END; }; 161 | 162 | media_list = ( 163 | any - '{' 164 | )** >{ TOKEN_START(jitify_token_type_misc); } %{ TOKEN_END; }; 165 | 166 | css_import = ( 167 | ( '@' /import/i ) >{ TOKEN_START(jitify_token_type_misc); } %{ TOKEN_END; } 168 | required_space term semicolon 169 | ); 170 | 171 | media = ( 172 | ( '@' /media/i ) >{ TOKEN_START(jitify_token_type_misc); } %{ TOKEN_END; } 173 | css_comment? 174 | required_space 175 | (css_comment optional_space?)* 176 | media_list :>> 177 | open_curly_brace 178 | optional_space_or_comment? 179 | (ruleset optional_space_or_comment?)* 180 | close_curly_brace 181 | ); 182 | 183 | css_charset = ( 184 | ( '@' /charset/i ) >{ TOKEN_START(jitify_token_type_misc); } %{ TOKEN_END; } 185 | required_space (base_term optional_space_or_comment?)** semicolon 186 | ); 187 | 188 | css_document = ( 189 | optional_space_or_comment | 190 | html_close_comment | 191 | html_open_comment | 192 | ruleset | 193 | css_import | 194 | css_charset | 195 | media 196 | )** >{ TOKEN_START(jitify_token_type_misc); }; 197 | 198 | }%% 199 | -------------------------------------------------------------------------------- /src/core/jitify_html.c: -------------------------------------------------------------------------------- 1 | #include 2 | #define JITIFY_INTERNAL 3 | #include "jitify_css.h" 4 | #include "jitify_html.h" 5 | 6 | jitify_token_type_t jitify_type_html_anchor_open = "HTML anchor"; 7 | jitify_token_type_t jitify_type_html_comment = "HTML comment"; 8 | jitify_token_type_t jitify_type_html_img_open = "HTML img"; 9 | jitify_token_type_t jitify_type_html_link_open = "HTML link"; 10 | jitify_token_type_t jitify_type_html_script_open = "HTML script"; 11 | jitify_token_type_t jitify_type_html_space = "HTML space"; 12 | jitify_token_type_t jitify_type_html_tag = "HTML tag"; 13 | 14 | extern int jitify_html_scan(jitify_lexer_t *lexer, const void *data, size_t length, int is_eof); 15 | 16 | static jitify_status_t html_tag_transform(jitify_lexer_t *lexer, const char *buf, size_t length, 17 | size_t starting_offset, const char *attr_name) 18 | { 19 | int modified = 0; 20 | jitify_html_state_t *state = lexer->state; 21 | size_t num_attrs; 22 | 23 | /* If we're minifying this HTML document, set 24 | * modified=true to force the tag to be 25 | * reconstructed with minimal spacing. 26 | */ 27 | if (lexer->remove_space) { 28 | modified = 1; 29 | } 30 | 31 | jitify_lexer_resolve_attrs(lexer, buf, starting_offset); 32 | num_attrs = jitify_array_length(lexer->attrs); 33 | if (num_attrs) { 34 | jitify_attr_t *attr = jitify_array_get(lexer->attrs, 0); 35 | switch (attr->key.len) { 36 | case 3: 37 | if (!strncasecmp(attr->key.data.buf, "pre", 3)) { 38 | if (state->leading_slash) { 39 | state->nominify_depth--; 40 | } 41 | else { 42 | state->nominify_depth++; 43 | } 44 | } 45 | break; 46 | case 8: 47 | if (!strncasecmp(attr->key.data.buf, "textarea", 8)) { 48 | if (state->leading_slash) { 49 | state->nominify_depth--; 50 | } 51 | else { 52 | state->nominify_depth++; 53 | } 54 | } 55 | } 56 | } 57 | 58 | /* TODO: Add link rewriting here */ 59 | 60 | if (!modified) 61 | { 62 | /* No modification needed; send the full token as-is */ 63 | if (jitify_write(lexer, buf, length) < 0) { 64 | return JITIFY_ERROR; 65 | } 66 | else { 67 | return JITIFY_OK; 68 | } 69 | } 70 | else { 71 | size_t i; 72 | /* Reconstruct the tag based on the current attr values */ 73 | jitify_write(lexer, "<", 1); 74 | if (state->leading_slash) { 75 | jitify_write(lexer, "/", 1); 76 | } 77 | num_attrs = jitify_array_length(lexer->attrs); 78 | for (i = 0; i < num_attrs; i++) { 79 | jitify_attr_t *attr = jitify_array_get(lexer->attrs, i); 80 | if (attr->key.len) { 81 | if (i) { 82 | jitify_write(lexer, " ", 1); 83 | } 84 | jitify_write(lexer, attr->key.data.buf, attr->key.len); 85 | if (attr->value.len) { 86 | jitify_write(lexer, "=", 1); 87 | if (attr->quote) { 88 | jitify_write(lexer, &(attr->quote), 1); 89 | } 90 | jitify_write(lexer, attr->value.data.buf, attr->value.len); 91 | if (attr->quote) { 92 | jitify_write(lexer, &(attr->quote), 1); 93 | } 94 | } 95 | } 96 | } 97 | if (state->trailing_slash) { 98 | jitify_write(lexer, "/", 1); 99 | } 100 | jitify_write(lexer, ">", 1); 101 | return JITIFY_OK; 102 | } 103 | } 104 | 105 | static jitify_status_t html_transform(jitify_lexer_t *lexer, const void *data, size_t length, size_t starting_offset) 106 | { 107 | const char *buf = data; 108 | jitify_html_state_t *state = lexer->state; 109 | 110 | if (lexer->token_type == jitify_type_html_space) { 111 | if (lexer->remove_space && 112 | (state->nominify_depth == 0)) { 113 | if (state->space_contains_newlines) { 114 | buf = "\n"; 115 | } 116 | else { 117 | buf = " "; 118 | } 119 | length = 1; 120 | } 121 | } 122 | else if (lexer->token_type == jitify_type_html_comment) { 123 | if (lexer->remove_comments && !state->conditional_comment) { 124 | buf = " "; 125 | length = 1; 126 | } 127 | } 128 | else if (lexer->token_type == jitify_type_css_comment) { 129 | if (lexer->remove_comments) { 130 | return JITIFY_OK; 131 | } 132 | } 133 | else if (lexer->token_type == jitify_type_css_optional_whitespace) { 134 | if (lexer->remove_space) { 135 | return JITIFY_OK; 136 | } 137 | } 138 | else if ((lexer->token_type == jitify_type_css_selector) || (lexer->token_type == jitify_type_css_term)) { 139 | if (lexer->remove_space && (state->last_token_type == lexer->token_type)) { 140 | /* The space we just skipped was actually necessary, so add a space back in */ 141 | if (jitify_write(lexer, " ", 1) < 0) { 142 | return JITIFY_ERROR; 143 | } 144 | } 145 | } 146 | else if (lexer->token_type == jitify_type_html_tag) { 147 | return html_tag_transform(lexer, buf, length, starting_offset, "?"); 148 | } 149 | else if (lexer->token_type == jitify_type_html_anchor_open) { 150 | return html_tag_transform(lexer, buf, length, starting_offset, "href"); 151 | } 152 | else if (lexer->token_type == jitify_type_html_img_open) { 153 | return html_tag_transform(lexer, buf, length, starting_offset, "src"); 154 | } 155 | else if (lexer->token_type == jitify_type_html_link_open) { 156 | return html_tag_transform(lexer, buf, length, starting_offset, "href"); 157 | } 158 | else if (lexer->token_type == jitify_type_html_script_open) { 159 | return html_tag_transform(lexer, buf, length, starting_offset, "src"); 160 | } 161 | 162 | state->last_token_type = lexer->token_type; 163 | if (jitify_write(lexer, buf, length) < 0) { 164 | return JITIFY_ERROR; 165 | } 166 | else { 167 | return JITIFY_OK; 168 | } 169 | } 170 | 171 | static void html_cleanup(jitify_lexer_t *lexer) 172 | { 173 | jitify_free(lexer->pool, lexer->state); 174 | } 175 | 176 | jitify_lexer_t *jitify_html_lexer_create(jitify_pool_t *pool, jitify_output_stream_t *out) 177 | { 178 | jitify_lexer_t *lexer = jitify_lexer_create(pool, out); 179 | jitify_html_state_t *state = jitify_calloc(pool, sizeof(*state)); 180 | state->conditional_comment = 0; 181 | state->space_contains_newlines = 0; 182 | lexer->state = state; 183 | lexer->scan = jitify_html_scan; 184 | lexer->transform = html_transform; 185 | lexer->cleanup = html_cleanup; 186 | return lexer; 187 | } 188 | -------------------------------------------------------------------------------- /src/core/jitify_html.h: -------------------------------------------------------------------------------- 1 | #ifndef jitify_html_h 2 | #define jitify_html_h 3 | 4 | #include "jitify_lexer.h" 5 | 6 | #ifdef JITIFY_INTERNAL 7 | 8 | extern jitify_token_type_t jitify_type_html_anchor_open; 9 | extern jitify_token_type_t jitify_type_html_comment; 10 | extern jitify_token_type_t jitify_type_html_img_open; 11 | extern jitify_token_type_t jitify_type_html_link_open; 12 | extern jitify_token_type_t jitify_type_html_script_open; 13 | extern jitify_token_type_t jitify_type_html_space; 14 | extern jitify_token_type_t jitify_type_html_tag; 15 | 16 | typedef struct { 17 | int conditional_comment; 18 | int space_contains_newlines; 19 | size_t tagname_offset; 20 | size_t tagname_len; 21 | int leading_slash; 22 | int trailing_slash; 23 | int nominify_depth; 24 | jitify_token_type_t last_token_type; 25 | } jitify_html_state_t; 26 | 27 | #endif /* JITIFY_INTERNAL */ 28 | 29 | #endif /* !defined(jitify_html_h) */ 30 | -------------------------------------------------------------------------------- /src/core/jitify_html_lexer.rl: -------------------------------------------------------------------------------- 1 | #include 2 | #define JITIFY_INTERNAL 3 | #include "jitify_css.h" 4 | #include "jitify_html.h" 5 | 6 | /* HTML grammar based on http://www.w3.org/TR/WD-html-lex/ 7 | */ 8 | 9 | %%{ 10 | machine jitify_html; 11 | include jitify_common "jitify_lexer_common.rl"; 12 | include css_grammar "jitify_css_lexer_common.rl"; 13 | 14 | tag_close = '/'? @{ state->trailing_slash = 1; } '>'; 15 | 16 | name_char = (alnum | '-' | '_' | '.' | ':'); 17 | name_start_char = (alpha | '_'); 18 | name = name_start_char name_char**; 19 | 20 | action leave_content { 21 | TOKEN_END; 22 | } 23 | 24 | conditional_comment = '[if' %{ state->conditional_comment = 1; }; 25 | 26 | comment = '--' %{ TOKEN_TYPE(jitify_type_html_comment); state->conditional_comment = 0; } 27 | (any | conditional_comment)* :>> '-->' %{ TOKEN_END; }; 28 | 29 | misc_directive = any* :>> '>'; 30 | 31 | directive = ( 32 | '!' (comment | misc_directive) 33 | ); 34 | 35 | attr_name = ( 36 | alpha (alnum | '-' | '_' | ':')** 37 | ) 38 | >{ ATTR_KEY_START; } 39 | %{ ATTR_KEY_END; }; 40 | 41 | unquoted_attr_char = ( any - ( space | '>' | '\\' | '"' | "'" ) ); 42 | unquoted_attr_value = (unquoted_attr_char unquoted_attr_char**) 43 | >{ ATTR_VALUE_START; 44 | ATTR_SET_QUOTE(0); } 45 | %{ ATTR_VALUE_END; }; 46 | 47 | single_quoted_attr_value = "'" @{ ATTR_SET_QUOTE('\''); } 48 | ( /[^']*/ ) >{ ATTR_VALUE_START; } %{ ATTR_VALUE_END; } 49 | "'"; 50 | 51 | double_quoted_attr_value = '"' @{ ATTR_SET_QUOTE('"'); } 52 | ( /[^"]*/ ) 53 | >{ ATTR_VALUE_START; } 54 | %{ ATTR_VALUE_END; } 55 | '"'; 56 | 57 | attr_value = ( 58 | unquoted_attr_value | 59 | single_quoted_attr_value | 60 | double_quoted_attr_value 61 | ); 62 | 63 | unparsed_attr_name = ( 64 | alpha (alnum | '-' | '_' | ':')* 65 | ); 66 | 67 | unparsed_attr_value = ( 68 | ( any - ( space | '>' | '\\' | '"' | "'" ) )+ | 69 | "'" /[^']*/ "'" | 70 | '"' /[^"]*/ '"' 71 | ); 72 | 73 | unparsed_attr = ( 74 | unparsed_attr_name space* ('=' space* unparsed_attr_value)? 75 | ); 76 | 77 | preformatted_close = '' @{ state->nominify_depth--; }; 78 | 79 | preformatted_open= (/pre/i | /textarea/i) @{ state->nominify_depth++; } 80 | (space+ unparsed_attr)* space* tag_close; 81 | 82 | script_close = ''; 83 | 84 | tag_attrs = (space+ %{ ATTR_END;} ( attr_name <: space* ( '=' space* attr_value <: space*)? )*); 85 | 86 | script = ( 87 | /script/i 88 | >{ ATTR_KEY_START; } 89 | %{ TOKEN_TYPE(jitify_type_html_tag); 90 | ATTR_KEY_END; } 91 | tag_attrs? tag_close 92 | %{ TOKEN_END; 93 | TOKEN_START(jitify_token_type_misc); } 94 | (any* - ( any* script_close any* ) ) script_close 95 | ); 96 | 97 | style = ( 98 | /style/i 99 | >{ ATTR_KEY_START; } 100 | %{ TOKEN_TYPE(jitify_type_html_tag); 101 | ATTR_KEY_END; } 102 | tag_attrs? tag_close 103 | %{ TOKEN_END; } 104 | css_document? ( '' ) 105 | >{ TOKEN_TYPE(jitify_token_type_misc); } 106 | %{ TOKEN_END; } 107 | ); 108 | 109 | misc_tag = ( 110 | '/'? 111 | @{ state->leading_slash = 1; } 112 | attr_name 113 | tag_attrs? 114 | tag_close 115 | ) 116 | >{ TOKEN_TYPE(jitify_type_html_tag); }; 117 | 118 | _xml_tag_close = '?>'; 119 | 120 | xml_tag = ( '?' (any* - _xml_tag_close) :>> _xml_tag_close ) 121 | >{ TOKEN_TYPE(jitify_token_type_misc); }; 122 | 123 | element = ( 124 | script 125 | | 126 | xml_tag 127 | | 128 | style 129 | | 130 | misc_tag 131 | | 132 | directive 133 | ); 134 | 135 | html_space = ( 136 | ( space - ( '\r' | '\n' ) ) | 137 | ( '\r' | '\n' ) @{ state->space_contains_newlines = 1; } 138 | )+ 139 | >{ TOKEN_START(jitify_type_html_space); 140 | state->space_contains_newlines = 0; } 141 | %{ TOKEN_END; }; 142 | 143 | content = ( 144 | any - (space | '<' ) 145 | )+ 146 | >{ TOKEN_START(jitify_token_type_misc); } 147 | %{ TOKEN_END; }; 148 | 149 | main := ( 150 | byte_order_mark? 151 | ( 152 | ( '<' 153 | >{ TOKEN_START(jitify_token_type_misc); 154 | state->leading_slash = 0; 155 | state->trailing_slash = 0; } 156 | element 157 | %{ TOKEN_END; 158 | RESET_ATTRS; } 159 | ) 160 | | 161 | html_space 162 | | 163 | content 164 | )** 165 | ) $err(main_err); 166 | 167 | write data; 168 | }%% 169 | 170 | int jitify_html_scan(jitify_lexer_t *lexer, const void *data, size_t length, int is_eof) 171 | { 172 | const char *p = data, *pe = data + length; 173 | const char *eof = is_eof ? pe : NULL; 174 | jitify_html_state_t *state = lexer->state; 175 | if (!lexer->initialized) { 176 | %% write init; 177 | lexer->initialized = 1; 178 | } 179 | %% write exec; 180 | return p - (const char *)data; 181 | } 182 | -------------------------------------------------------------------------------- /src/core/jitify_js.c: -------------------------------------------------------------------------------- 1 | #include 2 | #define JITIFY_INTERNAL 3 | #include "jitify_js.h" 4 | 5 | /* Javascript transformations 6 | * Minification logic based on the algorithms of JSMin: http://www.crockford.com/javascript/jsmin.html 7 | */ 8 | 9 | extern int jitify_js_scan(jitify_lexer_t *lexer, const void *data, size_t length, int is_eof); 10 | 11 | jitify_token_type_t jitify_type_js_whitespace = "JS space"; 12 | jitify_token_type_t jitify_type_js_newline = "JS newline"; 13 | jitify_token_type_t jitify_type_js_comment = "JS block comment"; 14 | jitify_token_type_t jitify_type_js_line_comment = "JS line comment"; 15 | 16 | static int is_ident_char(char c) 17 | { 18 | return isalnum(c) || (c == '_') || (c == '$') || (c == '\\') || (c >= 127) || (c < 0); 19 | } 20 | 21 | static jitify_status_t js_transform(jitify_lexer_t *lexer, const void *data, size_t length, size_t starting_offset) 22 | { 23 | const char *buf = data; 24 | jitify_js_state_t *state = lexer->state; 25 | 26 | if (length == 0) { 27 | return JITIFY_OK; 28 | } 29 | 30 | /* If comment removal is enabled, replace any line comment with 31 | * a newline and any block comment with a space. The minify logic 32 | * that happens later might further reduce these. 33 | */ 34 | if (lexer->remove_comments) { 35 | if (lexer->token_type == jitify_type_js_comment) { 36 | buf = " "; 37 | length = 1; 38 | lexer->token_type = jitify_type_js_whitespace; 39 | } 40 | else if (lexer->token_type == jitify_type_js_line_comment) { 41 | buf = "\n"; 42 | length = 1; 43 | lexer->token_type = jitify_type_js_newline; 44 | } 45 | } 46 | 47 | if (!lexer->remove_space) { 48 | const char *last = buf + length; 49 | while (--last >= buf) { 50 | if ((*last != ' ') && (*last != '\t')) { 51 | state->last_written = *last; 52 | break; 53 | } 54 | } 55 | if (jitify_write(lexer, buf, length) < 0) { 56 | return JITIFY_ERROR; 57 | } 58 | else { 59 | return JITIFY_OK; 60 | } 61 | } 62 | 63 | /* In JavaScript, whether it's safe to remove a space or newline 64 | * may depend on the characters before and after the whitespace. 65 | * To do minification in a streaming lexer, where the characters 66 | * before and after the whitespace might not even be in the same 67 | * buffer, we keep track of the last-written character in 68 | * state->last_written and hold any space or newline in 69 | * state->pending until the next token is seen. 70 | */ 71 | if (lexer->token_type == jitify_type_js_whitespace) { 72 | if (state->pending) { 73 | /* There's already a space or newline pending, so we don't need this space */ 74 | } 75 | else if (is_ident_char(state->last_written)) { 76 | /* We might need to output this space, so hold onto it */ 77 | state->pending = ' '; 78 | } 79 | return JITIFY_OK; 80 | } 81 | else if (lexer->token_type == jitify_type_js_newline) { 82 | if (state->pending == '\n') { 83 | /* There's already a newline pending, so we don't need this one */ 84 | } 85 | else if (state->pending == ' ') { 86 | /* There was a space at the end of the line; discard it */ 87 | state->pending = '\n'; 88 | } 89 | else if (is_ident_char(state->last_written) || 90 | (state->last_written == '}') || (state->last_written == ']') || (state->last_written == ')') || 91 | (state->last_written == '+') || (state->last_written == '-') || (state->last_written == '"') || 92 | (state->last_written == '\'')) { 93 | /* We might need to output this newline, depending on what follows it */ 94 | state->pending = '\n'; 95 | } 96 | return JITIFY_OK; 97 | } 98 | else { 99 | if (state->pending == '\n') { 100 | if (is_ident_char(*buf) || (*buf == '{') || (*buf == '[') || (*buf == '(') || (*buf == '+') || (*buf == '-')) { 101 | if (jitify_write(lexer, &(state->pending), 1) < 0) { 102 | return JITIFY_ERROR; 103 | } 104 | state->last_written = state->pending; 105 | } 106 | state->pending = 0; 107 | } 108 | else if (state->pending == ' ') { 109 | if (is_ident_char(*buf)) { 110 | if (jitify_write(lexer, &(state->pending), 1) < 0) { 111 | return JITIFY_ERROR; 112 | } 113 | } 114 | state->pending = 0; 115 | } 116 | state->last_written = buf[length - 1]; 117 | if (jitify_write(lexer, buf, length) < 0) { 118 | return JITIFY_ERROR; 119 | } 120 | else { 121 | return JITIFY_OK; 122 | } 123 | } 124 | } 125 | 126 | static void js_cleanup(jitify_lexer_t *lexer) 127 | { 128 | jitify_free(lexer->pool, lexer->state); 129 | } 130 | 131 | jitify_lexer_t *jitify_js_lexer_create(jitify_pool_t *pool, jitify_output_stream_t *out) 132 | { 133 | jitify_lexer_t *lexer = jitify_lexer_create(pool, out); 134 | jitify_js_state_t *state = jitify_calloc(pool, sizeof(*state)); 135 | state->last_written = '\n'; 136 | state->pending = 0; 137 | state->html_comment = 0; 138 | lexer->state = state; 139 | lexer->scan = jitify_js_scan; 140 | lexer->transform = js_transform; 141 | lexer->cleanup = js_cleanup; 142 | return lexer; 143 | } 144 | -------------------------------------------------------------------------------- /src/core/jitify_js.h: -------------------------------------------------------------------------------- 1 | #ifndef jitify_js_h 2 | #define jitify_js_h 3 | 4 | #include "jitify_lexer.h" 5 | 6 | #ifdef JITIFY_INTERNAL 7 | 8 | extern jitify_token_type_t jitify_type_js_whitespace; 9 | extern jitify_token_type_t jitify_type_js_newline; 10 | extern jitify_token_type_t jitify_type_js_comment; 11 | extern jitify_token_type_t jitify_type_js_line_comment; 12 | 13 | typedef struct { 14 | char last_written; 15 | char pending; 16 | int html_comment; 17 | int slash_elem_complete; 18 | } jitify_js_state_t; 19 | 20 | #endif /* JITIFY_INTERNAL */ 21 | 22 | #endif /* !defined(jitify_js_h) */ 23 | -------------------------------------------------------------------------------- /src/core/jitify_js_lexer.rl: -------------------------------------------------------------------------------- 1 | #include 2 | #define JITIFY_INTERNAL 3 | #include "jitify_js.h" 4 | 5 | /* JavaScript grammar based on the ECMA-262 5th Edition spec 6 | * http://www.ecma-international.org/publications/standards/Ecma-262.htm 7 | */ 8 | 9 | %%{ 10 | machine jitify_js; 11 | include jitify_common "jitify_lexer_common.rl"; 12 | 13 | js_space = /[ \t]/+ 14 | >{ TOKEN_START(jitify_type_js_whitespace); } %{ TOKEN_END; }; 15 | 16 | _line_end = ( /\r/? /\n/ ); 17 | 18 | js_line_end = _line_end+ 19 | >{ TOKEN_START(jitify_type_js_newline); } %{ TOKEN_END; }; 20 | 21 | html_comment ='-->' %{ state->html_comment = 1; }; 22 | 23 | single_quoted = ( 24 | "'" ( [^'\\] | /\\./)* "'" 25 | ) >{ TOKEN_START(jitify_token_type_misc); } %{ TOKEN_END; }; 26 | 27 | double_quoted = ( 28 | '"' ( [^"\\] | /\\./ )* '"' 29 | ) >{ TOKEN_START(jitify_token_type_misc); } %{ TOKEN_END; }; 30 | 31 | js_misc = ( 32 | any - [ \t\r\n'"/] 33 | )+ >{ TOKEN_START(jitify_token_type_misc); } %{ TOKEN_END; }; 34 | 35 | line_comment = ( 36 | ( ( any | html_comment ) - _line_end )* :>> _line_end @{ state->slash_elem_complete = 1; } 37 | ) >{ TOKEN_TYPE(jitify_type_js_line_comment); state->html_comment = 0; } 38 | $eof{ state->slash_elem_complete = 1; } 39 | ; 40 | 41 | block_comment = ( 42 | ( any )* :>> '*/' @{ state->slash_elem_complete = 1; } 43 | ) >{ TOKEN_TYPE(jitify_type_js_comment); }; 44 | 45 | regex = ( 46 | ( [^\\/] | /\\./ )+ :>> '/' @{ state->slash_elem_complete = 1; } 47 | ); 48 | 49 | slash_element_perhaps_regex := ( 50 | '/' @{ TOKEN_START(jitify_token_type_misc); state->slash_elem_complete = 0; } 51 | ( 52 | ( '*' block_comment ) | 53 | ( '/' line_comment ) | 54 | ( ( any - ( '*' | '/' ) ) @{ fhold; } regex ) 55 | ) 56 | ) 57 | %{ // TODO: check if this is even reachable. It could only happen at EOF 58 | TOKEN_END; 59 | fhold; 60 | fgoto program; 61 | } 62 | $err{ 63 | if (state->slash_elem_complete) { 64 | /* If we got here by reaching the character after the end of 65 | * a syntactically valid slash_element, it's not actually an 66 | * error. We've simply reached the end of what this sub-lexer 67 | * knows how to parse. 68 | */ 69 | TOKEN_END; 70 | fhold; 71 | fgoto program; 72 | } 73 | else { 74 | // TODO: invoke main_err 75 | } 76 | } 77 | ; 78 | 79 | slash_element_not_regex := ( 80 | '/' @{ TOKEN_START(jitify_token_type_misc); state->slash_elem_complete = 1; } 81 | ( 82 | ( '*' >{ state->slash_elem_complete = 0; } block_comment ) | 83 | ( '/' >{ state->slash_elem_complete = 0; } line_comment ) 84 | )? 85 | ) 86 | %{ // TODO: check if this is even reachable. It could only happen at EOF 87 | TOKEN_END; 88 | fhold; 89 | fgoto program; 90 | } 91 | $err{ 92 | if (state->slash_elem_complete) { 93 | /* Non-error case--see comments under slash_element_perhaps_regex for more info */ 94 | TOKEN_END; 95 | fhold; 96 | fgoto program; 97 | } 98 | else { 99 | // TODO: invoke main_err 100 | } 101 | } 102 | ; 103 | 104 | # ECMAscript actually uses two different lexical grammars for different 105 | # syntactic contexts. One of those lexical grammars allows the division 106 | # operator '/', while the other allows the regular expression format '/.../'. 107 | # The ECMAscript spec recommends that the parser simply choose the right 108 | # lexical grammar for every token based on the syntactic context. Because 109 | # the Jitify Core doesn't use a separate parser on top of the lexer, the 110 | # lexer itself must somehow determine what grammar to use when it finds 111 | # a token starting with a slash. Fortunately, the JS lexer already keeps 112 | # track of the last-written non-space character as part of the minification 113 | # algorithm, and that character provides enough context to choose the 114 | # right sub-lexer.` 115 | slash_element = '/' 116 | >{ 117 | int could_be_regex = 0; 118 | switch (state->last_written) { 119 | case '(': 120 | case ',': 121 | case '=': 122 | case '[': 123 | case '!': 124 | case '&': 125 | case '|': 126 | case '?': 127 | case '{': 128 | case '}': 129 | case ';': 130 | case ':': 131 | case '\r': 132 | case '\n': 133 | could_be_regex = 1; 134 | } 135 | fhold; 136 | if (could_be_regex) { 137 | fgoto slash_element_perhaps_regex; 138 | } 139 | else { 140 | fgoto slash_element_not_regex; 141 | } 142 | }; 143 | 144 | program = ( 145 | js_space | 146 | js_line_end | 147 | single_quoted | 148 | double_quoted | 149 | slash_element | 150 | js_misc 151 | )**; 152 | 153 | main := ( 154 | byte_order_mark? 155 | program 156 | ) $err(main_err); 157 | 158 | write data; 159 | }%% 160 | 161 | int jitify_js_scan(jitify_lexer_t *lexer, const void *data, size_t length, int is_eof) 162 | { 163 | const char *p = data, *pe = data + length; 164 | const char *eof = is_eof ? pe : NULL; 165 | jitify_js_state_t *state = lexer->state; 166 | if (!lexer->initialized) { 167 | %% write init; 168 | lexer->initialized = 1; 169 | } 170 | %% write exec; 171 | return p - (const char *)data; 172 | } 173 | -------------------------------------------------------------------------------- /src/core/jitify_lexer.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #define JITIFY_INTERNAL 4 | #include "jitify_lexer.h" 5 | 6 | jitify_token_type_t jitify_token_type_misc = "Miscellaneous"; 7 | 8 | static int failsafe_send(jitify_lexer_t *lexer, const void *data, size_t len, size_t offset) 9 | { 10 | lexer->token_type = jitify_token_type_misc; 11 | switch (lexer->transform(lexer, data, len, offset)) { 12 | case JITIFY_OK: 13 | return (int)len; 14 | case JITIFY_AGAIN: 15 | return 0; 16 | default: 17 | return -1; 18 | } 19 | } 20 | 21 | void jitify_lexer_set_minify_rules(jitify_lexer_t *lexer, int remove_space, int remove_comments) 22 | { 23 | lexer->remove_space = remove_space; 24 | lexer->remove_comments = remove_comments; 25 | } 26 | 27 | void jitify_lexer_set_max_setaside(jitify_lexer_t *lexer, size_t max) 28 | { 29 | lexer->setaside_max = max; 30 | } 31 | 32 | static void setaside_buffer(jitify_lexer_t *lexer, size_t remaining) 33 | { 34 | if (!lexer->setaside) { 35 | lexer->setaside = jitify_malloc(lexer->pool, lexer->setaside_max); 36 | } 37 | if (lexer->setaside_len == 0) { 38 | lexer->setaside_offset = lexer->starting_offset + (lexer->token_start - lexer->buf); 39 | } 40 | memcpy(lexer->setaside + lexer->setaside_len, lexer->token_start, remaining); 41 | lexer->setaside_len += remaining; 42 | } 43 | 44 | int jitify_lexer_scan(jitify_lexer_t *lexer, const void *data, size_t len, int is_eof) 45 | { 46 | int rc; 47 | struct timeval start_time, end_time; 48 | long elapsed_usec; 49 | int bytes_scanned; 50 | 51 | lexer->setaside_overflow = 0; 52 | lexer->buf = data; 53 | lexer->err = NULL; 54 | gettimeofday(&start_time, NULL); 55 | if (lexer->failsafe_mode) { 56 | rc = failsafe_send(lexer, data, len, lexer->starting_offset); 57 | bytes_scanned = len; 58 | } 59 | else { 60 | lexer->token_start = data; 61 | bytes_scanned = lexer->scan(lexer, data, len, is_eof); 62 | if (bytes_scanned < 0) { 63 | rc = bytes_scanned; 64 | } 65 | else if (bytes_scanned == (int)len) { 66 | if (lexer->token_start) { 67 | size_t remaining = (const char *)data + len - lexer->token_start; 68 | if (remaining) { 69 | /* Partially matched token at end of buffer */ 70 | if (remaining + lexer->setaside_len <= lexer->setaside_max) { 71 | /* We have enough space to set aside this partial token until we get more data */ 72 | setaside_buffer(lexer, remaining); 73 | } 74 | else { 75 | /* Not enough space to set aside this token, so send it unmodified */ 76 | lexer->token_type = jitify_token_type_misc; 77 | if (lexer->setaside_len) { 78 | lexer->transform(lexer, lexer->setaside, lexer->setaside_len, lexer->setaside_offset); 79 | lexer->setaside_len = 0; 80 | } 81 | lexer->transform(lexer, lexer->token_start, remaining, CURRENT_OFFSET(lexer->token_start)); 82 | /* TODO check transform return code */ 83 | lexer->setaside_overflow = 1; 84 | } 85 | } 86 | } 87 | rc = bytes_scanned; 88 | } 89 | else { 90 | if (lexer->failsafe_mode) { 91 | rc = failsafe_send(lexer, (char *)data + bytes_scanned, len - bytes_scanned, lexer->starting_offset + bytes_scanned); 92 | bytes_scanned = len; 93 | } 94 | else { 95 | rc = bytes_scanned; 96 | } 97 | } 98 | } 99 | gettimeofday(&end_time, NULL); 100 | elapsed_usec = (end_time.tv_sec * 1000000 + end_time.tv_usec) - (start_time.tv_sec * 1000000 + start_time.tv_usec); 101 | lexer->duration += elapsed_usec; 102 | lexer->bytes_in += bytes_scanned; 103 | if (bytes_scanned > 0) { 104 | lexer->starting_offset += bytes_scanned; 105 | } 106 | return rc; 107 | } 108 | 109 | void jitify_lexer_destroy(jitify_lexer_t *lexer) 110 | { 111 | if (lexer) { 112 | if (lexer->cleanup) { 113 | lexer->cleanup(lexer); 114 | } 115 | jitify_array_destroy(lexer->attrs); 116 | jitify_free(lexer->pool, lexer->setaside); 117 | jitify_free(lexer->pool, lexer); 118 | } 119 | } 120 | 121 | #define DEFAULT_MAX_SETASIDE 1024 122 | 123 | jitify_lexer_t *jitify_lexer_create(jitify_pool_t *pool, jitify_output_stream_t *out) 124 | { 125 | jitify_lexer_t *lexer = jitify_calloc(pool, sizeof(*lexer)); 126 | lexer->pool = pool; 127 | lexer->out = out; 128 | lexer->initialized = 0; 129 | lexer->failsafe_mode = 0; 130 | lexer->remove_space = 0; 131 | lexer->remove_comments = 0; 132 | lexer->setaside_max = DEFAULT_MAX_SETASIDE; 133 | lexer->token_type = jitify_token_type_misc; 134 | lexer->attrs = jitify_array_create(pool, sizeof(jitify_attr_t)); 135 | return lexer; 136 | } 137 | 138 | typedef struct { 139 | const char *content_type; 140 | jitify_lexer_t *(*create_lexer)(jitify_pool_t *pool, jitify_output_stream_t *out); 141 | } content_type_to_lexer_t; 142 | 143 | static content_type_to_lexer_t content_type_map[] = { 144 | { "text/css", jitify_css_lexer_create }, 145 | { "text/html", jitify_html_lexer_create }, 146 | { "text/javascript", jitify_js_lexer_create }, 147 | { "application/javascript", jitify_js_lexer_create }, 148 | { "application/x-javascript", jitify_js_lexer_create }, 149 | { NULL, NULL } 150 | }; 151 | 152 | jitify_lexer_t *jitify_lexer_for_content_type(const char *content_type, jitify_pool_t *pool, jitify_output_stream_t *out) 153 | { 154 | const char *delimiter; 155 | content_type_to_lexer_t *mapping; 156 | if (!content_type) { 157 | return NULL; 158 | } 159 | delimiter = strchr(content_type, ';'); 160 | if (delimiter) { 161 | size_t length = delimiter - content_type; 162 | char *new_content_type = jitify_malloc(pool, length + 1); 163 | memcpy(new_content_type, content_type, length); 164 | new_content_type[length] = 0; 165 | content_type = new_content_type; 166 | } 167 | // TODO: replace with a sub-O(n) lookup if the number of map entries ever exceeds single digits 168 | for (mapping = content_type_map; mapping->content_type; mapping++) { 169 | if (!strcasecmp(content_type, mapping->content_type)) { 170 | return mapping->create_lexer(pool, out); 171 | } 172 | } 173 | return NULL; 174 | } 175 | 176 | size_t jitify_lexer_get_bytes_in(jitify_lexer_t *lexer) 177 | { 178 | return lexer->bytes_in; 179 | } 180 | 181 | size_t jitify_lexer_get_bytes_out(jitify_lexer_t *lexer) 182 | { 183 | return lexer->bytes_out; 184 | } 185 | 186 | size_t jitify_lexer_get_processing_time(jitify_lexer_t *lexer) 187 | { 188 | return lexer->duration; 189 | } 190 | 191 | const char *jitify_lexer_get_err(jitify_lexer_t *lexer) 192 | { 193 | return lexer->err; 194 | } 195 | 196 | void jitify_transform_with_setaside(jitify_lexer_t *lexer, const char *p) 197 | { 198 | size_t length = p - lexer->token_start; 199 | if (length + lexer->setaside_len <= lexer->setaside_max) { 200 | memcpy(lexer->setaside + lexer->setaside_len, lexer->token_start, length); 201 | lexer->transform(lexer, lexer->setaside, lexer->setaside_len + length, lexer->setaside_offset); 202 | // TODO: check transform return code 203 | } 204 | else { 205 | lexer->token_type = jitify_token_type_misc; 206 | lexer->transform(lexer, lexer->setaside, lexer->setaside_len, lexer->setaside_offset); 207 | lexer->transform(lexer, lexer->token_start, length, lexer->starting_offset); // TODO: double-check the offset 208 | } 209 | lexer->setaside_len = 0; 210 | } 211 | 212 | void jitify_lexer_resolve_attrs(jitify_lexer_t *lexer, const char *buf, size_t starting_offset) 213 | { 214 | size_t i, num_attrs; 215 | if (!lexer || lexer->attrs_resolved) { 216 | return; 217 | } 218 | if (!lexer->attrs) { 219 | lexer->attrs_resolved = 1; 220 | return; 221 | } 222 | num_attrs = jitify_array_length(lexer->attrs); 223 | for (i = 0; i < num_attrs; i++) { 224 | jitify_attr_t *attr = jitify_array_get(lexer->attrs, i); 225 | if (attr->key.len) { 226 | attr->key.data.buf = buf + attr->key.data.offset - starting_offset; 227 | } 228 | if (attr->value.len) { 229 | attr->value.data.buf = buf + attr->value.data.offset - starting_offset; 230 | } 231 | } 232 | lexer->attrs_resolved = 1; 233 | } 234 | 235 | void jitify_err_checkpoint(jitify_lexer_t *lexer) 236 | { 237 | /* Do nothing -- this function exists solely as a 238 | convenient place to put a debugger breakpoint 239 | */ 240 | } 241 | 242 | #define INITIAL_ARRAY_SIZE 1 243 | 244 | void jitify_lexer_add_cdnify_rule(jitify_lexer_t *lexer, const char *prefix, const char *replacement) 245 | { 246 | jitify_cdnify_rule_t *rule; 247 | if (lexer->cdnify_rules_size <= lexer->cdnify_rules_len) { 248 | jitify_cdnify_rule_t *new_rules; 249 | if (lexer->cdnify_rules_size) { 250 | lexer->cdnify_rules_size *= 2; 251 | } 252 | else { 253 | lexer->cdnify_rules_size = INITIAL_ARRAY_SIZE; 254 | } 255 | new_rules = jitify_malloc(lexer->pool, lexer->cdnify_rules_size * sizeof(jitify_cdnify_rule_t)); 256 | if (lexer->cdnify_rules_len) { 257 | memcpy(new_rules, lexer->cdnify_rules, lexer->cdnify_rules_len * sizeof(jitify_cdnify_rule_t)); 258 | } 259 | if (lexer->cdnify_rules) { 260 | jitify_free(lexer->pool, lexer->cdnify_rules); 261 | } 262 | lexer->cdnify_rules = new_rules; 263 | } 264 | rule = &(lexer->cdnify_rules[lexer->cdnify_rules_len++]); 265 | rule->prefix.data = prefix; 266 | rule->prefix.len = strlen(prefix); 267 | if (replacement) { 268 | rule->replacement = jitify_malloc(lexer->pool, sizeof(jitify_str_t)); 269 | rule->replacement->data = replacement; 270 | rule->replacement->len = strlen(replacement); 271 | } 272 | else { 273 | rule->replacement = NULL; 274 | } 275 | } 276 | -------------------------------------------------------------------------------- /src/core/jitify_lexer.h: -------------------------------------------------------------------------------- 1 | #ifndef jitify_lexer_h 2 | #define jitify_lexer_h 3 | 4 | #include 5 | 6 | #include "jitify.h" 7 | 8 | #ifdef JITIFY_INTERNAL 9 | 10 | typedef const char *jitify_token_type_t; 11 | 12 | extern jitify_token_type_t jitify_token_type_misc; 13 | 14 | typedef struct { 15 | const char *data; 16 | size_t len; 17 | } jitify_str_t; 18 | 19 | typedef struct { 20 | union { 21 | size_t offset; /* Relative to start of entire document */ 22 | const char *buf; 23 | } data; 24 | size_t len; 25 | } jitify_str_ref_t; 26 | 27 | typedef struct { 28 | jitify_str_ref_t key; 29 | jitify_str_ref_t value; 30 | int key_setaside; 31 | int value_setaside; 32 | char quote; 33 | } jitify_attr_t; 34 | 35 | typedef struct { 36 | jitify_str_t prefix; 37 | jitify_str_t *replacement; /* NULL means "don't CDNify a link that matches this prefix" */ 38 | } jitify_cdnify_rule_t; 39 | 40 | struct jitify_lexer_s { 41 | void *state; 42 | jitify_pool_t *pool; 43 | jitify_output_stream_t *out; 44 | 45 | int initialized; 46 | int failsafe_mode; 47 | 48 | /* Minification rules */ 49 | int remove_space; 50 | int remove_comments; 51 | 52 | /* Link rewriting rules */ 53 | jitify_cdnify_rule_t *cdnify_rules; 54 | size_t cdnify_rules_len; /* Number of items in cdnify_rules */ 55 | size_t cdnify_rules_size; /* Max @ of items that cdnify_rules is sized to hold */ 56 | 57 | jitify_status_t (*transform)(jitify_lexer_t *lexer, const void *data, size_t length, size_t offset); 58 | int (*scan)(jitify_lexer_t *lexer, const void *data, size_t length, int is_eof); 59 | void (*cleanup)(jitify_lexer_t *lexer); 60 | 61 | char *setaside; 62 | size_t setaside_max; 63 | size_t setaside_len; 64 | int setaside_overflow; /* True iff a cross-buffer token exceeded setaside_max */ 65 | size_t setaside_offset; /* Offset from start of document of 1st byte of setaside */ 66 | 67 | jitify_token_type_t token_type; 68 | const char *token_start; 69 | 70 | long duration; /* Cumulative time spent in this lexer, in usec */ 71 | size_t bytes_in; /* Cumulative input bytes processed by this lexer */ 72 | size_t bytes_out; /* Cumulative bytes of output produced by this lexer */ 73 | 74 | const char *err; /* Location of error within input buf, NULL if no error */ 75 | 76 | const char *buf; /* Start of current buffer */ 77 | size_t starting_offset; /* Offset from start of document of 1st byte of current buffer */ 78 | 79 | size_t subtoken_offset; /* Offset from start of document of current subtoken */ 80 | 81 | jitify_array_t *attrs; /* Array of jitify_attr_t* */ 82 | jitify_attr_t *current_attr; /* Points into attrs or is NULL */ 83 | int attrs_resolved; /* whether the keys and values in attrs have been converted from offsets to char* */ 84 | 85 | /* The following fields support Ragel-generated parsers */ 86 | int cs; 87 | int act; 88 | }; 89 | 90 | extern jitify_lexer_t *jitify_lexer_create(jitify_pool_t *pool, jitify_output_stream_t *out); 91 | 92 | extern void jitify_transform_with_setaside(jitify_lexer_t *lexer, const char *p); 93 | 94 | extern void jitify_lexer_resolve_attrs(jitify_lexer_t *lexer, const char *buf, size_t starting_offset); 95 | 96 | #define CURRENT_OFFSET(ptr) \ 97 | (lexer->starting_offset + (ptr - lexer->buf)) 98 | 99 | #define TOKEN_TYPE(t) \ 100 | lexer->token_type = (t) 101 | 102 | #define TOKEN_START(t) \ 103 | lexer->token_start = p; \ 104 | TOKEN_TYPE(t) 105 | 106 | #define TOKEN_END \ 107 | if (lexer->setaside_len) { \ 108 | jitify_transform_with_setaside(lexer, p); \ 109 | } \ 110 | else { \ 111 | lexer->transform(lexer, lexer->token_start, \ 112 | p - lexer->token_start, \ 113 | CURRENT_OFFSET(lexer->token_start)); \ 114 | } \ 115 | TOKEN_START(jitify_token_type_misc); \ 116 | lexer->current_attr = NULL 117 | 118 | #define ADD_ATTR \ 119 | if (!lexer->current_attr) { \ 120 | lexer->current_attr = \ 121 | jitify_array_push(lexer->attrs); \ 122 | } 123 | 124 | #define ATTR_SET_QUOTE(q) \ 125 | lexer->current_attr->quote = q 126 | 127 | #define ATTR_KEY_START \ 128 | ADD_ATTR; \ 129 | lexer->subtoken_offset = CURRENT_OFFSET(p) 130 | 131 | #define ATTR_KEY_END \ 132 | lexer->current_attr->key.data.offset = \ 133 | lexer->subtoken_offset; \ 134 | lexer->current_attr->key.len = \ 135 | CURRENT_OFFSET(p) - lexer->subtoken_offset 136 | 137 | #define ATTR_VALUE_START \ 138 | ADD_ATTR; \ 139 | lexer->subtoken_offset = CURRENT_OFFSET(p) 140 | 141 | #define ATTR_END \ 142 | lexer->current_attr = NULL 143 | 144 | #define ATTR_VALUE_END \ 145 | lexer->current_attr->value.data.offset = \ 146 | lexer->subtoken_offset; \ 147 | lexer->current_attr->value.len = \ 148 | CURRENT_OFFSET(p) - lexer->subtoken_offset; \ 149 | ATTR_END 150 | 151 | #define RESET_ATTRS \ 152 | jitify_array_clear(lexer->attrs); \ 153 | lexer->attrs_resolved = 0 154 | 155 | #endif /* JITIFY_INTERNAL */ 156 | 157 | #endif /* !defined(jitify_lexer_h) */ 158 | -------------------------------------------------------------------------------- /src/core/jitify_lexer_common.rl: -------------------------------------------------------------------------------- 1 | %%{ 2 | machine jitify_common; 3 | 4 | access lexer->; 5 | 6 | action main_err { 7 | TOKEN_TYPE(jitify_token_type_misc); 8 | TOKEN_END; 9 | lexer->err = p; 10 | lexer->failsafe_mode = 1; 11 | jitify_err_checkpoint(lexer); 12 | p--; 13 | fbreak; 14 | } 15 | 16 | byte_order_mark = ( 17 | 0xEF 0xBB 0xBF 18 | ) >{ TOKEN_START(jitify_token_type_misc); } %{ TOKEN_END; }; 19 | 20 | }%% 21 | -------------------------------------------------------------------------------- /src/core/jitify_pool.c: -------------------------------------------------------------------------------- 1 | #define JITIFY_INTERNAL 2 | #include "jitify.h" 3 | 4 | void *jitify_malloc(jitify_pool_t *pool, size_t length) 5 | { 6 | return pool->malloc(pool, length); 7 | } 8 | 9 | void *jitify_calloc(jitify_pool_t *pool, size_t length) 10 | { 11 | return pool->calloc(pool, length); 12 | } 13 | 14 | void jitify_free(jitify_pool_t *pool, void *object) 15 | { 16 | pool->free(pool, object); 17 | } 18 | 19 | void jitify_pool_destroy(jitify_pool_t *pool) 20 | { 21 | if (pool && pool->cleanup) { 22 | pool->cleanup(pool); 23 | } 24 | } 25 | 26 | static void *malloc_wrapper(jitify_pool_t *pool, size_t length) 27 | { 28 | return malloc(length); 29 | } 30 | 31 | static void *calloc_wrapper(jitify_pool_t *pool, size_t length) 32 | { 33 | return calloc(1, length); 34 | } 35 | 36 | static void free_wrapper(jitify_pool_t *pool, void *object) 37 | { 38 | free(object); 39 | } 40 | 41 | static void malloc_pool_cleanup(jitify_pool_t *pool) 42 | { 43 | free(pool); 44 | } 45 | 46 | jitify_pool_t *jitify_malloc_pool_create() 47 | { 48 | jitify_pool_t *p = malloc(sizeof(*p)); 49 | if (p) { 50 | p->state = NULL; 51 | p->malloc = malloc_wrapper; 52 | p->calloc = calloc_wrapper; 53 | p->free = free_wrapper; 54 | p->cleanup = malloc_pool_cleanup; 55 | } 56 | return p; 57 | } 58 | -------------------------------------------------------------------------------- /src/core/jitify_stream.c: -------------------------------------------------------------------------------- 1 | #define JITIFY_INTERNAL 2 | #include "jitify.h" 3 | #include "jitify_lexer.h" 4 | 5 | 6 | int jitify_write(jitify_lexer_t *lexer, const void *data, size_t length) 7 | { 8 | int rv; 9 | rv = lexer->out->write(lexer->out, data, length); 10 | lexer->bytes_out += length; 11 | return rv; 12 | } 13 | 14 | void jitify_output_stream_destroy(jitify_output_stream_t *stream) 15 | { 16 | if (stream) { 17 | if (stream->cleanup) { 18 | stream->cleanup(stream); 19 | } 20 | jitify_free(stream->pool, stream); 21 | } 22 | } 23 | 24 | static int file_write(jitify_output_stream_t *stream, const void *data, size_t length) 25 | { 26 | #if 1 27 | size_t bytes_written = fwrite(data, 1, length, (FILE *)stream->state); 28 | if (bytes_written < length) { 29 | return -1; 30 | } 31 | else { 32 | return (int)bytes_written; 33 | } 34 | #else 35 | return (int)length; 36 | #endif 37 | } 38 | 39 | jitify_output_stream_t *jitify_stdio_output_stream_create(jitify_pool_t *pool, FILE *out) 40 | { 41 | jitify_output_stream_t *stream = jitify_calloc(pool, sizeof(*stream)); 42 | stream->state = out; 43 | stream->pool = pool; 44 | stream->write = file_write; 45 | return stream; 46 | } 47 | -------------------------------------------------------------------------------- /src/nginx/config: -------------------------------------------------------------------------------- 1 | # This file, run at build time by the nginx configure script, 2 | # tells the nginx build systems what source files comprise 3 | # mod_jitify. 4 | 5 | ngx_addon_name=mod_jitify_module 6 | 7 | #HTTP_AUX_FILTER_MODULES="$HTTP_AUX_FILTER_MODULES jitify_module" 8 | HTTP_FILTER_MODULES=`echo $HTTP_FILTER_MODULES | sed -e 's/ngx_http_postpone_filter_module/jitify_module ngx_http_postpone_filter_module/'` 9 | 10 | for MODULE_SOURCE_FILE in `find $ngx_addon_dir $ngx_addon_dir/../core -type f -name "*.c"` ;do 11 | NGX_ADDON_SRCS="$NGX_ADDON_SRCS $MODULE_SOURCE_FILE" 12 | done 13 | 14 | echo "adding mod_jitify, core modules are:" 15 | echo "$HTTP_FILTER_MODULES" 16 | -------------------------------------------------------------------------------- /src/nginx/config.make: -------------------------------------------------------------------------------- 1 | cat << END >> $NGX_MAKEFILE 2 | MOD_JITIFY_SRC_DIR=$ngx_addon_dir 3 | END 4 | 5 | cat << 'END' >> $NGX_MAKEFILE 6 | CFLAGS += -Werror -I$(MOD_JITIFY_SRC_DIR) -I$(MOD_JITIFY_SRC_DIR)/../core 7 | END 8 | -------------------------------------------------------------------------------- /src/nginx/jitify_nginx_glue.c: -------------------------------------------------------------------------------- 1 | #define JITIFY_INTERNAL 2 | #include "jitify_nginx_glue.h" 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | extern ngx_module_t jitify_module; 9 | 10 | static void *nginx_malloc_wrapper(jitify_pool_t *pool, size_t len) 11 | { 12 | return ngx_palloc(pool->state, len); 13 | } 14 | 15 | static void *nginx_calloc_wrapper(jitify_pool_t *pool, size_t len) 16 | { 17 | return ngx_pcalloc(pool->state, len); 18 | } 19 | 20 | static void nginx_free_wrapper(jitify_pool_t *pool, void *block) 21 | { 22 | } 23 | 24 | jitify_pool_t *jitify_nginx_pool_create(ngx_pool_t *pool) 25 | { 26 | jitify_pool_t *jpool = ngx_pcalloc(pool, sizeof(*jpool)); 27 | jpool->state = pool; 28 | jpool->malloc = nginx_malloc_wrapper; 29 | jpool->calloc = nginx_calloc_wrapper; 30 | jpool->free = nginx_free_wrapper; 31 | return jpool; 32 | } 33 | 34 | static int nginx_buf_write(jitify_output_stream_t *stream, const void *data, size_t len) 35 | { 36 | jitify_nginx_chain_t *chain = stream->state; 37 | const char *cdata = data; 38 | size_t bytes_remaining; 39 | if (!data) { 40 | return 0; 41 | } 42 | bytes_remaining = len; 43 | while (bytes_remaining) { 44 | ngx_chain_t *link; 45 | size_t write_size; 46 | link = chain->last; 47 | if (!link || !link->buf->temporary || (link->buf->last == link->buf->end)) { 48 | ngx_buf_t *buf = ngx_create_temp_buf(chain->pool, ngx_pagesize); 49 | link = ngx_alloc_chain_link(chain->pool); 50 | link->next = NULL; 51 | link->buf = buf; 52 | if (chain->last) { 53 | chain->last->next = link; 54 | } 55 | else { 56 | chain->first = link; 57 | } 58 | chain->last = link; 59 | } 60 | write_size = link->buf->end - link->buf->last; 61 | if (write_size > bytes_remaining) { 62 | write_size = bytes_remaining; 63 | } 64 | memcpy(link->buf->last, cdata, write_size); 65 | cdata += write_size; 66 | link->buf->last += write_size; 67 | bytes_remaining -= write_size; 68 | } 69 | return len; 70 | } 71 | 72 | jitify_output_stream_t *jitify_nginx_output_stream_create(jitify_pool_t *pool) 73 | { 74 | jitify_output_stream_t *stream = jitify_calloc(pool, sizeof(*stream)); 75 | stream->pool = pool; 76 | stream->write = nginx_buf_write; 77 | return stream; 78 | } 79 | 80 | char *jitify_nginx_strdup(jitify_pool_t *pool, ngx_str_t *str) 81 | { 82 | size_t len; 83 | char *buf; 84 | if (!str || !str->data) { 85 | return NULL; 86 | } 87 | len = str->len; 88 | buf = jitify_malloc(pool, len + 1); 89 | memcpy(buf, str->data, len); 90 | buf[len] = 0; 91 | return buf; 92 | } 93 | 94 | void jitify_nginx_add_eof(jitify_nginx_chain_t *chain) 95 | { 96 | ngx_chain_t *link = ngx_pcalloc(chain->pool, sizeof(*chain)); 97 | ngx_buf_t *buf = ngx_calloc_buf(chain->pool); 98 | buf->last_buf = 1; 99 | link->buf = buf; 100 | if (chain->last) { 101 | chain->last->next = link; 102 | } 103 | else { 104 | chain->first = link; 105 | } 106 | chain->last = link; 107 | } 108 | -------------------------------------------------------------------------------- /src/nginx/jitify_nginx_glue.h: -------------------------------------------------------------------------------- 1 | #ifndef jitify_nginx_glue_h 2 | #define jitify_nginx_glue_h 3 | 4 | /* It's important that these three includes, in this order, are 5 | * the first things included within all compilation units. Otherwise, 6 | * on some platforms (e.g., Fedora 8 on 32-bit x86), off_t gets defined 7 | * as a different-sized type in mod_jitify than in the nginx core. 8 | * As key data structures like ngx_http_request_t contain off_t fields, 9 | * the sizes and field offsets of these structures will be different 10 | * between the nginx core and mod_jitify, resulting in inscrutable 11 | * segfaults. 12 | */ 13 | #include 14 | #include 15 | #include 16 | 17 | #include "jitify.h" 18 | 19 | extern jitify_pool_t *jitify_nginx_pool_create(ngx_pool_t *pool); 20 | 21 | extern jitify_output_stream_t *jitify_nginx_output_stream_create(jitify_pool_t *pool); 22 | 23 | extern jitify_output_stream_t *jitify_nginx_output_stream_create(jitify_pool_t *pool); 24 | 25 | /* Utility function to allocate, from a pool, a null-terminated copy of one of nginx's pointer/length strings */ 26 | extern char *jitify_nginx_strdup(jitify_pool_t *pool, ngx_str_t *str); 27 | 28 | /* Wrapper for an expandable list of buffers */ 29 | typedef struct { 30 | ngx_chain_t *first; 31 | ngx_chain_t *last; 32 | ngx_pool_t *pool; 33 | } jitify_nginx_chain_t; 34 | 35 | extern void jitify_nginx_add_eof(jitify_nginx_chain_t *chain); 36 | 37 | #endif /* !defined(jitify_nginx_glue_h) */ 38 | -------------------------------------------------------------------------------- /src/nginx/mod_jitify.c: -------------------------------------------------------------------------------- 1 | #define JITIFY_INTERNAL 2 | #include "jitify_nginx_glue.h" 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | static ngx_http_output_header_filter_pt jitify_next_header_filter; 9 | static ngx_http_output_body_filter_pt jitify_next_body_filter; 10 | 11 | ngx_module_t jitify_module; 12 | 13 | typedef struct { 14 | ngx_flag_t minify; 15 | } jitify_conf_t; 16 | 17 | typedef struct { 18 | jitify_pool_t *pool; 19 | jitify_lexer_t *lexer; 20 | jitify_output_stream_t *out; 21 | } jitify_filter_ctx_t; 22 | 23 | static ngx_int_t jitify_header_filter(ngx_http_request_t *r) 24 | { 25 | ngx_log_t *log = r->connection->log; 26 | jitify_conf_t *jconf; 27 | if (r != r->main) { 28 | return jitify_next_header_filter(r); 29 | } 30 | jconf = ngx_http_get_module_loc_conf(r, jitify_module); 31 | if (!jconf) { 32 | ngx_log_error(NGX_LOG_WARN, log, 0, "internal error: mod_jitify configuration missing"); 33 | return jitify_next_header_filter(r); 34 | } 35 | if (jconf->minify) { 36 | jitify_filter_ctx_t *jctx = ngx_pcalloc(r->pool, sizeof(*jctx)); 37 | jctx->pool = jitify_nginx_pool_create(r->pool); 38 | if (r->headers_out.content_type.data) { 39 | jitify_output_stream_t *out = jitify_nginx_output_stream_create(jctx->pool); 40 | jctx->lexer = jitify_lexer_for_content_type(jitify_nginx_strdup(jctx->pool, &(r->headers_out.content_type)), 41 | jctx->pool, out); 42 | jctx->out = out; 43 | } 44 | 45 | if (jctx->lexer) { 46 | /* Clear the response headers that might be invalidated 47 | when the response body is modified */ 48 | ngx_log_error(NGX_LOG_DEBUG, log, 0, "enabling content scanning for uri=%V content-type=%V", 49 | &(r->uri), &(r->headers_out.content_type)); 50 | ngx_http_clear_content_length(r); 51 | ngx_http_clear_last_modified(r); 52 | ngx_http_clear_accept_ranges(r); 53 | 54 | jitify_lexer_set_minify_rules(jctx->lexer, jconf->minify, jconf->minify); 55 | ngx_http_set_ctx(r, jctx, jitify_module); 56 | 57 | r->main_filter_need_in_memory = 1; 58 | } 59 | else { 60 | ngx_log_error(NGX_LOG_DEBUG, log, 0, "no lexer for uri=%V content-type=%V", 61 | &(r->uri), &(r->headers_out.content_type)); 62 | } 63 | } 64 | return jitify_next_header_filter(r); 65 | } 66 | 67 | #define DEFAULT_ERR_LEN 80 68 | 69 | static ngx_int_t jitify_body_filter(ngx_http_request_t *r, ngx_chain_t *in) 70 | { 71 | ngx_log_t *log = r->connection->log; 72 | jitify_filter_ctx_t *jctx; 73 | jitify_nginx_chain_t out; 74 | int send_flush, send_eof; 75 | jctx = ngx_http_get_module_ctx(r->main, jitify_module); 76 | if (!jctx || !jctx->lexer) { 77 | ngx_log_error(NGX_LOG_DEBUG, log, 0, "jitify filter skipping request for uri=%V", 78 | &(r->uri)); 79 | return jitify_next_body_filter(r, in); 80 | } 81 | out.first = out.last = NULL; 82 | out.pool = r->pool; 83 | jctx->out->state = &out; 84 | send_flush = send_eof = 0; 85 | 86 | while (in) { 87 | ngx_buf_t *buf = in->buf; 88 | if (buf->last_buf) { 89 | send_eof = 1; 90 | } 91 | if (buf->last_buf || (buf->last > buf->pos)) { 92 | const char *err; 93 | jitify_lexer_scan(jctx->lexer, buf->pos, buf->last - buf->pos, buf->last_buf); 94 | err = jitify_lexer_get_err(jctx->lexer); 95 | if (err) { 96 | char err_buf[DEFAULT_ERR_LEN + 1]; 97 | size_t err_len = DEFAULT_ERR_LEN; 98 | size_t max_err_len = (const char*)(buf->last) - err; 99 | if (err_len > max_err_len) { 100 | err_len = max_err_len; 101 | } 102 | memcpy(err_buf, err, err_len); 103 | err_buf[err_len] = 0; 104 | ngx_log_error(NGX_LOG_WARN, log, 0, "parse error in %V near '%s', entering failsafe mode", &(r->uri), err_buf); 105 | } 106 | } 107 | if (buf->flush) { 108 | send_flush = 1; 109 | } 110 | 111 | /* Setting buf->pos=buf->last enables the nginx core to recycle this buffer */ 112 | if (buf->pos < buf->last) { 113 | buf->pos = buf->last; 114 | } 115 | in = in->next; 116 | } 117 | 118 | if (send_eof) { 119 | size_t processing_time_in_usec = jitify_lexer_get_processing_time(jctx->lexer); 120 | size_t bytes_in = jitify_lexer_get_bytes_in(jctx->lexer); 121 | size_t bytes_out = jitify_lexer_get_bytes_out(jctx->lexer); 122 | ngx_log_error(NGX_LOG_INFO, log, 0, "Jitify stats: bytes_in=%l bytes_out=%l, nsec/byte=%l for %V", 123 | (long)bytes_in, (long)bytes_out, 124 | (long)(bytes_in ? processing_time_in_usec * 1000 / bytes_in : 0), 125 | &(r->uri)); 126 | jitify_nginx_add_eof(&out); 127 | } 128 | if (send_flush && out.last) { 129 | out.last->buf->flush = 1; 130 | } 131 | if (out.first) { 132 | return jitify_next_body_filter(r, out.first); 133 | } 134 | else { 135 | return NGX_OK; 136 | } 137 | } 138 | 139 | static ngx_int_t jitify_post_config(ngx_conf_t *cf) 140 | { 141 | jitify_next_header_filter = ngx_http_top_header_filter; 142 | ngx_http_top_header_filter = jitify_header_filter; 143 | jitify_next_body_filter = ngx_http_top_body_filter; 144 | ngx_http_top_body_filter = jitify_body_filter; 145 | return NGX_OK; 146 | } 147 | 148 | static void *jitify_create_conf(ngx_conf_t *cf) 149 | { 150 | jitify_conf_t *conf; 151 | 152 | conf = ngx_pcalloc(cf->pool, sizeof(*conf)); 153 | if (conf) { 154 | conf->minify = NGX_CONF_UNSET; 155 | } 156 | return conf; 157 | } 158 | 159 | static char *jitify_merge_conf(ngx_conf_t *cf, void *parent, void *child) 160 | { 161 | jitify_conf_t *prev = parent; 162 | jitify_conf_t *conf = child; 163 | 164 | ngx_conf_merge_value(conf->minify, prev->minify, 0); 165 | return NGX_CONF_OK; 166 | } 167 | 168 | static ngx_http_module_t jitify_module_ctx = { 169 | NULL, /* pre-config */ 170 | jitify_post_config, /* post-config */ 171 | NULL, /* create main (top-level) config struct */ 172 | NULL, /* init main (top-level) config struct */ 173 | NULL, /* create server-level config struct */ 174 | NULL, /* merge server-level config struct */ 175 | jitify_create_conf, /* create location-level config struct */ 176 | jitify_merge_conf /* merge location-level config struct */ 177 | }; 178 | 179 | static ngx_command_t jitify_commands[] = { 180 | { 181 | /* minify on|off */ 182 | ngx_string("minify"), 183 | NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, 184 | ngx_conf_set_flag_slot, 185 | NGX_HTTP_LOC_CONF_OFFSET, 186 | offsetof(jitify_conf_t, minify), 187 | NULL 188 | }, 189 | ngx_null_command 190 | }; 191 | 192 | ngx_module_t jitify_module = { 193 | NGX_MODULE_V1, 194 | &jitify_module_ctx, 195 | jitify_commands, 196 | NGX_HTTP_MODULE, 197 | NULL, /* init master */ 198 | NULL, /* init module */ 199 | NULL, /* init process */ 200 | NULL, /* init thread */ 201 | NULL, /* cleanup thread */ 202 | NULL, /* cleanup process */ 203 | NULL, /* cleanup master */ 204 | NGX_MODULE_V1_PADDING 205 | }; 206 | -------------------------------------------------------------------------------- /src/tools/jitify.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | 11 | static const char *PROGRAM_NAME = "jitify"; 12 | #define CONTENT_TYPE_CSS 1 13 | #define CONTENT_TYPE_HTML 2 14 | #define CONTENT_TYPE_JS 3 15 | 16 | static size_t block_size = 8192; 17 | static int max_setaside = -1; 18 | 19 | static int content_type = 0; 20 | static int remove_space = 0; 21 | static int remove_comments = 0; 22 | 23 | static void usage() 24 | { 25 | fprintf(stderr, "usage:\n"); 26 | fprintf(stderr, " %s [options] (--css | --js | --html) # read from stdin, write to stdout\n", PROGRAM_NAME); 27 | fprintf(stderr, " %s [options] filename # read from file, write to stdout\n", PROGRAM_NAME); 28 | fprintf(stderr, "options:\n"); 29 | fprintf(stderr, " --remove-space # remove unnecessary whitespace\n"); 30 | fprintf(stderr, " --remove-comments # remove comments\n"); 31 | fprintf(stderr, " --minify # equivalent to \"--remove-space --remove-comments\"\n"); 32 | fprintf(stderr, " --block-size= # process the input at most n bytes at a time\n"); 33 | } 34 | 35 | static int get_content_type(const char *filename) 36 | { 37 | const char *extension = strrchr(filename, '.'); 38 | if (!extension) { 39 | return 0; 40 | } 41 | extension++; 42 | if (!strcasecmp("htm", extension) || !strcasecmp("html", extension)) { 43 | return CONTENT_TYPE_HTML; 44 | } 45 | if (!strcasecmp("css", extension)) { 46 | return CONTENT_TYPE_CSS; 47 | } 48 | if (!strcasecmp("js", extension)) { 49 | return CONTENT_TYPE_JS; 50 | } 51 | return 0; 52 | } 53 | 54 | static void process_file(int fd) 55 | { 56 | jitify_pool_t *p = jitify_malloc_pool_create(); 57 | jitify_output_stream_t *out = jitify_stdio_output_stream_create(p, stdout); 58 | jitify_lexer_t *lexer; 59 | int bytes_read; 60 | size_t bytes_in, bytes_out, duration; 61 | 62 | char *block; 63 | 64 | switch (content_type) { 65 | case CONTENT_TYPE_CSS: 66 | lexer = jitify_css_lexer_create(p, out); 67 | break; 68 | case CONTENT_TYPE_JS: 69 | lexer = jitify_js_lexer_create(p, out); 70 | break; 71 | case CONTENT_TYPE_HTML: 72 | lexer = jitify_html_lexer_create(p, out); 73 | break; 74 | default: 75 | fprintf(stderr, "%s: internal error\n", PROGRAM_NAME); 76 | return; 77 | } 78 | 79 | if (max_setaside >= 0) { 80 | jitify_lexer_set_max_setaside(lexer, (size_t)max_setaside); 81 | } 82 | jitify_lexer_set_minify_rules(lexer, remove_space, remove_comments); 83 | 84 | block = jitify_malloc(p, block_size); 85 | while ((bytes_read = read(fd, block, block_size)) > 0) { 86 | const char *err; 87 | jitify_lexer_scan(lexer, block, bytes_read, 0); 88 | err = jitify_lexer_get_err(lexer); 89 | if (err) { 90 | size_t err_len = 20; 91 | size_t max_len = (block + bytes_read) - err; 92 | char *err_buf; 93 | if (err_len > max_len) { 94 | err_len = max_len; 95 | } 96 | err_buf = jitify_malloc(p, err_len + 1); 97 | memcpy(err_buf, err, err_len); 98 | err_buf[err_len] = 1; 99 | fprintf(stderr, "parsing error detected near '%s'\n", err_buf); 100 | } 101 | } 102 | if (bytes_read == 0) { 103 | jitify_lexer_scan(lexer, "NULL", 0, 1); 104 | } 105 | bytes_in = jitify_lexer_get_bytes_in(lexer); 106 | bytes_out = jitify_lexer_get_bytes_out(lexer); 107 | duration = jitify_lexer_get_processing_time(lexer); 108 | if (bytes_in) { 109 | fprintf(stderr, "%lu bytes in, %lu bytes out, %lu usec (%lu nsec/byte)\n", 110 | (unsigned long)bytes_in, (unsigned long)bytes_out, (unsigned long)duration, 111 | (unsigned long)((1000 * duration)/bytes_in)); 112 | } 113 | 114 | jitify_free(p, block); 115 | jitify_lexer_destroy(lexer); 116 | jitify_output_stream_destroy(out); 117 | jitify_pool_destroy(p); 118 | } 119 | 120 | #define OPT_MINIFY 1 121 | #define OPT_BLOCK_SIZE 2 122 | #define OPT_MAX_SETASIDE 3 123 | 124 | int main(int argc, char **argv) 125 | { 126 | struct option opts[] = { 127 | { "css", no_argument, &content_type, CONTENT_TYPE_CSS }, 128 | { "html", no_argument, &content_type, CONTENT_TYPE_HTML }, 129 | { "js", no_argument, &content_type, CONTENT_TYPE_JS }, 130 | { "max-setaside", required_argument, NULL, OPT_MAX_SETASIDE }, 131 | { "remove-space", no_argument, &remove_space, 1 }, 132 | { "remove-comments", no_argument, &remove_comments, 1}, 133 | { "minify", no_argument, NULL, OPT_MINIFY }, 134 | { "block-size", required_argument, NULL, OPT_BLOCK_SIZE }, 135 | { NULL, 0, 0, 0 } 136 | }; 137 | int opt; 138 | int fd; 139 | 140 | do { 141 | opt = getopt_long(argc, argv, "", opts, NULL); 142 | switch (opt) { 143 | case OPT_MINIFY: 144 | remove_space = 1; 145 | remove_comments = 1; 146 | break; 147 | case OPT_BLOCK_SIZE: 148 | block_size = atoi(optarg); 149 | break; 150 | case OPT_MAX_SETASIDE: 151 | max_setaside = atoi(optarg); 152 | break; 153 | } 154 | } while (opt != -1); 155 | argc -= optind; 156 | argv += optind; 157 | if (argc > 1) { 158 | usage(); 159 | return 1; 160 | } 161 | else if (argc < 1) { 162 | if (content_type == 0) { 163 | usage(); 164 | return 1; 165 | } 166 | fd = 0; 167 | } 168 | else { 169 | if (content_type == 0) { 170 | content_type = get_content_type(*argv); 171 | if (content_type == 0) { 172 | fprintf(stderr, "%s: cannot determine content-type of %s\n", PROGRAM_NAME, *argv); 173 | fprintf(stderr, " (use --css, --html, or --js to specify)\n"); 174 | return 2; 175 | } 176 | } 177 | fd = open(*argv, O_RDONLY); 178 | if (fd < 0) { 179 | fprintf(stderr, "%s: cannot read %s\n", PROGRAM_NAME, *argv); 180 | return 3; 181 | } 182 | } 183 | process_file(fd); 184 | if (fd != 0) { 185 | close(fd); 186 | } 187 | return 0; 188 | } 189 | --------------------------------------------------------------------------------