├── .gitignore
├── LICENSE
├── MultipartParser.h
├── MultipartReader.h
├── README.markdown
├── Rakefile
├── formidable_parser.js
├── input.txt
├── multipart.cpp
└── rack-parser.rb


/.gitignore:
--------------------------------------------------------------------------------
1 | *.dSYM
2 | multipart
3 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | Copyright (c) 2010 Hongli Lai
 2 | 
 3 | Permission is hereby granted, free of charge, to any person obtaining a copy
 4 | of this software and associated documentation files (the "Software"), to deal
 5 | in the Software without restriction, including without limitation the rights
 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 7 | copies of the Software, and to permit persons to whom the Software is
 8 | furnished to do so, subject to the following conditions:
 9 | 
10 | The above copyright notice and this permission notice shall be included in
11 | all copies or substantial portions of the Software.
12 | 
13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 | THE SOFTWARE.


--------------------------------------------------------------------------------
/MultipartParser.h:
--------------------------------------------------------------------------------
  1 | #ifndef _MULTIPART_PARSER_H_
  2 | #define _MULTIPART_PARSER_H_
  3 | 
  4 | #include <sys/types.h>
  5 | #include <string>
  6 | #include <stdexcept>
  7 | #include <cstring>
  8 | 
  9 | class MultipartParser {
 10 | public:
 11 | 	typedef void (*Callback)(const char *buffer, size_t start, size_t end, void *userData);
 12 | 	
 13 | private:
 14 | 	static const char CR     = 13;
 15 | 	static const char LF     = 10;
 16 | 	static const char SPACE  = 32;
 17 | 	static const char HYPHEN = 45;
 18 | 	static const char COLON  = 58;
 19 | 	static const size_t UNMARKED = (size_t) -1;
 20 | 	
 21 | 	enum State {
 22 | 		ERROR,
 23 | 		START,
 24 | 		START_BOUNDARY,
 25 | 		HEADER_FIELD_START,
 26 | 		HEADER_FIELD,
 27 | 		HEADER_VALUE_START,
 28 | 		HEADER_VALUE,
 29 | 		HEADER_VALUE_ALMOST_DONE,
 30 | 		HEADERS_ALMOST_DONE,
 31 | 		PART_DATA_START,
 32 | 		PART_DATA,
 33 | 		PART_END,
 34 | 		END
 35 | 	};
 36 | 	
 37 | 	enum Flags {
 38 | 		PART_BOUNDARY = 1,
 39 | 		LAST_BOUNDARY = 2
 40 | 	};
 41 | 	
 42 | 	std::string boundary;
 43 | 	const char *boundaryData;
 44 | 	size_t boundarySize;
 45 | 	bool boundaryIndex[256];
 46 | 	char *lookbehind;
 47 | 	size_t lookbehindSize;
 48 | 	State state;
 49 | 	int flags;
 50 | 	size_t index;
 51 | 	size_t headerFieldMark;
 52 | 	size_t headerValueMark;
 53 | 	size_t partDataMark;
 54 | 	const char *errorReason;
 55 | 	
 56 | 	void resetCallbacks() {
 57 | 		onPartBegin   = NULL;
 58 | 		onHeaderField = NULL;
 59 | 		onHeaderValue = NULL;
 60 | 		onHeaderEnd   = NULL;
 61 | 		onHeadersEnd  = NULL;
 62 | 		onPartData    = NULL;
 63 | 		onPartEnd     = NULL;
 64 | 		onEnd         = NULL;
 65 | 		userData      = NULL;
 66 | 	}
 67 | 	
 68 | 	void indexBoundary() {
 69 | 		const char *current;
 70 | 		const char *end = boundaryData + boundarySize;
 71 | 		
 72 | 		memset(boundaryIndex, 0, sizeof(boundaryIndex));
 73 | 		
 74 | 		for (current = boundaryData; current < end; current++) {
 75 | 			boundaryIndex[(unsigned char) *current] = true;
 76 | 		}
 77 | 	}
 78 | 	
 79 | 	void callback(Callback cb, const char *buffer = NULL, size_t start = UNMARKED,
 80 | 		size_t end = UNMARKED, bool allowEmpty = false)
 81 | 	{
 82 | 		if (start != UNMARKED && start == end && !allowEmpty) {
 83 | 			return;
 84 | 		}
 85 | 		if (cb != NULL) {
 86 | 			cb(buffer, start, end, userData);
 87 | 		}
 88 | 	}
 89 | 	
 90 | 	void dataCallback(Callback cb, size_t &mark, const char *buffer, size_t i, size_t bufferLen,
 91 | 		bool clear, bool allowEmpty = false)
 92 | 	{
 93 | 		if (mark == UNMARKED) {
 94 | 			return;
 95 | 		}
 96 | 		
 97 | 		if (!clear) {
 98 | 			callback(cb, buffer, mark, bufferLen, allowEmpty);
 99 | 			mark = 0;
100 | 		} else {
101 | 			callback(cb, buffer, mark, i, allowEmpty);
102 | 			mark = UNMARKED;
103 | 		}
104 | 	}
105 | 	
106 | 	char lower(char c) const {
107 | 		return c | 0x20;
108 | 	}
109 | 	
110 | 	inline bool isBoundaryChar(char c) const {
111 | 		return boundaryIndex[(unsigned char) c];
112 | 	}
113 | 	
114 | 	bool isHeaderFieldCharacter(char c) const {
115 | 		return (c >= 'a' && c <= 'z')
116 | 			|| (c >= 'A' && c <= 'Z')
117 | 			|| c == HYPHEN;
118 | 	}
119 | 	
120 | 	void setError(const char *message) {
121 | 		state = ERROR;
122 | 		errorReason = message;
123 | 	}
124 | 	
125 | 	void processPartData(size_t &prevIndex, size_t &index, const char *buffer,
126 | 		size_t len, size_t boundaryEnd, size_t &i, char c, State &state, int &flags)
127 | 	{
128 | 		prevIndex = index;
129 | 		
130 | 		if (index == 0) {
131 | 			// boyer-moore derived algorithm to safely skip non-boundary data
132 | 			while (i + boundarySize <= len) {
133 | 				if (isBoundaryChar(buffer[i + boundaryEnd])) {
134 | 					break;
135 | 				}
136 | 				
137 | 				i += boundarySize;
138 | 			}
139 | 			if (i == len) {
140 | 				return;
141 | 			}
142 | 			c = buffer[i];
143 | 		}
144 | 		
145 | 		if (index < boundarySize) {
146 | 			if (boundary[index] == c) {
147 | 				if (index == 0) {
148 | 					dataCallback(onPartData, partDataMark, buffer, i, len, true);
149 | 				}
150 | 				index++;
151 | 			} else {
152 | 				index = 0;
153 | 			}
154 | 		} else if (index == boundarySize) {
155 | 			index++;
156 | 			if (c == CR) {
157 | 				// CR = part boundary
158 | 				flags |= PART_BOUNDARY;
159 | 			} else if (c == HYPHEN) {
160 | 				// HYPHEN = end boundary
161 | 				flags |= LAST_BOUNDARY;
162 | 			} else {
163 | 				index = 0;
164 | 			}
165 | 		} else if (index - 1 == boundarySize) {
166 | 			if (flags & PART_BOUNDARY) {
167 | 				index = 0;
168 | 				if (c == LF) {
169 | 					// unset the PART_BOUNDARY flag
170 | 					flags &= ~PART_BOUNDARY;
171 | 					callback(onPartEnd);
172 | 					callback(onPartBegin);
173 | 					state = HEADER_FIELD_START;
174 | 					return;
175 | 				}
176 | 			} else if (flags & LAST_BOUNDARY) {
177 | 				if (c == HYPHEN) {
178 |                     callback(onPartEnd);
179 |                     callback(onEnd);
180 |                     state = END;
181 | 				} else {
182 | 					index = 0;
183 | 				}
184 | 			} else {
185 | 				index = 0;
186 | 			}
187 | 		} else if (index - 2 == boundarySize) {
188 | 			if (c == CR) {
189 | 				index++;
190 | 			} else {
191 | 				index = 0;
192 | 			}
193 | 		} else if (index - boundarySize == 3) {
194 | 			index = 0;
195 | 			if (c == LF) {
196 | 				callback(onPartEnd);
197 | 				callback(onEnd);
198 | 				state = END;
199 | 				return;
200 | 			}
201 | 		}
202 | 		
203 | 		if (index > 0) {
204 | 			// when matching a possible boundary, keep a lookbehind reference
205 | 			// in case it turns out to be a false lead
206 | 			if (index - 1 >= lookbehindSize) {
207 | 				setError("Parser bug: index overflows lookbehind buffer. "
208 | 					"Please send bug report with input file attached.");
209 | 				throw std::out_of_range("index overflows lookbehind buffer");
210 | 			} else if (index - 1 < 0) {
211 | 				setError("Parser bug: index underflows lookbehind buffer. "
212 | 					"Please send bug report with input file attached.");
213 | 				throw std::out_of_range("index underflows lookbehind buffer");
214 | 			}
215 | 			lookbehind[index - 1] = c;
216 | 		} else if (prevIndex > 0) {
217 | 			// if our boundary turned out to be rubbish, the captured lookbehind
218 | 			// belongs to partData
219 | 			callback(onPartData, lookbehind, 0, prevIndex);
220 | 			prevIndex = 0;
221 | 			partDataMark = i;
222 | 			
223 | 			// reconsider the current character even so it interrupted the sequence
224 | 			// it could be the beginning of a new sequence
225 | 			i--;
226 | 		}
227 | 	}
228 | 	
229 | public:
230 | 	Callback onPartBegin;
231 | 	Callback onHeaderField;
232 | 	Callback onHeaderValue;
233 | 	Callback onHeaderEnd;
234 | 	Callback onHeadersEnd;
235 | 	Callback onPartData;
236 | 	Callback onPartEnd;
237 | 	Callback onEnd;
238 | 	void *userData;
239 | 	
240 | 	MultipartParser() {
241 | 		lookbehind = NULL;
242 | 		resetCallbacks();
243 | 		reset();
244 | 	}
245 | 	
246 | 	MultipartParser(const std::string &boundary) {
247 | 		lookbehind = NULL;
248 | 		resetCallbacks();
249 | 		setBoundary(boundary);
250 | 	}
251 | 	
252 | 	~MultipartParser() {
253 | 		delete[] lookbehind;
254 | 	}
255 | 	
256 | 	void reset() {
257 | 		delete[] lookbehind;
258 | 		state = ERROR;
259 | 		boundary.clear();
260 | 		boundaryData = boundary.c_str();
261 | 		boundarySize = 0;
262 | 		lookbehind = NULL;
263 | 		lookbehindSize = 0;
264 | 		flags = 0;
265 | 		index = 0;
266 | 		headerFieldMark = UNMARKED;
267 | 		headerValueMark = UNMARKED;
268 | 		partDataMark    = UNMARKED;
269 | 		errorReason     = "Parser uninitialized.";
270 | 	}
271 | 	
272 | 	void setBoundary(const std::string &boundary) {
273 | 		reset();
274 | 		this->boundary = "\r\n--" + boundary;
275 | 		boundaryData = this->boundary.c_str();
276 | 		boundarySize = this->boundary.size();
277 | 		indexBoundary();
278 | 		lookbehind = new char[boundarySize + 8];
279 | 		lookbehindSize = boundarySize + 8;
280 | 		state = START;
281 | 		errorReason = "No error.";
282 | 	}
283 | 	
284 | 	size_t feed(const char *buffer, size_t len) {
285 | 		if (state == ERROR || len == 0) {
286 | 			return 0;
287 | 		}
288 | 		
289 | 		State state         = this->state;
290 | 		int flags           = this->flags;
291 | 		size_t prevIndex    = this->index;
292 | 		size_t index        = this->index;
293 | 		size_t boundaryEnd  = boundarySize - 1;
294 | 		size_t i;
295 | 		char c, cl;
296 | 		
297 | 		for (i = 0; i < len; i++) {
298 | 			c = buffer[i];
299 | 			
300 | 			switch (state) {
301 | 			case ERROR:
302 | 				return i;
303 | 			case START:
304 | 				index = 0;
305 | 				state = START_BOUNDARY;
306 | 			case START_BOUNDARY:
307 | 				if (index == boundarySize - 2) {
308 | 					if (c != CR) {
309 | 						setError("Malformed. Expected CR after boundary.");
310 | 						return i;
311 | 					}
312 | 					index++;
313 | 					break;
314 | 				} else if (index - 1 == boundarySize - 2) {
315 | 					if (c != LF) {
316 | 						setError("Malformed. Expected LF after boundary CR.");
317 | 						return i;
318 | 					}
319 | 					index = 0;
320 | 					callback(onPartBegin);
321 | 					state = HEADER_FIELD_START;
322 | 					break;
323 | 				}
324 | 				if (c != boundary[index + 2]) {
325 | 					setError("Malformed. Found different boundary data than the given one.");
326 | 					return i;
327 | 				}
328 | 				index++;
329 | 				break;
330 | 			case HEADER_FIELD_START:
331 | 				state = HEADER_FIELD;
332 | 				headerFieldMark = i;
333 | 				index = 0;
334 | 			case HEADER_FIELD:
335 | 				if (c == CR) {
336 | 					headerFieldMark = UNMARKED;
337 | 					state = HEADERS_ALMOST_DONE;
338 | 					break;
339 | 				}
340 | 
341 | 				index++;
342 | 				if (c == HYPHEN) {
343 | 					break;
344 | 				}
345 | 
346 | 				if (c == COLON) {
347 | 					if (index == 1) {
348 | 						// empty header field
349 | 						setError("Malformed first header name character.");
350 | 						return i;
351 | 					}
352 | 					dataCallback(onHeaderField, headerFieldMark, buffer, i, len, true);
353 | 					state = HEADER_VALUE_START;
354 | 					break;
355 | 				}
356 | 
357 | 				cl = lower(c);
358 | 				if (cl < 'a' || cl > 'z') {
359 | 					setError("Malformed header name.");
360 | 					return i;
361 | 				}
362 | 				break;
363 | 			case HEADER_VALUE_START:
364 | 				if (c == SPACE) {
365 | 					break;
366 | 				}
367 | 				
368 | 				headerValueMark = i;
369 | 				state = HEADER_VALUE;
370 | 			case HEADER_VALUE:
371 | 				if (c == CR) {
372 | 					dataCallback(onHeaderValue, headerValueMark, buffer, i, len, true, true);
373 | 					callback(onHeaderEnd);
374 | 					state = HEADER_VALUE_ALMOST_DONE;
375 | 				}
376 | 				break;
377 | 			case HEADER_VALUE_ALMOST_DONE:
378 | 				if (c != LF) {
379 | 					setError("Malformed header value: LF expected after CR");
380 | 					return i;
381 | 				}
382 | 				
383 | 				state = HEADER_FIELD_START;
384 | 				break;
385 | 			case HEADERS_ALMOST_DONE:
386 | 				if (c != LF) {
387 | 					setError("Malformed header ending: LF expected after CR");
388 | 					return i;
389 | 				}
390 | 				
391 | 				callback(onHeadersEnd);
392 | 				state = PART_DATA_START;
393 | 				break;
394 | 			case PART_DATA_START:
395 | 				state = PART_DATA;
396 | 				partDataMark = i;
397 | 			case PART_DATA:
398 | 				processPartData(prevIndex, index, buffer, len, boundaryEnd, i, c, state, flags);
399 | 				break;
400 | 			default:
401 | 				return i;
402 | 			}
403 | 		}
404 | 		
405 | 		dataCallback(onHeaderField, headerFieldMark, buffer, i, len, false);
406 | 		dataCallback(onHeaderValue, headerValueMark, buffer, i, len, false);
407 | 		dataCallback(onPartData, partDataMark, buffer, i, len, false);
408 | 		
409 | 		this->index = index;
410 | 		this->state = state;
411 | 		this->flags = flags;
412 | 		
413 | 		return len;
414 | 	}
415 | 	
416 | 	bool succeeded() const {
417 | 		return state == END;
418 | 	}
419 | 	
420 | 	bool hasError() const {
421 | 		return state == ERROR;
422 | 	}
423 | 	
424 | 	bool stopped() const {
425 | 		return state == ERROR || state == END;
426 | 	}
427 | 	
428 | 	const char *getErrorMessage() const {
429 | 		return errorReason;
430 | 	}
431 | };
432 | 
433 | #endif /* _MULTIPART_PARSER_H_ */
434 | 


--------------------------------------------------------------------------------
/MultipartReader.h:
--------------------------------------------------------------------------------
  1 | #ifndef _MULTIPART_READER_H_
  2 | #define _MULTIPART_READER_H_
  3 | 
  4 | #include <map>
  5 | #include <utility>
  6 | #include "MultipartParser.h"
  7 | 
  8 | class MultipartHeaders: public std::multimap<std::string, std::string> {
  9 | private:
 10 | 	std::string empty;
 11 | public:
 12 | 	const std::string &operator[](const std::string &key) const {
 13 | 		const_iterator it = find(key);
 14 | 		if (it == end()) {
 15 | 			return empty;
 16 | 		} else {
 17 | 			return it->second;
 18 | 		}
 19 | 	}
 20 | };
 21 | 
 22 | class MultipartReader {
 23 | public:
 24 | 	typedef void (*PartBeginCallback)(const MultipartHeaders &headers, void *userData);
 25 | 	typedef void (*PartDataCallback)(const char *buffer, size_t size, void *userData);
 26 | 	typedef void (*Callback)(void *userData);
 27 | 
 28 | private:
 29 | 	MultipartParser parser;
 30 | 	bool headersProcessed;
 31 | 	MultipartHeaders currentHeaders;
 32 | 	std::string currentHeaderName, currentHeaderValue;
 33 | 	
 34 | 	void resetReaderCallbacks() {
 35 | 		onPartBegin = NULL;
 36 | 		onPartData  = NULL;
 37 | 		onPartEnd   = NULL;
 38 | 		onEnd       = NULL;
 39 |         userData    = NULL;
 40 | 	}
 41 | 	
 42 | 	void setParserCallbacks() {
 43 | 		parser.onPartBegin   = cbPartBegin;
 44 | 		parser.onHeaderField = cbHeaderField;
 45 | 		parser.onHeaderValue = cbHeaderValue;
 46 | 		parser.onHeaderEnd   = cbHeaderEnd;
 47 | 		parser.onHeadersEnd  = cbHeadersEnd;
 48 | 		parser.onPartData    = cbPartData;
 49 | 		parser.onPartEnd     = cbPartEnd;
 50 | 		parser.onEnd         = cbEnd;
 51 | 		parser.userData      = this;
 52 | 	}
 53 | 	
 54 | 	static void cbPartBegin(const char *buffer, size_t start, size_t end, void *userData) {
 55 | 		MultipartReader *self = (MultipartReader *) userData;
 56 | 		self->headersProcessed = false;
 57 | 		self->currentHeaders.clear();
 58 | 		self->currentHeaderName.clear();
 59 | 		self->currentHeaderValue.clear();
 60 | 	}
 61 | 	
 62 | 	static void cbHeaderField(const char *buffer, size_t start, size_t end, void *userData) {
 63 | 		MultipartReader *self = (MultipartReader *) userData;
 64 | 		self->currentHeaderName.append(buffer + start, end - start);
 65 | 	}
 66 | 	
 67 | 	static void cbHeaderValue(const char *buffer, size_t start, size_t end, void *userData) {
 68 | 		MultipartReader *self = (MultipartReader *) userData;
 69 | 		self->currentHeaderValue.append(buffer + start, end - start);
 70 | 	}
 71 | 	
 72 | 	static void cbHeaderEnd(const char *buffer, size_t start, size_t end, void *userData) {
 73 | 		MultipartReader *self = (MultipartReader *) userData;
 74 | 		self->currentHeaders.insert(std::make_pair(self->currentHeaderName,
 75 | 			self->currentHeaderValue));
 76 | 		self->currentHeaderName.clear();
 77 | 		self->currentHeaderValue.clear();
 78 | 	}
 79 | 	
 80 | 	static void cbHeadersEnd(const char *buffer, size_t start, size_t end, void *userData) {
 81 | 		MultipartReader *self = (MultipartReader *) userData;
 82 | 		if (self->onPartBegin != NULL) {
 83 | 			self->onPartBegin(self->currentHeaders, self->userData);
 84 | 		}
 85 | 		self->currentHeaders.clear();
 86 | 		self->currentHeaderName.clear();
 87 | 		self->currentHeaderValue.clear();
 88 | 	}
 89 | 	
 90 | 	static void cbPartData(const char *buffer, size_t start, size_t end, void *userData) {
 91 | 		MultipartReader *self = (MultipartReader *) userData;
 92 | 		if (self->onPartData != NULL) {
 93 | 			self->onPartData(buffer + start, end - start, self->userData);
 94 | 		}
 95 | 	}
 96 | 	
 97 | 	static void cbPartEnd(const char *buffer, size_t start, size_t end, void *userData) {
 98 | 		MultipartReader *self = (MultipartReader *) userData;
 99 | 		if (self->onPartEnd != NULL) {
100 | 			self->onPartEnd(self->userData);
101 | 		}
102 | 	}
103 | 	
104 | 	static void cbEnd(const char *buffer, size_t start, size_t end, void *userData) {
105 | 		MultipartReader *self = (MultipartReader *) userData;
106 | 		if (self->onEnd != NULL) {
107 | 			self->onEnd(self->userData);
108 | 		}
109 | 	}
110 | 	
111 | public:
112 | 	PartBeginCallback onPartBegin;
113 | 	PartDataCallback onPartData;
114 | 	Callback onPartEnd;
115 | 	Callback onEnd;
116 |     void *userData;
117 | 	
118 | 	MultipartReader() {
119 | 		resetReaderCallbacks();
120 | 		setParserCallbacks();
121 | 	}
122 | 	
123 | 	MultipartReader(const std::string &boundary): parser(boundary) {
124 | 		resetReaderCallbacks();
125 | 		setParserCallbacks();
126 | 	}
127 | 	
128 | 	void reset() {
129 | 		parser.reset();
130 | 	}
131 | 	
132 | 	void setBoundary(const std::string &boundary) {
133 | 		parser.setBoundary(boundary);
134 | 	}
135 | 	
136 | 	size_t feed(const char *buffer, size_t len) {
137 | 		return parser.feed(buffer, len);
138 | 	}
139 | 	
140 | 	bool succeeded() const {
141 | 		return parser.succeeded();
142 | 	}
143 | 	
144 | 	bool hasError() const {
145 | 		return parser.hasError();
146 | 	}
147 | 	
148 | 	bool stopped() const {
149 | 		return parser.stopped();
150 | 	}
151 | 	
152 | 	const char *getErrorMessage() const {
153 | 		return parser.getErrorMessage();
154 | 	}
155 | };
156 | 
157 | #endif /* _MULTIPART_READER_H_ */
158 | 


--------------------------------------------------------------------------------
/README.markdown:
--------------------------------------------------------------------------------
 1 | What is it?
 2 | ===========
 3 | An simple, efficient parser for multipart MIME messages, based on
 4 | [Formidable's](http://github.com/felixge/node-formidable) parser.
 5 | 
 6 | Why?
 7 | ----
 8 | MIME multipart messages are a total pain to parse because the grammar is
 9 | so insane. Furthermore, the MIME specification is incredibly large. This
10 | has led to an army of equally large and complex MIME libraries. If you
11 | just want to parse a MIME multipart message without hassle then using all
12 | of those libraries are less than ideal. They all tend to handle the kitchen
13 | sink (e.g. they handling email parsing and all kinds of other stuff you don't
14 | need) or they depend on other libraries that you may not want (e.g. APR, glib)
15 | or they are under-documented or under-tested or just not efficient (e.g.
16 | buffering all data in memory; good luck parsing a 2 GB file upload). You can
17 | write your own parser but because the multipart grammar is so much of a pain
18 | it's very easy to make mistakes.
19 | 
20 | Goals and highlights of this parser
21 | -----------------------------------
22 | 
23 |  * Multipart parsing, and only multipart parsing.
24 |  * Event-driven API.
25 |  * No dependencies on any external libraries, just straight C++ with STL.
26 |  * Efficient. Nothing in the input is buffered except what's absolutely
27 |    necessary for parsing.
28 |  * Only one level of multipart parsing. A multipart message part can itself
29 |    be a multipart message, but this parser doesn't attempt to provide a
30 |    complex API for handling nested multipart messages. Instead the developer
31 |    should just use another parser instance to parse nested messages.
32 |  * No I/O is handled for you. This parser won't depend on any particular
33 |    I/O library or even any particular operating system's I/O API. It won't
34 |    block on I/O by itself, giving you full control over when (not) to block.
35 |    It won't save data to files by itself, giving you full control over what to
36 |    do with the parsed data.
37 |  * Not thread-safe, but reentrant. No dependencies on any threading libraries.


--------------------------------------------------------------------------------
/Rakefile:
--------------------------------------------------------------------------------
 1 | task :default => 'multipart'
 2 | 
 3 | file 'multipart' => ['multipart.cpp', 'MultipartParser.h', 'MultipartReader.h'] do
 4 | 	sh 'g++ -Wall -g multipart.cpp -o multipart'
 5 | end
 6 | 
 7 | file 'random' do
 8 | 	sh "dd if=/dev/urandom of=random bs=1048576 count=100"
 9 | end
10 | 
11 | desc "Create a test multipart file"
12 | task :generate_test_file => 'random' do
13 | 	output   = ENV['OUTPUT']
14 | 	size     = (ENV['SIZE'] || 1024 * 1024 * 100).to_i
15 | 	boundary = ENV['BOUNDARY'] || '-----------------------------168072824752491622650073'
16 | 	raise 'OUTPUT must be specified' if !output
17 | 	
18 | 	puts "Creating #{output}"
19 | 	File.open(output, 'wb') do |f|
20 | 		f.write("--#{boundary}\r\n")
21 | 		f.write("content-type: text/plain\r\n")
22 | 		f.write("content-disposition: form-data; name=\"field1\"; filename=\"field1\"\r\n")
23 | 		f.write("foo-bar: abc\r\n")
24 | 		f.write("x: y\r\n")
25 | 		f.write("\r\n")
26 | 	end
27 | 	sh "cat random >> #{output}"
28 | 	puts "Postprocessing #{output}"
29 | 	File.open(output, 'ab') do |f|	
30 | 		f.write("\r\n--#{boundary}--\r\n")
31 | 	end
32 | end
33 | 
34 | task :benchmark => 'multipart' do
35 | 	sh "./multipart"
36 | 	sh "ruby rack-parser.rb"
37 | 	sh "node formidable_parser.js"
38 | end


--------------------------------------------------------------------------------
/formidable_parser.js:
--------------------------------------------------------------------------------
  1 | var Buffer = require('buffer').Buffer
  2 |   , s = 0
  3 |   , S =
  4 |     { PARSER_UNINITIALIZED: s++
  5 |     , START: s++
  6 |     , START_BOUNDARY: s++
  7 |     , HEADER_FIELD_START: s++
  8 |     , HEADER_FIELD: s++
  9 |     , HEADER_VALUE_START: s++
 10 |     , HEADER_VALUE: s++
 11 |     , HEADER_VALUE_ALMOST_DONE: s++
 12 |     , HEADERS_ALMOST_DONE: s++
 13 |     , PART_DATA_START: s++
 14 |     , PART_DATA: s++
 15 |     , PART_END: s++
 16 |     , END: s++
 17 |     }
 18 | 
 19 |   , f = 1
 20 |   , F =
 21 |     { PART_BOUNDARY: f
 22 |     , LAST_BOUNDARY: f *= 2
 23 |     }
 24 | 
 25 |   , LF = 10
 26 |   , CR = 13
 27 |   , SPACE = 32
 28 |   , HYPHEN = 45
 29 |   , COLON = 58
 30 |   , A = 97
 31 |   , Z = 122
 32 | 
 33 |   , lower = function(c) {
 34 |       return c | 0x20;
 35 |     };
 36 | 
 37 | for (var s in S) {
 38 |   exports[s] = S[s];
 39 | }
 40 | 
 41 | function MultipartParser() {
 42 |   this.boundary = null;
 43 |   this.boundaryChars = null;
 44 |   this.lookbehind = null;
 45 |   this.state = S.PARSER_UNINITIALIZED;
 46 | 
 47 |   this.index = null;
 48 |   this.flags = 0;
 49 | };
 50 | exports.MultipartParser = MultipartParser;
 51 | 
 52 | MultipartParser.prototype.initWithBoundary = function(str) {
 53 |   this.boundary = new Buffer(str.length+4);
 54 |   this.boundary.write('\r\n--', 'ascii', 0);
 55 |   this.boundary.write(str, 'ascii', 4);
 56 |   this.lookbehind = new Buffer(this.boundary.length+8);
 57 |   this.state = S.START;
 58 | 
 59 |   this.boundaryChars = {};
 60 |   for (var i = 0; i < this.boundary.length; i++) {
 61 |     this.boundaryChars[this.boundary[i]] = true;
 62 |   }
 63 | };
 64 | 
 65 | MultipartParser.prototype.write = function(buffer) {
 66 |   var self = this
 67 |     , i = 0
 68 |     , len = buffer.length
 69 |     , prevIndex = this.index
 70 |     , index = this.index
 71 |     , state = this.state
 72 |     , flags = this.flags
 73 |     , lookbehind = this.lookbehind
 74 |     , boundary = this.boundary
 75 |     , boundaryChars = this.boundaryChars
 76 |     , boundaryLength = this.boundary.length
 77 |     , boundaryEnd = boundaryLength - 1
 78 |     , bufferLength = buffer.length
 79 |     , c
 80 |     , cl
 81 | 
 82 |     , mark = function(name) {
 83 |         self[name+'Mark'] = i;
 84 |       }
 85 |     , clear = function(name) {
 86 |         delete self[name+'Mark'];
 87 |       }
 88 |     , callback = function(name, buffer, start, end) {
 89 |         if (start !== undefined && start === end) {
 90 |           return;
 91 |         }
 92 | 
 93 |         var callbackSymbol = 'on'+name.substr(0, 1).toUpperCase()+name.substr(1);
 94 |         if (callbackSymbol in self) {
 95 |           self[callbackSymbol](buffer, start, end);
 96 |         }
 97 |       }
 98 |     , dataCallback = function(name, clear) {
 99 |         var markSymbol = name+'Mark';
100 |         if (!(markSymbol in self)) {
101 |           return;
102 |         }
103 | 
104 |         if (!clear) {
105 |           callback(name, buffer, self[markSymbol], buffer.length);
106 |           self[markSymbol] = 0;
107 |         } else {
108 |           callback(name, buffer, self[markSymbol], i);
109 |           delete self[markSymbol];
110 |         }
111 |       };
112 | 
113 |   for (i = 0; i < len; i++) {
114 |     c = buffer[i];
115 |     switch (state) {
116 |       case S.PARSER_UNINITIALIZED:
117 |         return i;
118 |       case S.START:
119 |         index = 0;
120 |         state = S.START_BOUNDARY;
121 |       case S.START_BOUNDARY:
122 |         if (index == boundary.length - 2) {
123 |           if (c != CR) {
124 |             return i;
125 |           }
126 |           index++;
127 |           break;
128 |         } else if (index - 1 == boundary.length - 2) {
129 |           if (c != LF) {
130 |             return i;
131 |           }
132 |           index = 0;
133 |           callback('partBegin');
134 |           state = S.HEADER_FIELD_START;
135 |           break;
136 |         }
137 | 
138 |         if (c != boundary[index+2]) {
139 |           return i;
140 |         }
141 |         index++;
142 |         break;
143 |       case S.HEADER_FIELD_START:
144 |         state = S.HEADER_FIELD;
145 |         mark('headerField');
146 |       case S.HEADER_FIELD:
147 |         if (c == CR) {
148 |           clear('headerField');
149 |           state = S.HEADERS_ALMOST_DONE;
150 |           break;
151 |         }
152 | 
153 |         if (c == HYPHEN) {
154 |           break;
155 |         }
156 | 
157 |         if (c == COLON) {
158 |           dataCallback('headerField', true);
159 |           state = S.HEADER_VALUE_START;
160 |           break;
161 |         }
162 | 
163 |         cl = lower(c);
164 |         if (cl < A || cl > Z) {
165 |           return i;
166 |         }
167 |         break;
168 |       case S.HEADER_VALUE_START:
169 |         if (c == SPACE) {
170 |           break;
171 |         }
172 | 
173 |         mark('headerValue');
174 |         state = S.HEADER_VALUE;
175 |       case S.HEADER_VALUE:
176 |         if (c == CR) {
177 |           dataCallback('headerValue', true);
178 |           state = S.HEADER_VALUE_ALMOST_DONE;
179 |         }
180 |         break;
181 |       case S.HEADER_VALUE_ALMOST_DONE:
182 |         if (c != LF) {
183 |           return i;
184 |         }
185 |         state = S.HEADER_FIELD_START;
186 |         break;
187 |       case S.HEADERS_ALMOST_DONE:
188 |         if (c != LF) {
189 |           return i;
190 |         }
191 | 
192 |         state = S.PART_DATA_START;
193 |         break;
194 |       case S.PART_DATA_START:
195 |         state = S.PART_DATA
196 |         mark('partData');
197 |       case S.PART_DATA:
198 |         prevIndex = index;
199 | 
200 |         if (index == 0) {
201 |           // boyer-moore derrived algorithm to safely skip non-boundary data
202 |           while (i + boundaryLength <= bufferLength) {
203 |             if (buffer[i + boundaryEnd] in boundaryChars) {
204 |               break;
205 |             }
206 | 
207 |             i += boundaryLength;
208 |           }
209 |           c = buffer[i];
210 |         }
211 | 
212 |         if (index < boundary.length) {
213 |           if (boundary[index] == c) {
214 |             if (index == 0) {
215 |               dataCallback('partData', true);
216 |             }
217 |             index++;
218 |           } else {
219 |             index = 0;
220 |           }
221 |         } else if (index == boundary.length) {
222 |           index++;
223 |           if (c == CR) {
224 |             // CR = part boundary
225 |             flags |= F.PART_BOUNDARY;
226 |           } else if (c == HYPHEN) {
227 |             // HYPHEN = end boundary
228 |             flags |= F.LAST_BOUNDARY;
229 |           } else {
230 |             index = 0;
231 |           }
232 |         } else if (index - 1 == boundary.length)  {
233 |           if (flags & F.PART_BOUNDARY) {
234 |             index = 0;
235 |             if (c == LF) {
236 |               // unset the PART_BOUNDARY flag
237 |               flags &= ~F.PART_BOUNDARY;
238 |               callback('partEnd');
239 |               callback('partBegin');
240 |               state = S.HEADER_FIELD_START;
241 |               break;
242 |             }
243 |           } else if (flags & F.LAST_BOUNDARY) {
244 |             if (c == HYPHEN) {
245 |               index++;
246 |             } else {
247 |               index = 0;
248 |             }
249 |           } else {
250 |             index = 0;
251 |           }
252 |         } else if (index - 2 == boundary.length)  {
253 |           if (c == CR) {
254 |             index++;
255 |           } else {
256 |             index = 0;
257 |           }
258 |         } else if (index - boundary.length == 3)  {
259 |           index = 0;
260 |           if (c == LF) {
261 |             callback('partEnd');
262 |             callback('end');
263 |             state = S.END;
264 |             break;
265 |           }
266 |         }
267 | 
268 |         if (index > 0) {
269 |           // when matching a possible boundary, keep a lookbehind reference
270 |           // in case it turns out to be a false lead
271 |           lookbehind[index-1] = c;
272 |         } else if (prevIndex > 0) {
273 |           // if our boundary turned out to be rubbish, the captured lookbehind
274 |           // belongs to partData
275 |           callback('partData', lookbehind, 0, prevIndex);
276 |           prevIndex = 0;
277 |           mark('partData');
278 |         }
279 |         
280 |         break;
281 |       default:
282 |         return i;
283 |     }
284 |   }
285 | 
286 |   dataCallback('headerField');
287 |   dataCallback('headerValue');
288 |   dataCallback('partData');
289 | 
290 |   this.index = index;
291 |   this.state = state;
292 |   this.flags = flags;
293 | 
294 |   return len;
295 | };
296 | 
297 | MultipartParser.prototype.end = function() {
298 |   if (this.state != S.END) {
299 |     return new Error('MultipartParser.end(): stream ended unexpectedly');
300 |   }
301 | };
302 | 
303 | function sprintf ( ) {
304 |     // http://kevin.vanzonneveld.net
305 |     // +   original by: Ash Searle (http://hexmen.com/blog/)
306 |     // + namespaced by: Michael White (http://getsprink.com)
307 |     // +    tweaked by: Jack
308 |     // +   improved by: Kevin van Zonneveld (http://kevin.vanzonneveld.net)
309 |     // +      input by: Paulo Freitas
310 |     // +   improved by: Kevin van Zonneveld (http://kevin.vanzonneveld.net)
311 |     // +      input by: Brett Zamir (http://brett-zamir.me)
312 |     // +   improved by: Kevin van Zonneveld (http://kevin.vanzonneveld.net)
313 |     // *     example 1: sprintf("%01.2f", 123.1);
314 |     // *     returns 1: 123.10
315 |     // *     example 2: sprintf("[%10s]", 'monkey');
316 |     // *     returns 2: '[    monkey]'
317 |     // *     example 3: sprintf("[%'#10s]", 'monkey');
318 |     // *     returns 3: '[####monkey]'
319 | 
320 |     var regex = /%%|%(\d+\$)?([-+\'#0 ]*)(\*\d+\$|\*|\d+)?(\.(\*\d+\$|\*|\d+))?([scboxXuidfegEG])/g;
321 |     var a = arguments, i = 0, format = a[i++];
322 | 
323 |     // pad()
324 |     var pad = function (str, len, chr, leftJustify) {
325 |         if (!chr) {chr = ' ';}
326 |         var padding = (str.length >= len) ? '' : Array(1 + len - str.length >>> 0).join(chr);
327 |         return leftJustify ? str + padding : padding + str;
328 |     };
329 | 
330 |     // justify()
331 |     var justify = function (value, prefix, leftJustify, minWidth, zeroPad, customPadChar) {
332 |         var diff = minWidth - value.length;
333 |         if (diff > 0) {
334 |             if (leftJustify || !zeroPad) {
335 |                 value = pad(value, minWidth, customPadChar, leftJustify);
336 |             } else {
337 |                 value = value.slice(0, prefix.length) + pad('', diff, '0', true) + value.slice(prefix.length);
338 |             }
339 |         }
340 |         return value;
341 |     };
342 | 
343 |     // formatBaseX()
344 |     var formatBaseX = function (value, base, prefix, leftJustify, minWidth, precision, zeroPad) {
345 |         // Note: casts negative numbers to positive ones
346 |         var number = value >>> 0;
347 |         prefix = prefix && number && {'2': '0b', '8': '0', '16': '0x'}[base] || '';
348 |         value = prefix + pad(number.toString(base), precision || 0, '0', false);
349 |         return justify(value, prefix, leftJustify, minWidth, zeroPad);
350 |     };
351 | 
352 |     // formatString()
353 |     var formatString = function (value, leftJustify, minWidth, precision, zeroPad, customPadChar) {
354 |         if (precision != null) {
355 |             value = value.slice(0, precision);
356 |         }
357 |         return justify(value, '', leftJustify, minWidth, zeroPad, customPadChar);
358 |     };
359 | 
360 |     // doFormat()
361 |     var doFormat = function (substring, valueIndex, flags, minWidth, _, precision, type) {
362 |         var number;
363 |         var prefix;
364 |         var method;
365 |         var textTransform;
366 |         var value;
367 | 
368 |         if (substring == '%%') {return '%';}
369 | 
370 |         // parse flags
371 |         var leftJustify = false, positivePrefix = '', zeroPad = false, prefixBaseX = false, customPadChar = ' ';
372 |         var flagsl = flags.length;
373 |         for (var j = 0; flags && j < flagsl; j++) {
374 |             switch (flags.charAt(j)) {
375 |                 case ' ': positivePrefix = ' '; break;
376 |                 case '+': positivePrefix = '+'; break;
377 |                 case '-': leftJustify = true; break;
378 |                 case "'": customPadChar = flags.charAt(j+1); break;
379 |                 case '0': zeroPad = true; break;
380 |                 case '#': prefixBaseX = true; break;
381 |             }
382 |         }
383 | 
384 |         // parameters may be null, undefined, empty-string or real valued
385 |         // we want to ignore null, undefined and empty-string values
386 |         if (!minWidth) {
387 |             minWidth = 0;
388 |         } else if (minWidth == '*') {
389 |             minWidth = +a[i++];
390 |         } else if (minWidth.charAt(0) == '*') {
391 |             minWidth = +a[minWidth.slice(1, -1)];
392 |         } else {
393 |             minWidth = +minWidth;
394 |         }
395 | 
396 |         // Note: undocumented perl feature:
397 |         if (minWidth < 0) {
398 |             minWidth = -minWidth;
399 |             leftJustify = true;
400 |         }
401 | 
402 |         if (!isFinite(minWidth)) {
403 |             throw new Error('sprintf: (minimum-)width must be finite');
404 |         }
405 | 
406 |         if (!precision) {
407 |             precision = 'fFeE'.indexOf(type) > -1 ? 6 : (type == 'd') ? 0 : undefined;
408 |         } else if (precision == '*') {
409 |             precision = +a[i++];
410 |         } else if (precision.charAt(0) == '*') {
411 |             precision = +a[precision.slice(1, -1)];
412 |         } else {
413 |             precision = +precision;
414 |         }
415 | 
416 |         // grab value using valueIndex if required?
417 |         value = valueIndex ? a[valueIndex.slice(0, -1)] : a[i++];
418 | 
419 |         switch (type) {
420 |             case 's': return formatString(String(value), leftJustify, minWidth, precision, zeroPad, customPadChar);
421 |             case 'c': return formatString(String.fromCharCode(+value), leftJustify, minWidth, precision, zeroPad);
422 |             case 'b': return formatBaseX(value, 2, prefixBaseX, leftJustify, minWidth, precision, zeroPad);
423 |             case 'o': return formatBaseX(value, 8, prefixBaseX, leftJustify, minWidth, precision, zeroPad);
424 |             case 'x': return formatBaseX(value, 16, prefixBaseX, leftJustify, minWidth, precision, zeroPad);
425 |             case 'X': return formatBaseX(value, 16, prefixBaseX, leftJustify, minWidth, precision, zeroPad).toUpperCase();
426 |             case 'u': return formatBaseX(value, 10, prefixBaseX, leftJustify, minWidth, precision, zeroPad);
427 |             case 'i':
428 |             case 'd':
429 |                 number = parseInt(+value, 10);
430 |                 prefix = number < 0 ? '-' : positivePrefix;
431 |                 value = prefix + pad(String(Math.abs(number)), precision, '0', false);
432 |                 return justify(value, prefix, leftJustify, minWidth, zeroPad);
433 |             case 'e':
434 |             case 'E':
435 |             case 'f':
436 |             case 'F':
437 |             case 'g':
438 |             case 'G':
439 |                 number = +value;
440 |                 prefix = number < 0 ? '-' : positivePrefix;
441 |                 method = ['toExponential', 'toFixed', 'toPrecision']['efg'.indexOf(type.toLowerCase())];
442 |                 textTransform = ['toString', 'toUpperCase']['eEfFgG'.indexOf(type) % 2];
443 |                 value = prefix + Math.abs(number)[method](precision);
444 |                 return justify(value, prefix, leftJustify, minWidth, zeroPad)[textTransform]();
445 |             default: return substring;
446 |         }
447 |     };
448 | 
449 |     return format.replace(regex, doFormat);
450 | }
451 | 
452 | 
453 | 
454 | var fs = require('fs');
455 | var sys = require('sys');
456 | 
457 | 
458 | var FILENAME = 'input3.txt';
459 | //var BOUNDARY = 'abcd';
460 | var BOUNDARY = "-----------------------------168072824752491622650073";
461 | var TIMES = 10;
462 | var SLURP = true;
463 | var QUIET = true;
464 | 
465 | 
466 | function createParser() {
467 |   var parser = new MultipartParser();
468 |   parser.initWithBoundary(BOUNDARY);
469 |   if (!QUIET) {
470 |     parser.onPartBegin = function() {
471 |       sys.puts('onPartBegin');
472 |     };
473 |     parser.onHeaderField = function(buffer, start, end) {
474 |       sys.puts('onHeaderField: ' + buffer.slice(start, end));
475 |     };
476 |     parser.onHeaderValue = function(buffer, start, end) {
477 |       sys.puts('onHeaderValue: ' + buffer.slice(start, end));
478 |     };
479 |     parser.onPartData = function(buffer, start, end) {
480 |       //sys.puts('onPartData: ' + buffer.slice(start, end));
481 |       //sys.puts('onPartData: ' + (end - start) + ' bytes');
482 |     };
483 |     parser.onPartEnd = function() {
484 |       sys.puts('onPartEnd');
485 |     };
486 |     parser.onEnd = function() {
487 |       sys.puts('onEnd');
488 |     };
489 |   }
490 |   return parser;
491 | }
492 | 
493 | var parser, i, eof, bytesRead, buf, error, stime, etime;
494 | var stats = fs.statSync(FILENAME);
495 | if (SLURP) {
496 |   fd = fs.openSync(FILENAME, 'r', 0);
497 |   buf = new Buffer(stats.size, 'binary');
498 |   fs.readSync(fd, buf, 0, buf.length, null);
499 |   
500 |   stime = new Date();
501 |   for (i = 0; i < TIMES; i++) {
502 |     parser = createParser();
503 |     parser.write(buf);
504 |   }
505 | } else {
506 |   buf = new Buffer(1024 * 32, 'binary');
507 |   stime = new Date();
508 |   for (i = 0; i < TIMES; i++) {
509 |     parser = createParser();
510 |     if (!QUIET) {
511 |       sys.puts('-------------------');
512 |     }
513 |     fd = fs.openSync(FILENAME, 'r', 0);
514 |     eof = false;
515 |     totalRead = 0;
516 |     while (!eof) {
517 |       bytesRead = fs.readSync(fd, buf, 0, buf.length, null);
518 |       eof = bytesRead == 0;
519 |       if (!eof) {
520 |         parser.write(buf);
521 |       }
522 |     }
523 |     parser.end();
524 |   }
525 | }
526 | 
527 | etime = new Date();
528 | var diff = etime.getTime() - stime.getTime();
529 | sys.puts(sprintf("(JS)     Total: %.2fs   Per run: %.2fs   Throughput: %.2f MB/sec",
530 |   diff / 1000.0,
531 |   diff / TIMES / 1000.0,
532 |   (stats.size * TIMES) / (diff / 1000) / 1024.0 / 1024.0
533 | ));


--------------------------------------------------------------------------------
/input.txt:
--------------------------------------------------------------------------------
 1 | --abcd
 2 | content-type: text/plain
 3 | content-disposition: form-data; name="field1"; filename="field1"
 4 | foo-bar: abc
 5 | x: y
 6 | 
 7 | hello world
 8 | 
 9 | x
10 | 
11 | --abcd--
12 | 


--------------------------------------------------------------------------------
/multipart.cpp:
--------------------------------------------------------------------------------
  1 | #include "MultipartParser.h"
  2 | #include "MultipartReader.h"
  3 | #include <sys/stat.h>
  4 | #include <sys/time.h>
  5 | #include <sys/types.h>
  6 | #include <cstdio>
  7 | #include <unistd.h>
  8 | #include <cstdlib>
  9 | #include <cstring>
 10 | 
 11 | //#define TEST_PARSER
 12 | #define INPUT_FILE "input3.txt"
 13 | //#define BOUNDARY "abcd"
 14 | #define BOUNDARY "-----------------------------168072824752491622650073"
 15 | #define TIMES 10
 16 | #define SLURP
 17 | #define QUIET
 18 | 
 19 | 
 20 | using namespace std;
 21 | 
 22 | #ifdef TEST_PARSER
 23 | 	static void
 24 | 	onPartBegin(const char *buffer, size_t start, size_t end, void *userData) {
 25 | 		printf("onPartBegin\n");
 26 | 	}
 27 | 
 28 | 	static void
 29 | 	onHeaderField(const char *buffer, size_t start, size_t end, void *userData) {
 30 | 		printf("onHeaderField: (%s)\n", string(buffer + start, end - start).c_str());
 31 | 	}
 32 | 
 33 | 	static void
 34 | 	onHeaderValue(const char *buffer, size_t start, size_t end, void *userData) {
 35 | 		printf("onHeaderValue: (%s)\n", string(buffer + start, end - start).c_str());
 36 | 	}
 37 | 
 38 | 	static void
 39 | 	onPartData(const char *buffer, size_t start, size_t end, void *userData) {
 40 | 		printf("onPartData: (%s)\n", string(buffer + start, end - start).c_str());
 41 | 	}
 42 | 
 43 | 	static void
 44 | 	onPartEnd(const char *buffer, size_t start, size_t end, void *userData) {
 45 | 		printf("onPartEnd\n");
 46 | 	}
 47 | 
 48 | 	static void
 49 | 	onEnd(const char *buffer, size_t start, size_t end, void *userData) {
 50 | 		printf("onEnd\n");
 51 | 	}
 52 | #else
 53 | 	void onPartBegin(const MultipartHeaders &headers, void *userData) {
 54 | 		printf("onPartBegin:\n");
 55 | 		MultipartHeaders::const_iterator it;
 56 | 		MultipartHeaders::const_iterator end = headers.end();
 57 | 		for (it = headers.begin(); it != headers.end(); it++) {
 58 | 			printf("  %s = %s\n", it->first.c_str(), it->second.c_str());
 59 | 		}
 60 | 		printf("  aaa: %s\n", headers["aaa"].c_str());
 61 | 	}
 62 | 	
 63 | 	void onPartData(const char *buffer, size_t size, void *userData) {
 64 | 		//printf("onPartData: (%s)\n", string(buffer, size).c_str());
 65 | 	}
 66 | 	
 67 | 	void onPartEnd(void *userData) {
 68 | 		printf("onPartEnd\n");
 69 | 	}
 70 | 	
 71 | 	void onEnd(void *userData) {
 72 | 		printf("onEnd\n");
 73 | 	}
 74 | #endif
 75 | 
 76 | int
 77 | main() {
 78 | 	#ifdef TEST_PARSER
 79 | 		MultipartParser parser;
 80 | 		#ifndef QUIET
 81 | 			parser.onPartBegin = onPartBegin;
 82 | 			parser.onHeaderField = onHeaderField;
 83 | 			parser.onHeaderValue = onHeaderValue;
 84 | 			parser.onPartData = onPartData;
 85 | 			parser.onPartEnd = onPartEnd;
 86 | 			parser.onEnd = onEnd;
 87 | 		#endif
 88 | 	#else
 89 | 		MultipartReader parser;
 90 | 		#ifndef QUIET
 91 | 			parser.onPartBegin = onPartBegin;
 92 | 			parser.onPartData = onPartData;
 93 | 			parser.onPartEnd = onPartEnd;
 94 | 			parser.onEnd = onEnd;
 95 | 		#endif
 96 | 	#endif
 97 | 	
 98 | 	struct timeval stime, etime;
 99 | 	struct stat sbuf;
100 | 	
101 | 	stat(INPUT_FILE, &sbuf);
102 | 	
103 | 	#ifdef SLURP
104 | 		size_t bufsize = sbuf.st_size;
105 | 		char *buf = (char *) malloc(bufsize);
106 | 		
107 | 		FILE *f = fopen(INPUT_FILE, "rb");
108 | 		fread(buf, 1, bufsize, f);
109 | 		fclose(f);
110 | 		
111 | 		gettimeofday(&stime, NULL);
112 | 		for (int i = 0; i < TIMES; i++) {
113 | 			#ifndef QUIET
114 | 				printf("------------\n");
115 | 			#endif
116 | 			parser.setBoundary(BOUNDARY);
117 | 			
118 | 			size_t fed = 0;
119 | 			do {
120 | 				size_t ret = parser.feed(buf + fed, bufsize - fed);
121 | 				fed += ret;
122 | 			} while (fed < bufsize && !parser.stopped());
123 | 			#ifndef QUIET
124 | 				printf("%s\n", parser.getErrorMessage());
125 | 			#endif
126 | 		}
127 | 		gettimeofday(&etime, NULL);
128 | 	#else
129 | 		size_t bufsize = 1024 * 32;
130 | 		char *buf = (char *) malloc(bufsize);
131 | 		
132 | 		gettimeofday(&stime, NULL);
133 | 		for (int i = 0; i < TIMES; i++) {
134 | 			#ifndef QUIET
135 | 				printf("------------\n");
136 | 			#endif
137 | 			parser.setBoundary(BOUNDARY);
138 | 			
139 | 			FILE *f = fopen(INPUT_FILE, "rb");
140 | 			while (!parser.stopped() && !feof(f)) {
141 | 				size_t len = fread(buf, 1, bufsize, f);
142 | 				size_t fed = 0;
143 | 				do {
144 | 					size_t ret = parser.feed(buf + fed, len - fed);
145 | 					fed += ret;
146 | 				} while (fed < len && !parser.stopped());
147 | 			}
148 | 			#ifndef QUIET
149 | 				printf("%s\n", parser.getErrorMessage());
150 | 			#endif
151 | 			fclose(f);
152 | 		}
153 | 		gettimeofday(&etime, NULL);
154 | 	#endif
155 | 	
156 | 	unsigned long long a = (unsigned long long) stime.tv_sec * 1000000 + stime.tv_usec;
157 | 	unsigned long long b = (unsigned long long) etime.tv_sec * 1000000 + etime.tv_usec;
158 | 	printf("(C++)    Total: %.2fs   Per run: %.2fs   Throughput: %.2f MB/sec\n",
159 | 		(b - a) / 1000000.0,
160 | 		(b - a) / TIMES / 1000000.0,
161 | 		((unsigned long long) sbuf.st_size * TIMES) / ((b - a) / 1000000.0) / 1024.0 / 1024.0);
162 | 	
163 | 	return 0;
164 | }
165 | 


--------------------------------------------------------------------------------
/rack-parser.rb:
--------------------------------------------------------------------------------
  1 | # encoding: binary
  2 | require 'stringio'
  3 | 
  4 | module Utils
  5 |   if ''.respond_to?(:bytesize)
  6 |     def bytesize(string)
  7 |       string.bytesize
  8 |     end
  9 |   else
 10 |     def bytesize(string)
 11 |       string.size
 12 |     end
 13 |   end
 14 |   module_function :bytesize
 15 |   
 16 |   # Unescapes a URI escaped string. (Stolen from Camping).
 17 |   def unescape(s)
 18 |     s.tr('+', ' ').gsub(/((?:%[0-9a-fA-F]{2})+)/n){
 19 |       [$1.delete('%')].pack('H*')
 20 |     }
 21 |   end
 22 |   module_function :unescape
 23 |   
 24 |   def normalize_params(params, name, v = nil)
 25 |     name =~ %r(\A[\[\]]*([^\[\]]+)\]*)
 26 |     k = $1 || ''
 27 |     after = $' || ''
 28 | 
 29 |     return if k.empty?
 30 | 
 31 |     if after == ""
 32 |       params[k] = v
 33 |     elsif after == "[]"
 34 |       params[k] ||= []
 35 |       raise TypeError, "expected Array (got #{params[k].class.name}) for param `#{k}'" unless params[k].is_a?(Array)
 36 |       params[k] << v
 37 |     elsif after =~ %r(^\[\]\[([^\[\]]+)\]$) || after =~ %r(^\[\](.+)$)
 38 |       child_key = $1
 39 |       params[k] ||= []
 40 |       raise TypeError, "expected Array (got #{params[k].class.name}) for param `#{k}'" unless params[k].is_a?(Array)
 41 |       if params[k].last.is_a?(Hash) && !params[k].last.key?(child_key)
 42 |         normalize_params(params[k].last, child_key, v)
 43 |       else
 44 |         params[k] << normalize_params({}, child_key, v)
 45 |       end
 46 |     else
 47 |       params[k] ||= {}
 48 |       raise TypeError, "expected Hash (got #{params[k].class.name}) for param `#{k}'" unless params[k].is_a?(Hash)
 49 |       params[k] = normalize_params(params[k], after, v)
 50 |     end
 51 | 
 52 |     return params
 53 |   end
 54 |   module_function :normalize_params
 55 | end
 56 | 
 57 | module Multipart
 58 |   Tempfile = StringIO
 59 |   
 60 |   EOL = "\r\n"
 61 |   MULTIPART_BOUNDARY = "AaB03x"
 62 | 
 63 |   def self.parse_multipart(env)
 64 |     unless env['CONTENT_TYPE'] =~
 65 |         %r|\Amultipart/.*boundary=\"?([^\";,]+)\"?|n
 66 |       nil
 67 |     else
 68 |       boundary = "--#{$1}"
 69 | 
 70 |       params = {}
 71 |       buf = ""
 72 |       content_length = env['CONTENT_LENGTH'].to_i
 73 |       input = env['rack.input']
 74 |       input.rewind
 75 | 
 76 |       boundary_size = Utils.bytesize(boundary) + EOL.size
 77 |       bufsize = 16384
 78 | 
 79 |       content_length -= boundary_size
 80 | 
 81 |       read_buffer = ''
 82 | 
 83 |       status = input.read(boundary_size, read_buffer)
 84 |       raise EOFError, "bad content body"  unless status == boundary + EOL
 85 | 
 86 |       rx = /(?:#{EOL})?#{Regexp.quote boundary}(#{EOL}|--)/n
 87 | 
 88 |       loop {
 89 |         head = nil
 90 |         body = ''
 91 |         filename = content_type = name = nil
 92 | 
 93 |         until head && buf =~ rx
 94 |           if !head && i = buf.index(EOL+EOL)
 95 |             head = buf.slice!(0, i+2) # First \r\n
 96 |             buf.slice!(0, 2)          # Second \r\n
 97 | 
 98 |             token = /[^\s()<>,;:\\"\/\[\]?=]+/
 99 |             condisp = /Content-Disposition:\s*#{token}\s*/i
100 |             dispparm = /;\s*(#{token})=("(?:\\"|[^"])*"|#{token})*/
101 | 
102 |             rfc2183 = /^#{condisp}(#{dispparm})+$/i
103 |             broken_quoted = /^#{condisp}.*;\sfilename="(.*?)"(?:\s*$|\s*;\s*#{token}=)/i
104 |             broken_unquoted = /^#{condisp}.*;\sfilename=(#{token})/i
105 | 
106 |             if head =~ rfc2183
107 |               filename = Hash[head.scan(dispparm)]['filename']
108 |               filename = $1 if filename and filename =~ /^"(.*)"$/
109 |             elsif head =~ broken_quoted
110 |               filename = $1
111 |             elsif head =~ broken_unquoted
112 |               filename = $1
113 |             end
114 | 
115 |             if filename && filename !~ /\\[^\\"]/
116 |               filename = Utils.unescape(filename).gsub(/\\(.)/, '\1')
117 |             end
118 | 
119 |             content_type = head[/Content-Type: (.*)#{EOL}/ni, 1]
120 |             name = head[/Content-Disposition:.*\s+name="?([^\";]*)"?/ni, 1] || head[/Content-ID:\s*([^#{EOL}]*)/ni, 1]
121 | 
122 |             if filename
123 |               body = Tempfile.new("RackMultipart")
124 |               body.binmode  if body.respond_to?(:binmode)
125 |             end
126 | 
127 |             next
128 |           end
129 | 
130 |           # Save the read body part.
131 |           if head && (boundary_size+4 < buf.size)
132 |             body << buf.slice!(0, buf.size - (boundary_size+4))
133 |           end
134 | 
135 |           c = input.read(bufsize < content_length ? bufsize : content_length, read_buffer)
136 |           raise EOFError, "bad content body"  if c.nil? || c.empty?
137 |           buf << c
138 |           content_length -= c.size
139 |         end
140 | 
141 |         # Save the rest.
142 |         if i = buf.index(rx)
143 |           body << buf.slice!(0, i)
144 |           buf.slice!(0, boundary_size+2)
145 | 
146 |           content_length = -1  if $1 == "--"
147 |         end
148 | 
149 |         if filename == ""
150 |           # filename is blank which means no file has been selected
151 |           data = nil
152 |         elsif filename
153 |           body.rewind
154 | 
155 |           # Take the basename of the upload's original filename.
156 |           # This handles the full Windows paths given by Internet Explorer
157 |           # (and perhaps other broken user agents) without affecting
158 |           # those which give the lone filename.
159 |           filename = filename.split(/[\/\\]/).last
160 | 
161 |           data = {:filename => filename, :type => content_type,
162 |                   :name => name, :tempfile => body, :head => head}
163 |         elsif !filename && content_type
164 |           body.rewind
165 | 
166 |           # Generic multipart cases, not coming from a form
167 |           data = {:type => content_type,
168 |                   :name => name, :tempfile => body, :head => head}
169 |         else
170 |           data = body
171 |         end
172 | 
173 |         Utils.normalize_params(params, name, data) unless data.nil?
174 | 
175 |         # break if we're at the end of a buffer, but not if it is the end of a field
176 |         break if (buf.empty? && $1 != EOL) || content_length == -1
177 |       }
178 | 
179 |       input.rewind
180 | 
181 |       params
182 |     end
183 |   end
184 | end
185 | 
186 | require 'benchmark'
187 | 
188 | FILENAME = 'input3.txt'
189 | #BOUNDARY = "abcd"
190 | BOUNDARY = "-----------------------------168072824752491622650073"
191 | TIMES = 10
192 | SLURP = true
193 | 
194 | if SLURP
195 | 	io = StringIO.new(File.read(FILENAME))
196 | else
197 | 	io = File.open(FILENAME, 'rb')
198 | end
199 | env = {
200 |   'CONTENT_LENGTH' => File.size(FILENAME),
201 |   'CONTENT_TYPE' => "multipart/form-data; boundary=#{BOUNDARY}",
202 |   'rack.input' => io
203 | }
204 | result = Benchmark.measure do
205 |   TIMES.times do
206 |     Multipart.parse_multipart(env)
207 |   end
208 | end
209 | printf "(Ruby)   Total: %.2fs   Per run: %.2fs   Throughput: %.2f MB/sec\n",
210 | 	result.total,
211 | 	result.total / TIMES,
212 | 	(File.size(FILENAME) * TIMES) / result.total / 1024.0 / 1024.0


--------------------------------------------------------------------------------