├── .gitignore ├── .travis.yml ├── Cargo.toml ├── LICENSE ├── README.md ├── bench ├── client │ ├── Cargo.toml │ └── src │ │ └── main.rs └── server │ ├── Cargo.toml │ └── src │ └── main.rs └── src ├── config.rs ├── lib.rs ├── server.rs └── types.rs /.gitignore: -------------------------------------------------------------------------------- 1 | # Build 2 | target 3 | 4 | # Cargo 5 | *.lock 6 | 7 | # OS 8 | *.DS_Store 9 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: rust 2 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "hydrogen" 3 | version = "0.1.5" 4 | authors = ["Nathan Sizemore "] 5 | description = "Multithreaded Linux TCP socket server using epoll." 6 | keywords = ["epoll", "non-blocking", "socket-server"] 7 | readme = "README.md" 8 | license = "MPL-2.0" 9 | repository = "https://github.com/nathansizemore/hydrogen" 10 | documentation = "https://nathansizemore.github.io/hydrogen/hydrogen/index.html" 11 | 12 | 13 | [lib] 14 | name = "hydrogen" 15 | 16 | [dependencies] 17 | log = "^0.3" 18 | libc = "^0.2" 19 | errno = "^0.1.6" 20 | threadpool = "1" 21 | simple-slab = "^0.2" 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Mozilla Public License, version 2.0 2 | 3 | 1. Definitions 4 | 5 | 1.1. "Contributor" 6 | 7 | means each individual or legal entity that creates, contributes to the 8 | creation of, or owns Covered Software. 9 | 10 | 1.2. "Contributor Version" 11 | 12 | means the combination of the Contributions of others (if any) used by a 13 | Contributor and that particular Contributor's Contribution. 14 | 15 | 1.3. "Contribution" 16 | 17 | means Covered Software of a particular Contributor. 18 | 19 | 1.4. "Covered Software" 20 | 21 | means Source Code Form to which the initial Contributor has attached the 22 | notice in Exhibit A, the Executable Form of such Source Code Form, and 23 | Modifications of such Source Code Form, in each case including portions 24 | thereof. 25 | 26 | 1.5. "Incompatible With Secondary Licenses" 27 | means 28 | 29 | a. that the initial Contributor has attached the notice described in 30 | Exhibit B to the Covered Software; or 31 | 32 | b. that the Covered Software was made available under the terms of 33 | version 1.1 or earlier of the License, but not also under the terms of 34 | a Secondary License. 35 | 36 | 1.6. "Executable Form" 37 | 38 | means any form of the work other than Source Code Form. 39 | 40 | 1.7. "Larger Work" 41 | 42 | means a work that combines Covered Software with other material, in a 43 | separate file or files, that is not Covered Software. 44 | 45 | 1.8. "License" 46 | 47 | means this document. 48 | 49 | 1.9. "Licensable" 50 | 51 | means having the right to grant, to the maximum extent possible, whether 52 | at the time of the initial grant or subsequently, any and all of the 53 | rights conveyed by this License. 54 | 55 | 1.10. "Modifications" 56 | 57 | means any of the following: 58 | 59 | a. any file in Source Code Form that results from an addition to, 60 | deletion from, or modification of the contents of Covered Software; or 61 | 62 | b. any new file in Source Code Form that contains any Covered Software. 63 | 64 | 1.11. "Patent Claims" of a Contributor 65 | 66 | means any patent claim(s), including without limitation, method, 67 | process, and apparatus claims, in any patent Licensable by such 68 | Contributor that would be infringed, but for the grant of the License, 69 | by the making, using, selling, offering for sale, having made, import, 70 | or transfer of either its Contributions or its Contributor Version. 71 | 72 | 1.12. "Secondary License" 73 | 74 | means either the GNU General Public License, Version 2.0, the GNU Lesser 75 | General Public License, Version 2.1, the GNU Affero General Public 76 | License, Version 3.0, or any later versions of those licenses. 77 | 78 | 1.13. "Source Code Form" 79 | 80 | means the form of the work preferred for making modifications. 81 | 82 | 1.14. "You" (or "Your") 83 | 84 | means an individual or a legal entity exercising rights under this 85 | License. For legal entities, "You" includes any entity that controls, is 86 | controlled by, or is under common control with You. For purposes of this 87 | definition, "control" means (a) the power, direct or indirect, to cause 88 | the direction or management of such entity, whether by contract or 89 | otherwise, or (b) ownership of more than fifty percent (50%) of the 90 | outstanding shares or beneficial ownership of such entity. 91 | 92 | 93 | 2. License Grants and Conditions 94 | 95 | 2.1. Grants 96 | 97 | Each Contributor hereby grants You a world-wide, royalty-free, 98 | non-exclusive license: 99 | 100 | a. under intellectual property rights (other than patent or trademark) 101 | Licensable by such Contributor to use, reproduce, make available, 102 | modify, display, perform, distribute, and otherwise exploit its 103 | Contributions, either on an unmodified basis, with Modifications, or 104 | as part of a Larger Work; and 105 | 106 | b. under Patent Claims of such Contributor to make, use, sell, offer for 107 | sale, have made, import, and otherwise transfer either its 108 | Contributions or its Contributor Version. 109 | 110 | 2.2. Effective Date 111 | 112 | The licenses granted in Section 2.1 with respect to any Contribution 113 | become effective for each Contribution on the date the Contributor first 114 | distributes such Contribution. 115 | 116 | 2.3. Limitations on Grant Scope 117 | 118 | The licenses granted in this Section 2 are the only rights granted under 119 | this License. No additional rights or licenses will be implied from the 120 | distribution or licensing of Covered Software under this License. 121 | Notwithstanding Section 2.1(b) above, no patent license is granted by a 122 | Contributor: 123 | 124 | a. for any code that a Contributor has removed from Covered Software; or 125 | 126 | b. for infringements caused by: (i) Your and any other third party's 127 | modifications of Covered Software, or (ii) the combination of its 128 | Contributions with other software (except as part of its Contributor 129 | Version); or 130 | 131 | c. under Patent Claims infringed by Covered Software in the absence of 132 | its Contributions. 133 | 134 | This License does not grant any rights in the trademarks, service marks, 135 | or logos of any Contributor (except as may be necessary to comply with 136 | the notice requirements in Section 3.4). 137 | 138 | 2.4. Subsequent Licenses 139 | 140 | No Contributor makes additional grants as a result of Your choice to 141 | distribute the Covered Software under a subsequent version of this 142 | License (see Section 10.2) or under the terms of a Secondary License (if 143 | permitted under the terms of Section 3.3). 144 | 145 | 2.5. Representation 146 | 147 | Each Contributor represents that the Contributor believes its 148 | Contributions are its original creation(s) or it has sufficient rights to 149 | grant the rights to its Contributions conveyed by this License. 150 | 151 | 2.6. Fair Use 152 | 153 | This License is not intended to limit any rights You have under 154 | applicable copyright doctrines of fair use, fair dealing, or other 155 | equivalents. 156 | 157 | 2.7. Conditions 158 | 159 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in 160 | Section 2.1. 161 | 162 | 163 | 3. Responsibilities 164 | 165 | 3.1. Distribution of Source Form 166 | 167 | All distribution of Covered Software in Source Code Form, including any 168 | Modifications that You create or to which You contribute, must be under 169 | the terms of this License. You must inform recipients that the Source 170 | Code Form of the Covered Software is governed by the terms of this 171 | License, and how they can obtain a copy of this License. You may not 172 | attempt to alter or restrict the recipients' rights in the Source Code 173 | Form. 174 | 175 | 3.2. Distribution of Executable Form 176 | 177 | If You distribute Covered Software in Executable Form then: 178 | 179 | a. such Covered Software must also be made available in Source Code Form, 180 | as described in Section 3.1, and You must inform recipients of the 181 | Executable Form how they can obtain a copy of such Source Code Form by 182 | reasonable means in a timely manner, at a charge no more than the cost 183 | of distribution to the recipient; and 184 | 185 | b. You may distribute such Executable Form under the terms of this 186 | License, or sublicense it under different terms, provided that the 187 | license for the Executable Form does not attempt to limit or alter the 188 | recipients' rights in the Source Code Form under this License. 189 | 190 | 3.3. Distribution of a Larger Work 191 | 192 | You may create and distribute a Larger Work under terms of Your choice, 193 | provided that You also comply with the requirements of this License for 194 | the Covered Software. If the Larger Work is a combination of Covered 195 | Software with a work governed by one or more Secondary Licenses, and the 196 | Covered Software is not Incompatible With Secondary Licenses, this 197 | License permits You to additionally distribute such Covered Software 198 | under the terms of such Secondary License(s), so that the recipient of 199 | the Larger Work may, at their option, further distribute the Covered 200 | Software under the terms of either this License or such Secondary 201 | License(s). 202 | 203 | 3.4. Notices 204 | 205 | You may not remove or alter the substance of any license notices 206 | (including copyright notices, patent notices, disclaimers of warranty, or 207 | limitations of liability) contained within the Source Code Form of the 208 | Covered Software, except that You may alter any license notices to the 209 | extent required to remedy known factual inaccuracies. 210 | 211 | 3.5. Application of Additional Terms 212 | 213 | You may choose to offer, and to charge a fee for, warranty, support, 214 | indemnity or liability obligations to one or more recipients of Covered 215 | Software. However, You may do so only on Your own behalf, and not on 216 | behalf of any Contributor. You must make it absolutely clear that any 217 | such warranty, support, indemnity, or liability obligation is offered by 218 | You alone, and You hereby agree to indemnify every Contributor for any 219 | liability incurred by such Contributor as a result of warranty, support, 220 | indemnity or liability terms You offer. You may include additional 221 | disclaimers of warranty and limitations of liability specific to any 222 | jurisdiction. 223 | 224 | 4. Inability to Comply Due to Statute or Regulation 225 | 226 | If it is impossible for You to comply with any of the terms of this License 227 | with respect to some or all of the Covered Software due to statute, 228 | judicial order, or regulation then You must: (a) comply with the terms of 229 | this License to the maximum extent possible; and (b) describe the 230 | limitations and the code they affect. Such description must be placed in a 231 | text file included with all distributions of the Covered Software under 232 | this License. Except to the extent prohibited by statute or regulation, 233 | such description must be sufficiently detailed for a recipient of ordinary 234 | skill to be able to understand it. 235 | 236 | 5. Termination 237 | 238 | 5.1. The rights granted under this License will terminate automatically if You 239 | fail to comply with any of its terms. However, if You become compliant, 240 | then the rights granted under this License from a particular Contributor 241 | are reinstated (a) provisionally, unless and until such Contributor 242 | explicitly and finally terminates Your grants, and (b) on an ongoing 243 | basis, if such Contributor fails to notify You of the non-compliance by 244 | some reasonable means prior to 60 days after You have come back into 245 | compliance. Moreover, Your grants from a particular Contributor are 246 | reinstated on an ongoing basis if such Contributor notifies You of the 247 | non-compliance by some reasonable means, this is the first time You have 248 | received notice of non-compliance with this License from such 249 | Contributor, and You become compliant prior to 30 days after Your receipt 250 | of the notice. 251 | 252 | 5.2. If You initiate litigation against any entity by asserting a patent 253 | infringement claim (excluding declaratory judgment actions, 254 | counter-claims, and cross-claims) alleging that a Contributor Version 255 | directly or indirectly infringes any patent, then the rights granted to 256 | You by any and all Contributors for the Covered Software under Section 257 | 2.1 of this License shall terminate. 258 | 259 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user 260 | license agreements (excluding distributors and resellers) which have been 261 | validly granted by You or Your distributors under this License prior to 262 | termination shall survive termination. 263 | 264 | 6. Disclaimer of Warranty 265 | 266 | Covered Software is provided under this License on an "as is" basis, 267 | without warranty of any kind, either expressed, implied, or statutory, 268 | including, without limitation, warranties that the Covered Software is free 269 | of defects, merchantable, fit for a particular purpose or non-infringing. 270 | The entire risk as to the quality and performance of the Covered Software 271 | is with You. Should any Covered Software prove defective in any respect, 272 | You (not any Contributor) assume the cost of any necessary servicing, 273 | repair, or correction. This disclaimer of warranty constitutes an essential 274 | part of this License. No use of any Covered Software is authorized under 275 | this License except under this disclaimer. 276 | 277 | 7. Limitation of Liability 278 | 279 | Under no circumstances and under no legal theory, whether tort (including 280 | negligence), contract, or otherwise, shall any Contributor, or anyone who 281 | distributes Covered Software as permitted above, be liable to You for any 282 | direct, indirect, special, incidental, or consequential damages of any 283 | character including, without limitation, damages for lost profits, loss of 284 | goodwill, work stoppage, computer failure or malfunction, or any and all 285 | other commercial damages or losses, even if such party shall have been 286 | informed of the possibility of such damages. This limitation of liability 287 | shall not apply to liability for death or personal injury resulting from 288 | such party's negligence to the extent applicable law prohibits such 289 | limitation. Some jurisdictions do not allow the exclusion or limitation of 290 | incidental or consequential damages, so this exclusion and limitation may 291 | not apply to You. 292 | 293 | 8. Litigation 294 | 295 | Any litigation relating to this License may be brought only in the courts 296 | of a jurisdiction where the defendant maintains its principal place of 297 | business and such litigation shall be governed by laws of that 298 | jurisdiction, without reference to its conflict-of-law provisions. Nothing 299 | in this Section shall prevent a party's ability to bring cross-claims or 300 | counter-claims. 301 | 302 | 9. Miscellaneous 303 | 304 | This License represents the complete agreement concerning the subject 305 | matter hereof. If any provision of this License is held to be 306 | unenforceable, such provision shall be reformed only to the extent 307 | necessary to make it enforceable. Any law or regulation which provides that 308 | the language of a contract shall be construed against the drafter shall not 309 | be used to construe this License against a Contributor. 310 | 311 | 312 | 10. Versions of the License 313 | 314 | 10.1. New Versions 315 | 316 | Mozilla Foundation is the license steward. Except as provided in Section 317 | 10.3, no one other than the license steward has the right to modify or 318 | publish new versions of this License. Each version will be given a 319 | distinguishing version number. 320 | 321 | 10.2. Effect of New Versions 322 | 323 | You may distribute the Covered Software under the terms of the version 324 | of the License under which You originally received the Covered Software, 325 | or under the terms of any subsequent version published by the license 326 | steward. 327 | 328 | 10.3. Modified Versions 329 | 330 | If you create software not governed by this License, and you want to 331 | create a new license for such software, you may create and use a 332 | modified version of this License if you rename the license and remove 333 | any references to the name of the license steward (except to note that 334 | such modified license differs from this License). 335 | 336 | 10.4. Distributing Source Code Form that is Incompatible With Secondary 337 | Licenses If You choose to distribute Source Code Form that is 338 | Incompatible With Secondary Licenses under the terms of this version of 339 | the License, the notice described in Exhibit B of this License must be 340 | attached. 341 | 342 | Exhibit A - Source Code Form License Notice 343 | 344 | This Source Code Form is subject to the 345 | terms of the Mozilla Public License, v. 346 | 2.0. If a copy of the MPL was not 347 | distributed with this file, You can 348 | obtain one at 349 | http://mozilla.org/MPL/2.0/. 350 | 351 | If it is not possible or desirable to put the notice in a particular file, 352 | then You may include the notice in a location (such as a LICENSE file in a 353 | relevant directory) where a recipient would be likely to look for such a 354 | notice. 355 | 356 | You may add additional accurate notices of copyright ownership. 357 | 358 | Exhibit B - "Incompatible With Secondary Licenses" Notice 359 | 360 | This Source Code Form is "Incompatible 361 | With Secondary Licenses", as defined by 362 | the Mozilla Public License, v. 2.0. 363 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # hydrogen [][travis-badge] 2 | 3 | [Documentation][docs] 4 | 5 | hydrogen is a non-blocking socket server framework built atop [epoll][epoll-man-page]. 6 | It takes care of the tedious connection and I/O marshaling across threads, and 7 | leaves the specifics of I/O reading and writing up the consumer, through trait 8 | implementations. 9 | 10 | --- 11 | 12 | ## Streams 13 | 14 | hydrogen works with `Stream` trait Objects so any custom type can be used. 15 | [simple-stream][simple-stream-repo] was built in conjunction and offers several 16 | stream abstractions and types including Plain and Secured streams with basic 17 | and WebSocket framing. 18 | 19 | ## Multithreaded 20 | 21 | hydrogen is multithreaded. It uses one thread for accepting incoming 22 | connections, one for updating epoll reported event, and one used for 23 | marshalling I/O into a threadpool of a user specified size. 24 | 25 | ## Slab allocation 26 | 27 | The connection pool is managed as a slab, which means traversal times are 28 | similar to traversing a Vector, with an insertion and removal time of O(1). 29 | 30 | 31 | ## Examples 32 | 33 | ### [simple-stream][simple-stream-repo] 34 | 35 | ``` rust 36 | extern crate hydrogen; 37 | extern crate simple_stream as ss; 38 | 39 | use hydrogen; 40 | use hydrogen::{Stream as HydrogenStream, HydrogenSocket}; 41 | use ss::frame::Frame; 42 | use ss::frame::simple::{SimpleFrame, SimpleFrameBuilder}; 43 | use ss::{Socket, Plain, NonBlocking, SocketOptions}; 44 | 45 | 46 | 47 | // Hydrogen requires a type that implements `hydrogen::Stream`. 48 | // We'll implement it atop the `simple-stream` crate. 49 | #[derive(Clone)] 50 | pub struct Stream { 51 | inner: Plain 52 | } 53 | 54 | impl HydrogenStream for Stream { 55 | // This method is called when epoll reports data is available for reading. 56 | fn recv(&mut self) -> Result>, Error> { 57 | match self.inner.nb_recv() { 58 | Ok(frame_vec) => { 59 | let mut ret_buf = Vec::>::with_capacity(frame_vec.len()); 60 | for frame in frame_vec.iter() { 61 | ret_buf.push(frame.payload()); 62 | } 63 | Ok(ret_buf) 64 | } 65 | Err(e) => Err(e) 66 | } 67 | } 68 | 69 | // This method is called when a previous attempt to write has returned `ErrorKind::WouldBlock` 70 | // and epoll has reported that the socket is now writable. 71 | fn send(&mut self, buf: &[u8]) -> Result<(), Error> { 72 | let frame = SimpleFrame::new(buf); 73 | self.inner.nb_send(&frame) 74 | } 75 | 76 | // This method is called when connection has been reported as reset by epoll, or when any 77 | // `std::io::Error` has been returned. 78 | fn shutdown(&mut self) -> Result<(), Error> { 79 | self.inner.shutdown() 80 | } 81 | } 82 | impl AsRawFd for Stream { 83 | fn as_raw_fd(&self) -> RawFd { self.inner.as_raw_fd() } 84 | } 85 | 86 | 87 | // The following will be our server that handles all reported events 88 | struct Server; 89 | impl hydrogen::Handler for Server { 90 | fn on_server_created(&mut self, fd: RawFd) { 91 | // Do any secific flag/option setting on the underlying listening fd. 92 | // This will be the fd that accepts all incoming connections. 93 | } 94 | 95 | fn on_new_connection(&mut self, fd: RawFd) -> Arc> { 96 | // With the passed fd, create your type that implements `hydrogen::Stream` 97 | // and return it. 98 | } 99 | 100 | fn on_data_received(&mut self, socket: HydrogenSocket, buffer: Vec) { 101 | // Called when a complete, consumer defined, chunk of data has been read. 102 | } 103 | 104 | fn on_connection_removed(&mut self, fd: RawFd, err: Error) { 105 | // Called when a connection has been removed from the watch list, with the 106 | // `std::io::Error` as the reason removed. 107 | } 108 | } 109 | 110 | 111 | fn main() { 112 | hydrogen::begin(Server, hydrogen::Config { 113 | addr: "0.0.0.0".to_string(), 114 | port: 1337, 115 | max_threads: 8, 116 | pre_allocated: 100000 117 | }); 118 | } 119 | ``` 120 | 121 | ### `std::net::TcpStream` 122 | 123 | ``` rust 124 | extern crate hydrogen; 125 | 126 | use std::cell::UnsafeCell; 127 | use std::io::{Read, Write, Error, ErrorKind}; 128 | use std::net::{TcpStream, Shutdown}; 129 | use std::os::unix::io::{AsRawFd, RawFd}; 130 | use std::sync::Arc; 131 | 132 | use hydrogen::{Stream as HydrogenStream, HydrogenSocket}; 133 | 134 | 135 | pub struct Stream { 136 | inner: TcpStream 137 | } 138 | 139 | impl Stream { 140 | pub fn from_tcp_stream(tcp_stream: TcpStream) -> Stream { 141 | tcp_stream.set_nonblocking(true); 142 | Stream { 143 | inner: tcp_stream 144 | } 145 | } 146 | } 147 | 148 | impl HydrogenStream for Stream { 149 | // This method is called when epoll reports data is available for reading. 150 | fn recv(&mut self) -> Result>, Error> { 151 | let mut msgs = Vec::>::new(); 152 | 153 | // Our socket is set to non-blocking, we need to read until 154 | // there is an error or the system returns WouldBlock. 155 | // TcpStream offers no guarantee it will return in non-blocking mode. 156 | // Double check OS specifics on this when using. 157 | // https://doc.rust-lang.org/std/io/trait.Read.html#tymethod.read 158 | let mut total_read = Vec::::new(); 159 | loop { 160 | let mut buf = [0u8; 4098]; 161 | let read_result = self.inner.read(&mut buf); 162 | if read_result.is_err() { 163 | let err = read_result.unwrap_err(); 164 | if err.kind() == ErrorKind::WouldBlock { 165 | break; 166 | } 167 | 168 | return Err(err); 169 | } 170 | 171 | let num_read = read_result.unwrap(); 172 | total_read.extend_from_slice(&buf[0..num_read]); 173 | } 174 | 175 | // Multiple frames, or "msgs", could have been gathered here. Break up 176 | // your frames here and save remainer somewhere to come back to on the 177 | // next reads.... 178 | // 179 | // Frame break out code goes here 180 | // 181 | 182 | msgs.push(total_read); 183 | 184 | return Ok(msgs); 185 | } 186 | 187 | // This method is called when a previous attempt to write has returned `ErrorKind::WouldBlock` 188 | // and epoll has reported that the socket is now writable. 189 | fn send(&mut self, buf: &[u8]) -> Result<(), Error> { 190 | self.inner.write_all(buf) 191 | } 192 | 193 | // This method is called when connection has been reported as reset by epoll, or when any 194 | // `std::io::Error` has been returned. 195 | fn shutdown(&mut self) -> Result<(), Error> { 196 | self.inner.shutdown(Shutdown::Both) 197 | } 198 | } 199 | 200 | impl AsRawFd for Stream { 201 | fn as_raw_fd(&self) -> RawFd { self.inner.as_raw_fd() } 202 | } 203 | 204 | // The following will be our server that handles all reported events 205 | struct Server; 206 | impl hydrogen::Handler for Server { 207 | fn on_server_created(&mut self, fd: RawFd) { 208 | // Do any secific flag/option setting on the underlying listening fd. 209 | // This will be the fd that accepts all incoming connections. 210 | } 211 | 212 | fn on_new_connection(&mut self, fd: RawFd) -> Arc> { 213 | // With the passed fd, create your type that implements `hydrogen::Stream` 214 | // and return it. 215 | } 216 | 217 | fn on_data_received(&mut self, socket: HydrogenSocket, buffer: Vec) { 218 | // Called when a complete, consumer defined, chunk of data has been read. 219 | } 220 | 221 | fn on_connection_removed(&mut self, fd: RawFd, err: Error) { 222 | // Called when a connection has been removed from the watch list, with the 223 | // `std::io::Error` as the reason removed. 224 | } 225 | } 226 | 227 | 228 | fn main() { 229 | hydrogen::begin(Server, hydrogen::Config { 230 | addr: "0.0.0.0".to_string(), 231 | port: 1337, 232 | max_threads: 8, 233 | pre_allocated: 100000 234 | }); 235 | } 236 | ``` 237 | 238 | ## Author 239 | 240 | Nathan Sizemore, nathanrsizemore@gmail.com 241 | 242 | ## License 243 | 244 | hydrogen is available under the MPL-2.0 license. See the LICENSE file for more info. 245 | 246 | 247 | 248 | [travis-badge]: https://travis-ci.org/nathansizemore/hydrogen 249 | [docs]: https://nathansizemore.github.io/hydrogen/hydrogen/index.html 250 | [epoll-man-page]: http://man7.org/linux/man-pages/man7/epoll.7.html 251 | [simple-stream-repo]: https://github.com/nathansizemore/simple-stream 252 | -------------------------------------------------------------------------------- /bench/client/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "client" 3 | version = "0.1.0" 4 | authors = ["Nathan Sizemore "] 5 | 6 | [dependencies] 7 | log = "^0.3.6" 8 | time = "^0.1.35" 9 | env_logger = "^0.3.3" 10 | simple-stream = "^0.9.6" 11 | -------------------------------------------------------------------------------- /bench/client/src/main.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Nathan Sizemore 2 | // 3 | // This Source Code Form is subject to the terms of the 4 | // Mozilla Public License, v. 2.0. If a copy of the MPL was not 5 | // distributed with this file, You can obtain one at 6 | // http://mozilla.org/MPL/2.0/. 7 | 8 | 9 | #[macro_use] 10 | extern crate log; 11 | extern crate time; 12 | extern crate env_logger; 13 | extern crate simple_stream as ss; 14 | 15 | 16 | use std::thread; 17 | use std::net::TcpStream; 18 | use std::os::unix::io::IntoRawFd; 19 | 20 | use time::Timespec; 21 | use ss::frame::simple::{SimpleFrame, SimpleFrameBuilder}; 22 | use ss::{Socket, Plain, Blocking}; 23 | 24 | 25 | const TOTAL_MESSAGES: usize = 100000; 26 | 27 | static mut start_time: *mut Timespec = 0 as *mut Timespec; 28 | 29 | 30 | #[derive(Clone)] 31 | struct Client { 32 | stream: Plain 33 | } 34 | 35 | fn main() { 36 | env_logger::init().unwrap(); 37 | 38 | let tcp_stream = TcpStream::connect("127.0.0.1:1337").unwrap(); 39 | let fd = tcp_stream.into_raw_fd(); 40 | let socket = Socket::new(fd); 41 | let client = Client { 42 | stream: Plain::::new(socket) 43 | }; 44 | 45 | let client_clone = client.clone(); 46 | let rx_thread = thread::spawn(move || reader_thread(client_clone)); 47 | thread::spawn(move || writer_thread(client)); 48 | let _ = rx_thread.join(); 49 | } 50 | 51 | fn reader_thread(mut client: Client) { 52 | let mut frames_read: usize = 0; 53 | while frames_read < TOTAL_MESSAGES { 54 | match client.stream.b_recv() { 55 | Ok(_) => frames_read += 1, 56 | Err(e) => { 57 | error!("During recv: {}", e); 58 | return; 59 | } 60 | } 61 | } 62 | 63 | let end_time = time::get_time(); 64 | unsafe { 65 | info!("{} frames sent in {}", TOTAL_MESSAGES, (end_time - *start_time)); 66 | } 67 | } 68 | 69 | fn writer_thread(mut client: Client) { 70 | let mut ping = [0u8; 4]; 71 | ping[0] = 'p' as u8; 72 | ping[1] = 'i' as u8; 73 | ping[2] = 'n' as u8; 74 | ping[3] = 'g' as u8; 75 | 76 | let frame = SimpleFrame::new(&ping[..]); 77 | 78 | let t_spec = time::get_time(); 79 | unsafe { 80 | start_time = Box::into_raw(Box::new(t_spec)); 81 | } 82 | 83 | for _ in 0..TOTAL_MESSAGES { 84 | match client.stream.b_send(&frame) { 85 | Ok(_) => { } 86 | Err(e) => { 87 | error!("During send: {}", e); 88 | return; 89 | } 90 | } 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /bench/server/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "server" 3 | version = "0.1.0" 4 | authors = ["Nathan Sizemore "] 5 | 6 | [dependencies] 7 | log = "^0.3.6" 8 | libc = "^0.2.10" 9 | env_logger = "^0.3.3" 10 | simple-stream = "^0.9.6" 11 | hydrogen = { path = "../../" } 12 | -------------------------------------------------------------------------------- /bench/server/src/main.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Nathan Sizemore 2 | // 3 | // This Source Code Form is subject to the terms of the 4 | // Mozilla Public License, v. 2.0. If a copy of the MPL was not 5 | // distributed with this file, You can obtain one at 6 | // http://mozilla.org/MPL/2.0/. 7 | 8 | 9 | #[macro_use] 10 | extern crate log; 11 | extern crate libc; 12 | extern crate env_logger; 13 | extern crate hydrogen; 14 | extern crate simple_stream as ss; 15 | 16 | 17 | use std::mem; 18 | use std::io::Error; 19 | use std::sync::Arc; 20 | use std::cell::UnsafeCell; 21 | use std::os::unix::io::{RawFd, AsRawFd}; 22 | 23 | use hydrogen::{Stream as HydrogenStream, HydrogenSocket}; 24 | use ss::frame::Frame; 25 | use ss::frame::simple::{SimpleFrame, SimpleFrameBuilder}; 26 | use ss::{Socket, Plain, NonBlocking, SocketOptions}; 27 | 28 | 29 | 30 | struct Stream { 31 | inner: Plain 32 | } 33 | 34 | impl HydrogenStream for Stream { 35 | fn recv(&mut self) -> Result>, Error> { 36 | match self.inner.nb_recv() { 37 | Ok(frame_vec) => { 38 | let mut ret_buf = Vec::>::with_capacity(frame_vec.len()); 39 | for frame in frame_vec.iter() { 40 | ret_buf.push(frame.payload()); 41 | } 42 | Ok(ret_buf) 43 | } 44 | Err(e) => Err(e) 45 | } 46 | } 47 | fn send(&mut self, buf: &[u8]) -> Result<(), Error> { 48 | let frame = SimpleFrame::new(buf); 49 | self.inner.nb_send(&frame) 50 | } 51 | fn shutdown(&mut self) -> Result<(), Error> { 52 | self.inner.shutdown() 53 | } 54 | } 55 | 56 | impl AsRawFd for Stream { 57 | fn as_raw_fd(&self) -> RawFd { self.inner.as_raw_fd() } 58 | } 59 | 60 | struct Server; 61 | impl hydrogen::Handler for Server { 62 | fn on_server_created(&mut self, fd: RawFd) { 63 | let mut socket = Socket::new(fd); 64 | let _ = socket.set_reuseaddr(true); 65 | } 66 | 67 | fn on_new_connection(&mut self, fd: RawFd) -> Arc> { 68 | let mut socket = Socket::new(fd); 69 | let _ = socket.set_nonblocking(); 70 | let _ = socket.set_keepalive(true); 71 | 72 | unsafe { 73 | let opt = 1; 74 | let _ = libc::setsockopt(socket.as_raw_fd(), 75 | libc::IPPROTO_TCP, 76 | libc::TCP_NODELAY, 77 | &opt as *const _ as *const libc::c_void, 78 | mem::size_of::() as u32); 79 | } 80 | 81 | let plain_stream = Plain::::new(socket); 82 | let stream = Stream { 83 | inner: plain_stream 84 | }; 85 | 86 | Arc::new(UnsafeCell::new(stream)) 87 | } 88 | 89 | #[allow(unused_variables)] 90 | fn on_data_received(&mut self, socket: HydrogenSocket, buf: Vec) { 91 | let mut pong = [0u8; 4]; 92 | pong[0] = 'p' as u8; 93 | pong[1] = 'o' as u8; 94 | pong[2] = 'n' as u8; 95 | pong[3] = 'g' as u8; 96 | 97 | socket.send(&pong[..]); 98 | } 99 | 100 | #[allow(unused_variables)] 101 | fn on_connection_removed(&mut self, fd: RawFd, err: Error) { } 102 | } 103 | 104 | fn main() { 105 | env_logger::init().unwrap(); 106 | hydrogen::begin(Box::new(Server), hydrogen::Config { 107 | addr: "127.0.0.1".to_string(), 108 | port: 1337, 109 | max_threads: 2, 110 | pre_allocated: 100 111 | }); 112 | } 113 | -------------------------------------------------------------------------------- /src/config.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Nathan Sizemore 2 | // 3 | // This Source Code Form is subject to the terms of the 4 | // Mozilla Public License, v. 2.0. If a copy of the MPL was not 5 | // distributed with this file, You can obtain one at 6 | // http://mozilla.org/MPL/2.0/. 7 | 8 | 9 | /// Configuration options for server 10 | pub struct Config { 11 | /// Address to bind to 12 | pub addr: String, 13 | /// Port to bind to 14 | pub port: u16, 15 | /// The number of threads to use for I/O handling. 16 | /// The lib itself makes use of 3 threads. 17 | pub max_threads: usize, 18 | /// The amount of pre-allocated slab space for connections. 19 | /// This should be, roughly, the maximum amount of concurrent 20 | /// connections expected. 21 | pub pre_allocated: usize 22 | } 23 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Nathan Sizemore 2 | // 3 | // This Source Code Form is subject to the terms of the 4 | // Mozilla Public License, v. 2.0. If a copy of the MPL was not 5 | // distributed with this file, You can obtain one at 6 | // http://mozilla.org/MPL/2.0/. 7 | 8 | 9 | //! hydrogen is a non-blocking Edge Triggered TCP socket lib built atop [epoll][epoll-man-page] 10 | //! with performance, concurrency, and scalability as its main priorities. It takes care of the 11 | //! tedious connection and I/O marshalling across threads, and leaves the specifics of I/O reading 12 | //! and writing up the consumer, through trait implementations. 13 | //! 14 | //! # Streams 15 | //! 16 | //! hydrogen manages the state of connections through [`hydrogen::Stream`][stream-trait] 17 | //! [Trait Objects][trait-objects]. 18 | //! 19 | //! # Events 20 | //! 21 | //! hydrogen reports all events to the [`hydrogen::Handler`][handler] passed during creation. 22 | //! Interaction to `hydrogen::Stream` trait objects is made through a simple wrapper, 23 | //! `HydrogenSocket`, to ensure thread safety. 24 | //! 25 | //! # Example Usage 26 | //! 27 | //! The following is a simple snippet using the [simple-stream][simple-stream-repo] crate to 28 | //! provide the non-blocking I/O calls. 29 | //! 30 | //! 31 | //! ```ignore 32 | //! extern crate hydrogen; 33 | //! extern crate simple_stream as ss; 34 | //! 35 | //! use hydrogen; 36 | //! use hydrogen::{Stream as HydrogenStream, HydrogenSocket}; 37 | //! use ss::frame::Frame; 38 | //! use ss::frame::simple::{SimpleFrame, SimpleFrameBuilder}; 39 | //! use ss::{Socket, Plain, NonBlocking, SocketOptions}; 40 | //! 41 | //! 42 | //! #[derive(Clone)] 43 | //! pub struct Stream { 44 | //! inner: Plain 45 | //! } 46 | //! 47 | //! impl HydrogenStream for Stream { 48 | //! fn recv(&mut self) -> Result>, Error> { 49 | //! match self.inner.nb_recv() { 50 | //! Ok(frame_vec) => { 51 | //! let mut ret_buf = Vec::>::with_capacity(frame_vec.len()); 52 | //! for frame in frame_vec.iter() { 53 | //! ret_buf.push(frame.payload()); 54 | //! } 55 | //! Ok(ret_buf) 56 | //! } 57 | //! Err(e) => Err(e) 58 | //! } 59 | //! } 60 | //! 61 | //! fn send(&mut self, buf: &[u8]) -> Result<(), Error> { 62 | //! let frame = SimpleFrame::new(buf); 63 | //! self.inner.nb_send(&frame) 64 | //! } 65 | //! 66 | //! fn shutdown(&mut self) -> Result<(), Error> { 67 | //! self.inner.shutdown() 68 | //! } 69 | //! } 70 | //! impl AsRawFd for Stream { 71 | //! fn as_raw_fd(&self) -> RawFd { self.inner.as_raw_fd() } 72 | //! } 73 | //! 74 | //! 75 | //! struct Server; 76 | //! impl hydrogen::Handler for Server { 77 | //! fn on_server_created(&mut self, fd: RawFd) { 78 | //! 79 | //! } 80 | //! 81 | //! fn on_new_connection(&mut self, fd: RawFd) -> Arc> { 82 | //! 83 | //! } 84 | //! 85 | //! fn on_data_received(&mut self, socket: HydrogenSocket, buffer: Vec) { 86 | //! 87 | //! } 88 | //! 89 | //! fn on_connection_removed(&mut self, fd: RawFd, err: Error) { 90 | //! 91 | //! } 92 | //! } 93 | //! 94 | //! 95 | //! fn main() { 96 | //! hydrogen::begin(Server, hydrogen::Config { 97 | //! addr: "0.0.0.0".to_string(), 98 | //! port: 1337, 99 | //! max_threads: 8, 100 | //! pre_allocated: 100000 101 | //! }); 102 | //! } 103 | //! 104 | //! ``` 105 | //! 106 | //! 107 | //! 108 | //! [epoll-man-page]: http://man7.org/linux/man-pages/man7/epoll.7.html 109 | //! [stream-trait]: https://nathansizemore.github.io/hydrogen/hydrogen/trait.Stream.html 110 | //! [trait-objects]: https://doc.rust-lang.org/book/trait-objects.html 111 | //! [handler]: https://nathansizemore.github.io/hydrogen/hydrogen/trait.Handler.html 112 | //! [simple-stream-repo]: https://github.com/nathansizemore/simple-stream 113 | 114 | 115 | #[macro_use] 116 | extern crate log; 117 | extern crate libc; 118 | extern crate errno; 119 | extern crate threadpool; 120 | extern crate simple_slab; 121 | 122 | 123 | use std::io::Error; 124 | use std::sync::Arc; 125 | use std::cell::UnsafeCell; 126 | use std::os::unix::io::{RawFd, AsRawFd}; 127 | 128 | 129 | pub use config::Config; 130 | pub use types::HydrogenSocket; 131 | 132 | mod types; 133 | mod server; 134 | mod config; 135 | 136 | 137 | /// Trait object responsible for handling reported I/O events. 138 | pub trait Stream : AsRawFd + Send + Sync { 139 | /// Called when epoll reports data is available for read. 140 | /// 141 | /// This method should read until `ErrorKind::WouldBlock` is received. At that time, all 142 | /// complete messages should be returned, otherwise return the std::io::Error. 143 | fn recv(&mut self) -> Result>, Error>; 144 | /// Called as the internal writer for the HydrogenSocket wrapper. 145 | /// 146 | /// This method should write until all bytes have been written or any `std::io::Error` is 147 | /// returned. 148 | fn send(&mut self, buf: &[u8]) -> Result<(), Error>; 149 | /// This method is called when any error, other than `ErrorKind::WouldBlock`, is returned from 150 | /// a `recv` or `send` call. 151 | fn shutdown(&mut self) -> Result<(), Error>; 152 | } 153 | 154 | /// Events reported to lib consumer. 155 | pub trait Handler { 156 | /// This method is called once the listening RawFd has been created. 157 | /// 158 | /// It should be used to set/remove any flags on the underlying RawFd before `listen` is 159 | /// called on the fd. 160 | fn on_server_created(&mut self, fd: RawFd); 161 | /// This method is called whenever `accept` returns a new TCP connection. 162 | /// 163 | /// The returned trait object is added to the connection pool and the epoll interest list. 164 | fn on_new_connection(&mut self, fd: RawFd) -> Arc>; 165 | /// This method is called whenever the `recv` call returns an Ok(_) result. 166 | fn on_data_received(&mut self, socket: HydrogenSocket, buf: Vec); 167 | /// This method is called after a stream has been removed from the connection poll and epoll 168 | /// interest list, with the `std::io::Error` as the reason removed. 169 | /// 170 | /// At the time of this call, the underlying fd has been shutdown and closed. No system level 171 | /// shutdown is needed, only application level cleanup. 172 | fn on_connection_removed(&mut self, fd: RawFd, err: Error); 173 | } 174 | 175 | /// Starts the server with the passed configuration and handler. 176 | pub fn begin(handler: Box, cfg: Config) 177 | where T: Handler + Send + Sync + 'static 178 | { 179 | server::begin(handler, cfg); 180 | } 181 | -------------------------------------------------------------------------------- /src/server.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Nathan Sizemore 2 | // 3 | // This Source Code Form is subject to the terms of the 4 | // Mozilla Public License, v. 2.0. If a copy of the MPL was not 5 | // distributed with this file, You can obtain one at 6 | // http://mozilla.org/MPL/2.0/. 7 | 8 | 9 | use std::{mem, thread}; 10 | use std::error::Error as StdError; 11 | use std::io::{Error, ErrorKind}; 12 | use std::time::Duration; 13 | use std::cell::UnsafeCell; 14 | use std::sync::{Arc, Mutex}; 15 | use std::net::{TcpStream, TcpListener}; 16 | use std::os::unix::io::{RawFd, AsRawFd, IntoRawFd}; 17 | 18 | use libc; 19 | use errno::errno; 20 | use threadpool::ThreadPool; 21 | use simple_slab::Slab; 22 | 23 | use types::*; 24 | use config::Config; 25 | use super::Handler; 26 | 27 | 28 | // When added to epoll, these will be the conditions of kernel notification: 29 | // 30 | // EPOLLIN - Data is available in kernel buffer. 31 | // EPOLLRDHUP - Peer closed connection. 32 | // EPOLLET - Register in EdgeTrigger mode. 33 | // EPOLLONESHOT - After an event is pulled out with epoll_wait(2) the associated 34 | // file descriptor is internally disabled and no other events will 35 | // be reported by the epoll interface. 36 | const DEFAULT_EVENTS: i32 = libc::EPOLLIN | 37 | libc::EPOLLRDHUP | 38 | libc::EPOLLET | 39 | libc::EPOLLONESHOT; 40 | 41 | // Maximum number of events returned from epoll_wait 42 | const MAX_EVENTS: i32 = 100; 43 | 44 | // Useful to keep from passing a copy of a RawFd everywhere 45 | #[allow(non_upper_case_globals)] 46 | static mut epfd: RawFd = 0 as RawFd; 47 | 48 | 49 | pub fn begin(handler: Box, cfg: Config) { 50 | info!("Starting server..."); 51 | 52 | // Wrap handler in something we can share between threads 53 | let event_handler = EventHandler(Box::into_raw(handler)); 54 | 55 | // Create our new connections slab 56 | let new_connection_slab = Arc::new(Mutex::new(Slab::::with_capacity(10))); 57 | 58 | // Create our connection slab 59 | let mut_slab = MutSlab { 60 | inner: UnsafeCell::new(Slab::>::with_capacity(cfg.pre_allocated)) 61 | }; 62 | let connection_slab = Arc::new(mut_slab); 63 | 64 | // Start the event loop 65 | let threads = cfg.max_threads; 66 | let eh_clone = event_handler.clone(); 67 | let new_connections = new_connection_slab.clone(); 68 | unsafe { 69 | thread::Builder::new() 70 | .name("Event Loop".to_string()) 71 | .spawn(move || { 72 | event_loop(new_connections, connection_slab, eh_clone, threads) 73 | }) 74 | .unwrap(); 75 | } 76 | 77 | // Start the TcpListener loop 78 | let eh_clone = event_handler.clone(); 79 | let listener_thread = unsafe { 80 | thread::Builder::new() 81 | .name("TcpListener Loop".to_string()) 82 | .spawn(move || { listener_loop(cfg, new_connection_slab, eh_clone) }) 83 | .unwrap() 84 | }; 85 | let _ = listener_thread.join(); 86 | } 87 | 88 | unsafe fn listener_loop(cfg: Config, new_connections: NewConnectionSlab, handler: EventHandler) { 89 | info!("Starting incoming TCP connection listener..."); 90 | let listener_result = TcpListener::bind((&cfg.addr[..], cfg.port)); 91 | if listener_result.is_err() { 92 | let err = listener_result.unwrap_err(); 93 | error!("Creating TcpListener: {}", err); 94 | panic!(); 95 | } 96 | 97 | let listener = listener_result.unwrap(); 98 | setup_listener_options(&listener, handler.clone()); 99 | 100 | info!("Incoming TCP conecction listener started"); 101 | 102 | for accept_attempt in listener.incoming() { 103 | match accept_attempt { 104 | Ok(tcp_stream) => handle_new_connection(tcp_stream, &new_connections, handler.clone()), 105 | Err(e) => error!("Accepting connection: {}", e) 106 | }; 107 | } 108 | 109 | debug!("Dropping TcpListener"); 110 | 111 | drop(listener); 112 | } 113 | 114 | unsafe fn setup_listener_options(listener: &TcpListener, handler: EventHandler) { 115 | info!("Setting up listener options"); 116 | let fd = listener.as_raw_fd(); 117 | let EventHandler(handler_ptr) = handler; 118 | 119 | (*handler_ptr).on_server_created(fd); 120 | } 121 | 122 | unsafe fn handle_new_connection(tcp_stream: TcpStream, 123 | new_connections: &NewConnectionSlab, 124 | handler: EventHandler) 125 | { 126 | debug!("New connection received"); 127 | // Take ownership of tcp_stream's underlying file descriptor 128 | let fd = tcp_stream.into_raw_fd(); 129 | 130 | // Execute EventHandler's constructor 131 | let EventHandler(handler_ptr) = handler; 132 | let arc_stream = (*handler_ptr).on_new_connection(fd); 133 | 134 | // Create a connection structure 135 | let connection = Connection { 136 | fd: fd, 137 | err_mutex: Mutex::new(None), 138 | tx_mutex: Mutex::new(()), 139 | stream: arc_stream 140 | }; 141 | 142 | // Insert it into the NewConnectionSlab 143 | let mut slab = match (*new_connections).lock() { 144 | Ok(g) => g, 145 | Err(p) => p.into_inner() 146 | }; 147 | 148 | (&mut *slab).insert(connection); 149 | } 150 | 151 | /// Main event loop 152 | unsafe fn event_loop(new_connections: NewConnectionSlab, 153 | connection_slab: ConnectionSlab, 154 | handler: EventHandler, 155 | threads: usize) 156 | { 157 | info!("Event loop starting..."); 158 | const MAX_WAIT: i32 = 1000; // Milliseconds 159 | 160 | info!("Creating epoll instance..."); 161 | // Attempt to create an epoll instance 162 | let result = libc::epoll_create(1); 163 | if result < 0 { 164 | let err = Error::from_raw_os_error(errno().0 as i32); 165 | error!("Creating epoll instance: {}", err); 166 | panic!(); 167 | } 168 | 169 | // Epoll instance 170 | epfd = result; 171 | info!("Epoll instance created with fd: {}", epfd); 172 | 173 | info!("Creating I/O threadpool with {} threads", threads); 174 | 175 | // ThreadPool with user specified number of threads 176 | let thread_pool = ThreadPool::new(threads); 177 | 178 | // Our I/O queue for Connections needing various I/O operations. 179 | let arc_io_queue = Arc::new(Mutex::new(Vec::::with_capacity(MAX_EVENTS as usize))); 180 | 181 | // Start the I/O Sentinel 182 | let t_pool_clone = thread_pool.clone(); 183 | let handler_clone = handler.clone(); 184 | let io_queue = arc_io_queue.clone(); 185 | thread::Builder::new() 186 | .name("I/O Sentinel".to_string()) 187 | .spawn(move || { 188 | io_sentinel(io_queue, t_pool_clone, handler_clone) 189 | }) 190 | .unwrap(); 191 | 192 | // Scratch space for epoll returned events 193 | let mut event_buffer = Vec::::with_capacity(MAX_EVENTS as usize); 194 | event_buffer.set_len(MAX_EVENTS as usize); 195 | 196 | info!("Starting epoll_wait loop..."); 197 | loop { 198 | // Remove any connections in an error'd state. 199 | remove_stale_connections(&connection_slab, &thread_pool, &handler); 200 | 201 | // Insert any newly received connections into the connection_slab 202 | insert_new_connections(&new_connections, &connection_slab); 203 | 204 | // Check for any new events 205 | let result = libc::epoll_wait(epfd, event_buffer.as_mut_ptr(), MAX_EVENTS, MAX_WAIT); 206 | if result < 0 { 207 | let err = Error::from_raw_os_error(errno().0 as i32); 208 | error!("During epoll_wait: {}", err); 209 | panic!(); 210 | } 211 | 212 | let num_events = result as usize; 213 | update_io_events(&connection_slab, &arc_io_queue, &event_buffer[0..num_events]); 214 | } 215 | } 216 | 217 | /// Traverses through the connection slab and creates a list of connections that need dropped, 218 | /// then traverses that list, drops them, and informs the handler of client drop. 219 | unsafe fn remove_stale_connections(connection_slab: &ConnectionSlab, 220 | thread_pool: &ThreadPool, 221 | handler: &EventHandler) 222 | { 223 | let slab_ptr = (*connection_slab).inner.get(); 224 | 225 | let mut x: isize = 0; 226 | while x < (*slab_ptr).len() as isize { 227 | let err_state = { 228 | let state = (*slab_ptr)[x as usize].err_mutex.lock().unwrap(); 229 | if state.is_some() { 230 | let err_kind = (*state).as_ref().unwrap().kind(); 231 | let err_desc = (*state).as_ref().unwrap().description(); 232 | Some(Error::new(err_kind, err_desc)) 233 | } else { 234 | None 235 | } 236 | }; 237 | 238 | err_state.map(|e| { 239 | trace!("Found stale connection"); 240 | 241 | let arc_connection = (*slab_ptr).remove(x as usize); 242 | close_connection(&arc_connection); 243 | 244 | let fd = arc_connection.fd; 245 | let handler_clone = (*handler).clone(); 246 | thread_pool.execute(move || { 247 | let EventHandler(ptr) = handler_clone; 248 | (*ptr).on_connection_removed(fd, e); 249 | }); 250 | x -= 1; 251 | }); 252 | 253 | x += 1; 254 | } 255 | } 256 | 257 | /// Closes the connection's underlying file descriptor 258 | unsafe fn close_connection(connection: &Arc) { 259 | let fd = (*connection).fd; 260 | debug!("Closing fd: {}", fd); 261 | 262 | let result = libc::close(fd); 263 | if result < 0 { 264 | let err = Error::from_raw_os_error(errno().0 as i32); 265 | error!("Closing fd: {} {}", fd, err); 266 | } 267 | } 268 | 269 | /// Transfers Connections from the new_connections slab to the "main" connection_slab. 270 | unsafe fn insert_new_connections(new_connections: &NewConnectionSlab, 271 | connection_slab: &ConnectionSlab) 272 | { 273 | let mut new_slab = match new_connections.lock() { 274 | Ok(g) => g, 275 | Err(p) => p.into_inner() 276 | }; 277 | 278 | let num_connections = (&mut *new_slab).len(); 279 | let arc_main_slab = (*connection_slab).inner.get(); 280 | for _ in 0..num_connections { 281 | let connection = (&mut *new_slab).remove(0); 282 | let arc_connection = Arc::new(connection); 283 | (*arc_main_slab).insert(arc_connection.clone()); 284 | add_connection_to_epoll(&arc_connection); 285 | } 286 | } 287 | 288 | /// Adds a new connection to the epoll interest list. 289 | unsafe fn add_connection_to_epoll(arc_connection: &Arc) { 290 | let fd = (*arc_connection).fd; 291 | debug!("Adding fd {} to epoll", fd); 292 | let result = libc::epoll_ctl(epfd, 293 | libc::EPOLL_CTL_ADD, 294 | fd, 295 | &mut libc::epoll_event { 296 | events: DEFAULT_EVENTS as u32, 297 | u64: fd as u64 298 | }); 299 | 300 | if result < 0 { 301 | let err = Error::from_raw_os_error(errno().0 as i32); 302 | error!("Adding fd: {} to epoll: {}", fd, err); 303 | 304 | let mut err_state = match arc_connection.err_mutex.lock() { 305 | Ok(g) => g, 306 | Err(p) => p.into_inner() 307 | }; 308 | 309 | *err_state = Some(err); 310 | } 311 | } 312 | 313 | /// Re-arms a connection in the epoll interest list with the event mask. 314 | unsafe fn rearm_connection_in_epoll(arc_connection: &Arc, flags: i32) { 315 | let fd = arc_connection.fd; 316 | let events = DEFAULT_EVENTS | flags; 317 | 318 | trace!("EPOLL_CTL_MOD fd: {} flags: {:#b}", fd, (flags as u32)); 319 | 320 | let result = libc::epoll_ctl(epfd, 321 | libc::EPOLL_CTL_MOD, 322 | fd, 323 | &mut libc::epoll_event { events: events as u32, u64: fd as u64 }); 324 | 325 | if result < 0 { 326 | let err = Error::from_raw_os_error(errno().0 as i32); 327 | error!("EPOLL_CTL_MOD fd: {} {}", fd, err); 328 | 329 | let mut err_state = match arc_connection.err_mutex.lock() { 330 | Ok(g) => g, 331 | Err(p) => p.into_inner() 332 | }; 333 | 334 | *err_state = Some(err); 335 | } 336 | } 337 | 338 | /// Traverses the ConnectionSlab and updates any connection's state reported changed by epoll. 339 | unsafe fn update_io_events(connection_slab: &ConnectionSlab, 340 | arc_io_queue: &IoQueue, 341 | events: &[libc::epoll_event]) 342 | { 343 | const READ_EVENT: u32 = libc::EPOLLIN as u32; 344 | const WRITE_EVENT: u32 = libc::EPOLLOUT as u32; 345 | const CLOSE_EVENT: u32 = (libc::EPOLLRDHUP | libc::EPOLLERR | libc::EPOLLHUP) as u32; 346 | 347 | for event in events.iter() { 348 | // Locate the connection this event is for 349 | let fd = event.u64 as RawFd; 350 | 351 | trace!("Epoll event for fd: {} flags: {:#b}", fd, event.events); 352 | 353 | let find_result = find_connection_from_fd(fd, connection_slab); 354 | if find_result.is_err() { 355 | error!("Unable to find fd {} in ConnectionSlab", fd); 356 | continue; 357 | } 358 | 359 | let arc_connection = find_result.unwrap(); 360 | 361 | // Error/hangup occurred? 362 | let close_event = (event.events & CLOSE_EVENT) > 0; 363 | if close_event { 364 | { // Mutex lock 365 | let mut err_opt = match arc_connection.err_mutex.lock() { 366 | Ok(g) => g, 367 | Err(p) => p.into_inner() 368 | }; 369 | 370 | *err_opt = Some(Error::new(ErrorKind::ConnectionAborted, "ConnectionAborted")); 371 | } // Mutex unlock 372 | continue; 373 | } 374 | 375 | // Read or write branch 376 | let read_available = (event.events & READ_EVENT) > 0; 377 | let write_available = (event.events & WRITE_EVENT) > 0; 378 | if !read_available && !write_available { 379 | trace!("Event was neither read nor write: assuming hangup"); 380 | { // Mutex lock 381 | let mut err_opt = match arc_connection.err_mutex.lock() { 382 | Ok(g) => g, 383 | Err(p) => p.into_inner() 384 | }; 385 | 386 | *err_opt = Some(Error::new(ErrorKind::ConnectionAborted, "ConnectionAborted")); 387 | } // Mutex unlock 388 | continue; 389 | } 390 | 391 | let io_event = if read_available && write_available { 392 | trace!("Event: RW"); 393 | IoEvent::ReadWriteAvailable 394 | } else if read_available { 395 | trace!("Event: R"); 396 | IoEvent::ReadAvailable 397 | } else { 398 | trace!("Event W"); 399 | IoEvent::WriteAvailable 400 | }; 401 | 402 | let io_pair = IoPair { 403 | event: io_event, 404 | arc_connection: arc_connection 405 | }; 406 | 407 | trace!("Adding event to queue"); 408 | { // Mutex lock 409 | let mut io_queue = match arc_io_queue.lock() { 410 | Ok(g) => g, 411 | Err(p) => p.into_inner() 412 | }; 413 | 414 | (*io_queue).push(io_pair); 415 | } // Mutex unlock 416 | } 417 | } 418 | 419 | /// Given a fd and ConnectionSlab, returns the Connection associated with fd. 420 | unsafe fn find_connection_from_fd(fd: RawFd, 421 | connection_slab: &ConnectionSlab) 422 | -> Result, ()> 423 | { 424 | let slab_ptr = (*connection_slab).inner.get(); 425 | for ref arc_connection in (*slab_ptr).iter() { 426 | if (*arc_connection).fd == fd { 427 | return Ok((*arc_connection).clone()); 428 | } 429 | } 430 | 431 | Err(()) 432 | } 433 | 434 | unsafe fn io_sentinel(arc_io_queue: IoQueue, thread_pool: ThreadPool, handler: EventHandler) { 435 | info!("Starting I/O Sentinel"); 436 | // We want to wake up with the same interval consitency as the epoll_wait loop. 437 | // Plus a few ms for hopeful non-interference from mutex contention. 438 | let _100ms = 1000000 * 100; 439 | let wait_interval = Duration::new(0, _100ms); 440 | 441 | loop { 442 | thread::sleep(wait_interval); 443 | 444 | let io_queue; 445 | { // Mutex lock 446 | let mut queue = match arc_io_queue.lock() { 447 | Ok(g) => g, 448 | Err(p) => p.into_inner() 449 | }; 450 | 451 | let empty_queue = Vec::::with_capacity(MAX_EVENTS as usize); 452 | io_queue = mem::replace(&mut (*queue), empty_queue); 453 | } // Mutex unlock 454 | 455 | if io_queue.len() > 0 { 456 | trace!("Processing {} I/O events", io_queue.len()); 457 | } 458 | 459 | for ref io_pair in io_queue.iter() { 460 | let io_event = io_pair.event.clone(); 461 | let handler_clone = handler.clone(); 462 | let arc_connection = io_pair.arc_connection.clone(); 463 | thread_pool.execute(move || { 464 | let mut rearm_events = 0i32; 465 | if io_event == IoEvent::WriteAvailable 466 | || io_event == IoEvent::ReadWriteAvailable 467 | { 468 | let flags = handle_write_event(arc_connection.clone()); 469 | if flags == -1 { 470 | return; 471 | } 472 | rearm_events |= flags; 473 | } 474 | if io_event == IoEvent::ReadAvailable 475 | || io_event == IoEvent::ReadWriteAvailable 476 | { 477 | let flags = handle_read_event(arc_connection.clone(), handler_clone); 478 | if flags == -1 { 479 | return; 480 | } 481 | rearm_events |= flags; 482 | } 483 | 484 | rearm_connection_in_epoll(&arc_connection, rearm_events); 485 | }); 486 | } 487 | } 488 | } 489 | 490 | /// Handles an EPOLLOUT event. An empty buffer is sent down the tx line to 491 | /// force whatever was left in the tx_buffer into the kernel's outbound buffer. 492 | unsafe fn handle_write_event(arc_connection: Arc) -> i32 { 493 | debug!("Handling a write backlog event..."); 494 | let err; 495 | { // Mutex lock 496 | let _ = match arc_connection.tx_mutex.lock() { 497 | Ok(g) => g, 498 | Err(p) => p.into_inner() 499 | }; 500 | 501 | // Get a pointer into UnsafeCell 502 | let stream_ptr = arc_connection.stream.get(); 503 | 504 | let empty = Vec::::new(); 505 | let write_result = (*stream_ptr).send(&empty[..]); 506 | if write_result.is_ok() { 507 | debug!("Cleared backlog"); 508 | return 0i32; 509 | } 510 | 511 | err = write_result.unwrap_err(); 512 | if err.kind() == ErrorKind::WouldBlock { 513 | debug!("Backlog still not cleared, returning EPOLLOUT flags for fd"); 514 | return libc::EPOLLOUT; 515 | } 516 | } // Mutex unlock 517 | 518 | { // Mutex lock 519 | let mut err_state = match arc_connection.err_mutex.lock() { 520 | Ok(g) => g, 521 | Err(p) => p.into_inner() 522 | }; 523 | 524 | *err_state = Some(err); 525 | } // Mutex unlock 526 | 527 | return -1i32; 528 | } 529 | 530 | unsafe fn handle_read_event(arc_connection: Arc, handler: EventHandler) -> i32 { 531 | trace!("Handling read event"); 532 | let stream_ptr = arc_connection.stream.get(); 533 | 534 | // Attempt recv 535 | match (*stream_ptr).recv() { 536 | Ok(mut queue) => { 537 | trace!("Read {} msgs", queue.len()); 538 | for msg in queue.drain(..) { 539 | let EventHandler(ptr) = handler; 540 | let hydrogen_socket = HydrogenSocket::new(arc_connection.clone(), 541 | rearm_connection_in_epoll); 542 | (*ptr).on_data_received(hydrogen_socket, msg); 543 | } 544 | return libc::EPOLLIN; 545 | } 546 | 547 | Err(err) => { 548 | let kind = err.kind(); 549 | if kind == ErrorKind::WouldBlock { 550 | trace!("ErrorKind::WouldBlock"); 551 | return libc::EPOLLIN; 552 | } 553 | 554 | if kind != ErrorKind::UnexpectedEof 555 | && kind != ErrorKind::ConnectionReset 556 | && kind != ErrorKind::ConnectionAborted 557 | { 558 | error!("Unexpected during recv: {}", err); 559 | } else { 560 | debug!("Received during read: {}", err); 561 | } 562 | 563 | { // Mutex lock 564 | // If we're in a state of ShouldClose, no need to worry 565 | // about any other operations... 566 | let mut err_state = match arc_connection.err_mutex.lock() { 567 | Ok(g) => g, 568 | Err(p) => p.into_inner() 569 | }; 570 | 571 | *err_state = Some(err); 572 | } // Mutex unlock 573 | } 574 | }; 575 | 576 | return -1i32; 577 | } 578 | -------------------------------------------------------------------------------- /src/types.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Nathan Sizemore 2 | // 3 | // This Source Code Form is subject to the terms of the 4 | // Mozilla Public License, v. 2.0. If a copy of the MPL was not 5 | // distributed with this file, You can obtain one at 6 | // http://mozilla.org/MPL/2.0/. 7 | 8 | 9 | use std::io::{Error, ErrorKind}; 10 | use std::cell::UnsafeCell; 11 | use std::sync::{Arc, Mutex}; 12 | use std::os::unix::io::{RawFd, AsRawFd}; 13 | 14 | use libc; 15 | use simple_slab::Slab; 16 | 17 | use super::{Stream, Handler}; 18 | 19 | 20 | /// Memory region for all concurrent connections. 21 | pub type ConnectionSlab = Arc; 22 | /// Protected memory region for newly accepted connections. 23 | pub type NewConnectionSlab = Arc>>; 24 | /// Queue of Connections needing various I/O operations. 25 | pub type IoQueue = Arc>>; 26 | 27 | #[derive(Clone, PartialEq, Eq)] 28 | pub enum IoEvent { 29 | /// Epoll reported data is available on the socket for reading 30 | ReadAvailable, 31 | /// Epoll reported the socket is writable 32 | WriteAvailable, 33 | /// Epoll reported the socket is both writable and has data available for reading 34 | ReadWriteAvailable 35 | } 36 | 37 | #[derive(Clone)] 38 | pub struct IoPair { 39 | /// The type of I/O needed on this Connection 40 | pub event: IoEvent, 41 | /// The connection `event` is paired with 42 | pub arc_connection: Arc 43 | } 44 | 45 | pub struct Connection { 46 | /// Underlying file descriptor. 47 | pub fd: RawFd, 48 | /// A Some(Error) options means this connection is in 49 | /// an error'd state and should be closed. 50 | pub err_mutex: Mutex>, 51 | /// Mutex to ensure thread safe, ordered writes to our streams. 52 | /// They may have internal buffers 53 | pub tx_mutex: Mutex<()>, 54 | /// Socket (Stream implemented trait-object). 55 | pub stream: Arc> 56 | } 57 | unsafe impl Send for Connection {} 58 | unsafe impl Sync for Connection {} 59 | 60 | 61 | pub struct MutSlab { 62 | pub inner: UnsafeCell>> 63 | } 64 | unsafe impl Send for MutSlab {} 65 | unsafe impl Sync for MutSlab {} 66 | 67 | pub struct EventHandler(pub *mut Handler); 68 | unsafe impl Send for EventHandler {} 69 | unsafe impl Sync for EventHandler {} 70 | impl Clone for EventHandler { 71 | fn clone(&self) -> EventHandler { 72 | let EventHandler(ptr) = *self; 73 | unsafe { 74 | let same_location = &mut *ptr; 75 | EventHandler(same_location) 76 | } 77 | } 78 | } 79 | 80 | /// Thread-safe wrapper for consumer interaction with streams. 81 | pub struct HydrogenSocket { 82 | /// The connection this socket represents 83 | arc_connection: Arc, 84 | /// Function responsible for re-arming fd in epoll instance 85 | rearm_fn: unsafe fn(&Arc, i32) 86 | } 87 | 88 | impl Clone for HydrogenSocket { 89 | fn clone(&self) -> HydrogenSocket { 90 | let fn_ptr = self.rearm_fn; 91 | HydrogenSocket { 92 | arc_connection: self.arc_connection.clone(), 93 | rearm_fn: fn_ptr 94 | } 95 | } 96 | } 97 | 98 | impl HydrogenSocket { 99 | pub fn new(arc_connection: Arc, 100 | rearm_fn: unsafe fn(&Arc, i32)) 101 | -> HydrogenSocket 102 | { 103 | HydrogenSocket { 104 | arc_connection: arc_connection, 105 | rearm_fn: rearm_fn 106 | } 107 | } 108 | 109 | pub fn send(&self, buf: &[u8]) { 110 | let err; 111 | { // Mutex lock 112 | let _ = match self.arc_connection.tx_mutex.lock() { 113 | Ok(g) => g, 114 | Err(p) => p.into_inner() 115 | }; 116 | 117 | let stream_ptr = self.arc_connection.stream.get(); 118 | let write_result = unsafe { 119 | (*stream_ptr).send(buf) 120 | }; 121 | if write_result.is_ok() { 122 | trace!("HydrogenSocket.send OK"); 123 | return; 124 | } 125 | 126 | err = write_result.unwrap_err(); 127 | } // Mutex unlock 128 | 129 | match err.kind() { 130 | ErrorKind::WouldBlock => { 131 | trace!("HydrogenSocket.send received WouldBlock"); 132 | 133 | let execute = self.rearm_fn; 134 | unsafe { 135 | execute(&(self.arc_connection), libc::EPOLLOUT); 136 | } 137 | } 138 | _ => { 139 | trace!("HydrogenSocket.send received err"); 140 | 141 | { // Mutex lock 142 | let mut err_state = match self.arc_connection.err_mutex.lock() { 143 | Ok(g) => g, 144 | Err(p) => p.into_inner() 145 | }; 146 | *err_state = Some(err); 147 | } // Mutex unlock 148 | } 149 | } 150 | } 151 | 152 | pub fn shutdown(&mut self) -> Result<(), Error> { 153 | let stream_ptr = self.arc_connection.stream.get(); 154 | unsafe { 155 | (*stream_ptr).shutdown() 156 | } 157 | } 158 | } 159 | 160 | impl AsRawFd for HydrogenSocket { 161 | fn as_raw_fd(&self) -> RawFd { 162 | self.arc_connection.fd 163 | } 164 | } 165 | --------------------------------------------------------------------------------