├── .github └── workflows │ └── rust.yml ├── .gitignore ├── .travis.yml ├── Cargo.toml ├── LICENSE.md ├── README.md ├── rustfmt.toml └── src ├── errors.rs ├── lib.rs └── proto ├── error.rs ├── mod.rs ├── request.rs └── response.rs /.github/workflows/rust.yml: -------------------------------------------------------------------------------- 1 | name: Rust 2 | 3 | on: 4 | push: 5 | branches: [ "master" ] 6 | pull_request: 7 | branches: [ "master" ] 8 | 9 | env: 10 | CARGO_TERM_COLOR: always 11 | 12 | jobs: 13 | build: 14 | runs-on: ubuntu-latest 15 | 16 | steps: 17 | - name: Install beanstalkd 18 | run: | 19 | sudo apt-get update 20 | sudo apt-get install beanstalkd 21 | - name: Start beanstalkd 22 | run: | 23 | beanstalkd & 24 | - uses: actions/checkout@v3 25 | - uses: Swatinem/rust-cache@v2 26 | - name: Build 27 | run: cargo build --verbose 28 | - name: Run tests 29 | run: cargo test --verbose 30 | clippy: 31 | runs-on: ubuntu-latest 32 | steps: 33 | - uses: actions/checkout@v1 34 | - run: rustup component add clippy 35 | - uses: actions-rs/clippy-check@v1 36 | with: 37 | token: ${{ secrets.GITHUB_TOKEN }} 38 | args: --all-features 39 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | **/*.rs.bk 3 | Cargo.lock 4 | tags -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: rust 2 | dist: trusty 3 | cache: cargo 4 | addons: 5 | apt: 6 | packages: 7 | - beanstalkd 8 | rust: 9 | - stable -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tokio-beanstalkd" 3 | version = "0.5.0-alpha.1" 4 | authors = ["Bhargav Voleti "] 5 | edition = "2021" 6 | 7 | description = "Asynchronous client library for interacting with Beanstalkd work queue." 8 | readme = "README.md" 9 | 10 | homepage = "https://github.com/bIgBV/tokio-beanstalkd" 11 | repository = "https://github.com/bIgBV/tokio-beanstalkd" 12 | 13 | keywords = ["beanstalkd", "tokio", "asynchronous"] 14 | categories = ["api-bindings", "asynchronous", "network-programming"] 15 | 16 | license = "MIT" 17 | 18 | [dependencies] 19 | tokio = { version = "~1", features = ["net"] } 20 | tokio-util = { version = "0.7.10", features = ["codec"] } 21 | tokio-stream = "0.1.14" 22 | futures = "0.3.29" 23 | thiserror = "1.0.50" 24 | tracing = "0.1.40" 25 | 26 | [dev-dependencies] 27 | pretty_assertions = "1.4.0" 28 | tokio = { version = "1", features = ["full"] } 29 | tracing-test = { version = "0.2.4", features = ["no-env-filter"] } 30 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) [year] [fullname] 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # tokio-beanstalkd 2 | 3 | This crate provides a client for working with [Beanstalkd](https://beanstalkd.github.io/), a simple 4 | fast work queue. 5 | 6 | [![Build Status](https://travis-ci.org/bIgBV/tokio-beanstalkd.svg?branch=master)](https://travis-ci.org/bIgBV/tokio-beanstalkd) 7 | [![Crates.io](https://img.shields.io/crates/v/tokio-beanstalkd.svg)](https://crates.io/crates/tokio-beanstalkd) 8 | [![Documentation](https://docs.rs/tokio-beanstalkd/badge.svg)](https://docs.rs/tokio-beanstalkd/) 9 | 10 | This crate provides a client for working with [Beanstalkd](https://beanstalkd.github.io/), a simple 11 | fast work queue. 12 | 13 | # About Beanstalkd 14 | 15 | Beanstalkd is a simple fast work queue. It works at the TCP connection level, considering each TCP 16 | connection individually. A worker may have multiple connections to the Beanstalkd server and each 17 | connection will be considered separate. 18 | 19 | The protocol is ASCII text based but the data itself is just a bytestream. This means that the 20 | application is responsible for interpreting the data. 21 | 22 | ## Operation 23 | This library can serve as a client for both the application and the worker. The application would 24 | [Beanstalkd::put] jobs on the queue and the workers can [Beanstalkd::reserve] 25 | them. Once they are done with the job, they have to [Beanstalkd::delete] job. 26 | This is required for every job, or else Beanstalkd will not remove it fromits internal datastructres. 27 | 28 | If a worker cannot finish the job in it's TTR (Time To Run), then it can [Beanstalkd::release] 29 | the job. The application can use the [Beanstalkd::using] method to put jobs in a specific tube, 30 | and workers can use [Beanstalkd::watch] 31 | 32 | ## Interaction with Tokio 33 | 34 | The futures in this crate expect to be running under a `tokio::Runtime`. In the common case, 35 | you cannot resolve them solely using `.wait()`, but should instead use `tokio::run` or 36 | explicitly create a `tokio::Runtime` and then use `Runtime::block_on`. 37 | 38 | An simple example client could look something like this: 39 | 40 | ```rust 41 | # use tokio_beanstalkd::*; 42 | #[tokio::main] 43 | async fn main() { 44 | let mut bean = Beanstalkd::connect( 45 | &"127.0.0.1:11300" 46 | .parse() 47 | .expect("Unable to connect to Beanstalkd"), 48 | ) 49 | .await 50 | .unwrap(); 51 | 52 | bean.put(0, 1, 100, &b"update:42"[..]).await.unwrap(); 53 | 54 | // Use a particular tube 55 | bean.using("notifications").await.unwrap(); 56 | bean.put(0, 1, 100, &b"notify:100"[..]).await.unwrap(); 57 | } 58 | ``` 59 | 60 | And a worker could look something like this: 61 | ```rust 62 | # use tokio_beanstalkd::*; 63 | #[tokio::main] 64 | async fn main() { 65 | let mut bean = Beanstalkd::connect( 66 | &"127.0.0.1:11300" 67 | .parse() 68 | .expect("Unable to connect to Beanstalkd"), 69 | ) 70 | .await 71 | .unwrap(); 72 | 73 | let response = bean.reserve().await.unwrap(); 74 | // ... do something with the response ... 75 | // Delete the job once it is done 76 | bean.delete(response.id).await.unwrap(); 77 | } 78 | ``` -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | edition = "2021" 2 | -------------------------------------------------------------------------------- /src/errors.rs: -------------------------------------------------------------------------------- 1 | //! The errors returned by the different operations in the library 2 | use thiserror::Error; 3 | 4 | use crate::proto::error::{BeanError, EncodeError, ProtocolError}; 5 | 6 | /// Errors that can be returned for any command 7 | #[derive(Copy, Clone, Debug, Eq, PartialEq, Error)] 8 | #[non_exhaustive] 9 | pub enum BeanstalkError { 10 | /// The client sent a command line that was not well-formed. This can happen if the line does not 11 | /// end with \r\n, if non-numeric characters occur where an integer is expected, if the wrong 12 | /// number of arguments are present, or if the command line is mal-formed in any other way. 13 | /// 14 | /// This should not happen, if it does please file an issue. 15 | #[error("Client command was not well formatted")] 16 | BadFormat, 17 | 18 | /// The server cannot allocate enough memory for the job. The client should try again later. 19 | #[error("Server out of memory")] 20 | OutOfMemory, 21 | 22 | /// This indicates a bug in the server. It should never happen. If it does happen, please report it 23 | /// at 24 | #[error("Internal server error")] 25 | InternalError, 26 | /// The client sent a command that the server does not know. 27 | /// 28 | /// This should not happen, if it does please file an issue. 29 | #[error("Unknown command sent by client")] 30 | UnknownCommand, 31 | 32 | #[error("An unexpected response occurred")] 33 | UnexpectedResponse, 34 | 35 | #[error("Tube name too long")] 36 | TubeNameTooLong, 37 | 38 | #[error("IO Error")] 39 | IoError, 40 | } 41 | 42 | impl From for BeanstalkError { 43 | fn from(error: BeanError) -> Self { 44 | match error { 45 | BeanError::Protocol(ProtocolError::BadFormat) => BeanstalkError::BadFormat, 46 | BeanError::Protocol(ProtocolError::OutOfMemory) => BeanstalkError::OutOfMemory, 47 | BeanError::Protocol(ProtocolError::InternalError) => BeanstalkError::InternalError, 48 | BeanError::Protocol(ProtocolError::UnknownCommand) => BeanstalkError::UnknownCommand, 49 | _ => BeanstalkError::UnexpectedResponse, 50 | } 51 | } 52 | } 53 | 54 | impl From for BeanstalkError { 55 | fn from(error: EncodeError) -> Self { 56 | match error { 57 | EncodeError::TubeNameTooLong => BeanstalkError::TubeNameTooLong, 58 | EncodeError::IoError(_) => BeanstalkError::IoError, 59 | } 60 | } 61 | } 62 | 63 | /// Errors which can be casued due to a PUT command 64 | #[derive(Copy, Clone, Debug, Eq, PartialEq, Error)] 65 | #[non_exhaustive] 66 | pub enum Put { 67 | /// The server ran out of memory trying to grow the priority queue data structure. 68 | /// The client should try another server or disconnect and try again later. 69 | #[error("Server had to bury the request")] 70 | Buried, 71 | 72 | /// The job body must be followed by a CR-LF pair, that is, "\r\n". These two bytes are not counted 73 | /// in the job size given by the client in the put command line. 74 | /// 75 | /// This should never happen, if it does please file an issue. 76 | #[error("CRLF missing from the end of command")] 77 | ExpectedCRLF, 78 | 79 | /// The client has requested to put a job with a body larger than max-job-size bytes 80 | #[error("Job size exceeds max-job-size bytes")] 81 | JobTooBig, 82 | 83 | /// This means that the server has been put into "drain mode" and is no longer accepting new jobs. 84 | /// The client should try another server or disconnect and try again later. 85 | #[error("Server is in drain mode")] 86 | Draining, 87 | 88 | #[error("A protocol error occurred: {}", error)] 89 | Beanstalk { error: BeanstalkError }, 90 | } 91 | 92 | impl From for Put { 93 | fn from(error: BeanError) -> Self { 94 | match error { 95 | BeanError::Protocol(ProtocolError::ExpectedCRLF) => Put::ExpectedCRLF, 96 | BeanError::Protocol(ProtocolError::JobTooBig) => Put::JobTooBig, 97 | BeanError::Protocol(ProtocolError::Draining) => Put::Draining, 98 | _ => Put::Beanstalk { 99 | error: error.into(), 100 | }, 101 | } 102 | } 103 | } 104 | 105 | impl From for Put { 106 | fn from(error: EncodeError) -> Self { 107 | Put::Beanstalk { 108 | error: error.into(), 109 | } 110 | } 111 | } 112 | 113 | /// Errors which can occur when acting as a consumer/worker 114 | #[derive(Copy, Clone, Debug, Eq, PartialEq, Error)] 115 | #[non_exhaustive] 116 | pub enum Consumer { 117 | /// If the job does not exist or is not either reserved by the client 118 | #[error("Did not find a job of that Id")] 119 | NotFound, 120 | /// if the server ran out of memory trying to grow the priority queue data structure. 121 | #[error("Job got buried")] 122 | Buried, 123 | /// If the client attempts to ignore the only tube in its watch list. 124 | #[error("Tried to ignore the only tube being watched")] 125 | NotIgnored, 126 | 127 | #[error("A protocol error occurred: {}", error)] 128 | Beanstalk { error: BeanstalkError }, 129 | } 130 | 131 | impl From for Consumer { 132 | fn from(error: BeanError) -> Self { 133 | match error { 134 | BeanError::Protocol(ProtocolError::NotIgnored) => Consumer::NotIgnored, 135 | _ => Consumer::Beanstalk { 136 | error: error.into(), 137 | }, 138 | } 139 | } 140 | } 141 | 142 | impl From for Consumer { 143 | fn from(error: EncodeError) -> Self { 144 | Consumer::Beanstalk { 145 | error: error.into(), 146 | } 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | //! This crate provides a client for working with [Beanstalkd](https://beanstalkd.github.io/), a simple 2 | //! fast work queue. 3 | // 4 | //! # About Beanstalkd 5 | //! 6 | //! Beanstalkd is a simple fast work queue. It works at the TCP connection level, considering each TCP 7 | //! connection individually. A worker may have multiple connections to the Beanstalkd server and each 8 | //! connection will be considered separate. 9 | //! 10 | //! The protocol is ASCII text based but the data itself is just a bytestream. This means that the 11 | //! application is responsible for interpreting the data. 12 | //! 13 | //! ## Operation 14 | //! This library can serve as a client for both the application and the worker. The application would 15 | //! [Beanstalkd::put] jobs on the queue and the workers can [Beanstalkd::reserve] 16 | //! them. Once they are done with the job, they have to [Beanstalkd::delete] job. 17 | //! This is required for every job, or else Beanstalkd will not remove it fromits internal datastructres. 18 | //! 19 | //! If a worker cannot finish the job in it's TTR (Time To Run), then it can [Beanstalkd::release] 20 | //! the job. The application can use the [Beanstalkd::using] method to put jobs in a specific tube, 21 | //! and workers can use [Beanstalkd::watch] 22 | //! 23 | //! ## Interaction with Tokio 24 | //! 25 | //! The futures in this crate expect to be running under a `tokio::Runtime`. In the common case, 26 | //! you cannot resolve them solely using `.wait()`, but should instead use `tokio::run` or 27 | //! explicitly create a `tokio::Runtime` and then use `Runtime::block_on`. 28 | //! 29 | //! An simple example client could look something like this: 30 | //! 31 | //! ```no_run 32 | //! # use tokio_beanstalkd::*; 33 | //! #[tokio::main] 34 | //! async fn main() { 35 | //! let mut bean = Beanstalkd::connect( 36 | //! &"127.0.0.1:11300" 37 | //! .parse() 38 | //! .expect("Unable to connect to Beanstalkd"), 39 | //! ) 40 | //! .await 41 | //! .unwrap(); 42 | //! 43 | //! bean.put(0, 1, 100, &b"update:42"[..]).await.unwrap(); 44 | //! 45 | //! // Use a particular tube 46 | //! bean.using("notifications").await.unwrap(); 47 | //! bean.put(0, 1, 100, &b"notify:100"[..]).await.unwrap(); 48 | //! } 49 | //! ``` 50 | //! 51 | //! And a worker could look something like this: 52 | //! ```no_run 53 | //! # use tokio_beanstalkd::*; 54 | //! #[tokio::main] 55 | //! async fn main() { 56 | //! let mut bean = Beanstalkd::connect( 57 | //! &"127.0.0.1:11300" 58 | //! .parse() 59 | //! .expect("Unable to connect to Beanstalkd"), 60 | //! ) 61 | //! .await 62 | //! .unwrap(); 63 | //! 64 | //! let Response::Reserved(job) = bean.reserve().await.unwrap() else { 65 | //! panic!("Error reserving job"); 66 | //! }; 67 | //! // ... do something with the response ... 68 | //! // Delete the job once it is done 69 | //! bean.delete(job.id).await.unwrap(); 70 | //! } 71 | //! ``` 72 | 73 | #![warn(rust_2018_idioms)] 74 | 75 | pub mod errors; 76 | mod proto; 77 | 78 | use std::borrow::Cow; 79 | use std::net::SocketAddr; 80 | 81 | use futures::SinkExt; 82 | use tokio_stream::StreamExt; 83 | use tokio_util::codec::Framed; 84 | 85 | pub use crate::proto::response::*; 86 | pub use crate::proto::{Id, Tube}; 87 | // Request doesn't have to be a public type 88 | use crate::proto::Request; 89 | 90 | use crate::errors::{BeanstalkError, Consumer, Put}; 91 | 92 | /// Connection to the Beanstalkd server. 93 | /// 94 | /// All interactions with Beanstalkd happen by calling methods on a `Beanstalkd` instance. 95 | /// 96 | /// Even though there is a `quit` command, Beanstalkd consideres a closed connection as the 97 | /// end of communication, so just dropping this struct will close the connection. 98 | #[derive(Debug)] 99 | pub struct Beanstalkd { 100 | connection: Framed, 101 | } 102 | 103 | pub type Error = Box; 104 | 105 | // FIXME: log out unexpected errors using env_logger 106 | impl Beanstalkd { 107 | /// Connect to a Beanstalkd instance. 108 | /// 109 | /// A successful TCP connect is considered the start of communication. 110 | pub async fn connect(addr: &SocketAddr) -> Result { 111 | let c = tokio::net::TcpStream::connect(addr).await?; 112 | Ok(Beanstalkd::setup(c)) 113 | } 114 | 115 | fn setup(stream: tokio::net::TcpStream) -> Self { 116 | let bean = Framed::new(stream, proto::CommandCodec::new()); 117 | Beanstalkd { connection: bean } 118 | } 119 | 120 | async fn response(&mut self) -> Result { 121 | use proto::error::{BeanError, ProtocolError}; 122 | 123 | match self.connection.next().await { 124 | Some(r) => r, 125 | None => Err(BeanError::Protocol(ProtocolError::StreamClosed)), 126 | } 127 | } 128 | 129 | /// The "put" command is for any process that wants to insert a job into the queue. 130 | /// 131 | /// It inserts a job into the client's currently used tube (see the `use` command 132 | /// below). 133 | /// 134 | /// - `priority` is an integer < 2**32. Jobs with smaller priority values will be 135 | /// scheduled before jobs with larger priorities. The most urgent priority is 0; 136 | /// the least urgent priority is 4,294,967,295. 137 | /// - `delay` is an integer number of seconds to wait before putting the job in 138 | /// the ready queue. The job will be in the "delayed" state during this time. 139 | /// - `ttr` -- time to run -- is an integer number of seconds to allow a worker 140 | /// to run this job. This time is counted from the moment a worker reserves 141 | /// this job. If the worker does not delete, release, or bury the job within 142 | /// `ttr` seconds, the job will time out and the server will release the job. 143 | /// The minimum ttr is 1. If the client sends 0, the server will silently 144 | /// increase the ttr to 1. 145 | /// - `data` is the job body -- a sequence of bytes of length \ from the 146 | /// previous line. 147 | /// 148 | /// After sending the command line and body, the client waits for a reply, which 149 | /// is the integer id of the new job. 150 | pub async fn put( 151 | &mut self, 152 | priority: u32, 153 | delay: u32, 154 | ttr: u32, 155 | data: D, 156 | ) -> Result 157 | where 158 | D: Into>, 159 | { 160 | let data = data.into(); 161 | self.connection 162 | .send(proto::Request::Put { 163 | priority, 164 | delay, 165 | ttr, 166 | data, 167 | }) 168 | .await?; 169 | 170 | self.response().await.map_err(Into::into) 171 | } 172 | 173 | /// Reserve a [proto::response::Job] to process. 174 | /// 175 | /// FIXME: need to handle different responses returned at different TTR vs reserve-with-timeout times 176 | /// 177 | /// A process that wants to consume jobs from the queue uses `reserve`, 178 | /// `[Beanstalkd::delete]`, 179 | /// `[Beanstalkd::release]`, and 180 | /// `[Beanstalkd::bury]`. 181 | pub async fn reserve(&mut self) -> Result { 182 | self.connection.send(proto::Request::Reserve).await?; 183 | 184 | self.response().await.map_err(Into::into) 185 | } 186 | 187 | /// The "use" command is for producers. Subsequent put commands will put jobs into 188 | /// the tube specified by this command. If no use command has been issued, jobs 189 | /// will be put into the tube named "default". 190 | /// 191 | /// - `tube` is a name at most 200 bytes. It specifies the tube to use. If the 192 | /// tube does not exist, it will be created. 193 | pub async fn using(&mut self, tube: &'static str) -> Result { 194 | self.connection.send(Request::Use { tube }).await?; 195 | 196 | self.response().await.map_err(Into::into) 197 | } 198 | 199 | /// The delete command removes a job from the server entirely. It is normally used 200 | /// by the client when the job has successfully run to completion. A client can 201 | /// delete jobs that it has reserved, ready jobs, delayed jobs, and jobs that are 202 | /// buried. 203 | /// 204 | /// - `id` is the job id to delete. 205 | pub async fn delete(&mut self, id: T) -> Result 206 | where 207 | T: Into, 208 | { 209 | self.connection 210 | .send(Request::Delete { id: id.into() }) 211 | .await?; 212 | 213 | self.response().await.map_err(Into::into) 214 | } 215 | 216 | /// The release command puts a reserved job back into the ready queue (and marks 217 | /// its state as "ready") to be run by any client. It is normally used when the job 218 | /// fails because of a transitory error. 219 | /// 220 | /// - `id` is the job id to release. 221 | /// 222 | /// - `pri` is a new priority to assign to the job. 223 | /// 224 | /// - `delay` is an integer number of seconds to wait before putting the job in 225 | /// the ready queue. The job will be in the "delayed" state during this time. 226 | pub async fn release( 227 | &mut self, 228 | id: u32, 229 | priority: u32, 230 | delay: u32, 231 | ) -> Result { 232 | self.connection 233 | .send(Request::Release { 234 | id: id.into(), 235 | priority, 236 | delay, 237 | }) 238 | .await?; 239 | 240 | self.response().await.map_err(Into::into) 241 | } 242 | 243 | /// The "touch" command allows a worker to request more time to work on a job. 244 | /// This is useful for jobs that potentially take a long time, but you still want 245 | /// the benefits of a TTR pulling a job away from an unresponsive worker. A worker 246 | /// may periodically tell the server that it's still alive and processing a job 247 | /// (e.g. it may do this on DEADLINE_SOON). The command postpones the auto 248 | /// release of a reserved job until TTR seconds from when the command is issued. 249 | /// 250 | /// - `id` is the ID of a job reserved by the current connection. 251 | pub async fn touch(&mut self, id: u32) -> Result { 252 | self.connection 253 | .send(Request::Touch { id: id.into() }) 254 | .await?; 255 | 256 | self.response().await.map_err(Into::into) 257 | } 258 | 259 | /// The bury command puts a job into the "buried" state. Buried jobs are put into a 260 | /// FIFO linked list and will not be touched by the server again until a client 261 | /// kicks them with the "kick" command. 262 | /// 263 | /// - `id` is the job id to release. 264 | /// 265 | /// - `prioritiy` is a new priority to assign to the job. 266 | pub async fn bury(&mut self, id: u32, priority: u32) -> Result { 267 | self.connection 268 | .send(Request::Bury { 269 | id: id.into(), 270 | priority, 271 | }) 272 | .await?; 273 | 274 | self.response().await.map_err(Into::into) 275 | } 276 | 277 | /// The "watch" command adds the named tube to the watch list for the current 278 | /// connection. A reserve command will take a job from any of the tubes in the 279 | /// watch list. For each new connection, the watch list initially consists of one 280 | /// tube, named "default". 281 | /// 282 | /// - \ is a name at most 200 bytes. It specifies a tube to add to the watch 283 | /// list. If the tube doesn't exist, it will be created. 284 | /// 285 | /// The value returned is the count of the tubes being watched by the current connection. 286 | pub async fn watch(&mut self, tube: &'static str) -> Result { 287 | self.connection.send(Request::Watch { tube }).await?; 288 | 289 | self.response().await.map_err(Into::into) 290 | } 291 | 292 | /// The "ignore" command is for consumers. It removes the named tube from the 293 | /// watch list for the current connection. 294 | /// 295 | /// - \ is a name at most 200 bytes. It specifies a tube to add to the watch 296 | /// list. If the tube doesn't exist, it will be created. 297 | /// 298 | /// A successful response is: 299 | /// 300 | /// - The count of the number of tubes currently watching 301 | pub async fn ignore(&mut self, tube: &'static str) -> Result { 302 | self.connection.send(Request::Ignore { tube }).await?; 303 | 304 | self.response().await.map_err(Into::into) 305 | } 306 | 307 | /// The peek command lets the client inspect a job in the system. There are four 308 | /// types of jobs as enumerated in [PeekType]. All but the 309 | /// first operate only on the currently used tube. 310 | /// 311 | /// * It takes a [PeekType] representing the type of peek 312 | /// operation to perform 313 | /// 314 | /// * And returns a [proto::response::Job] on success. 315 | pub async fn peek(&mut self, peek_type: PeekType) -> Result { 316 | let request = match peek_type { 317 | PeekType::Ready => Request::PeekReady, 318 | PeekType::Delayed => Request::PeekDelay, 319 | PeekType::Buried => Request::PeekBuried, 320 | PeekType::Normal(id) => Request::Peek { id }, 321 | }; 322 | 323 | self.connection.send(request).await?; 324 | 325 | self.response().await.map_err(Into::into) 326 | } 327 | 328 | /// The kick command applies only to the currently used tube. It moves jobs into 329 | /// the ready queue. If there are any buried jobs, it will only kick buried jobs. 330 | /// Otherwise it will kick delayed jobs. 331 | /// 332 | /// * It takes a `bound` which is the number of jobs it will kick 333 | /// * The response is a u32 representing the number of jobs kicked by the server 334 | pub async fn kick(&mut self, bound: u32) -> Result { 335 | self.connection.send(Request::Kick { bound }).await?; 336 | 337 | self.response().await.map_err(Into::into) 338 | } 339 | 340 | /// The kick-job command is a variant of kick that operates with a single job 341 | /// identified by its job id. If the given job id exists and is in a buried or 342 | /// delayed state, it will be moved to the ready queue of the the same tube where it 343 | /// currently belongs. 344 | /// 345 | /// * It takes an `id` of the job to be kicked 346 | /// * And returns `()` on success 347 | pub async fn kick_job(&mut self, id: Id) -> Result { 348 | self.connection.send(Request::KickJob { id }).await?; 349 | 350 | self.response().await.map_err(Into::into) 351 | } 352 | } 353 | 354 | /// The type of [Beanstalkd::peek] request you want to make 355 | pub enum PeekType { 356 | /// The next ready job 357 | Ready, 358 | 359 | /// The next delayed job 360 | Delayed, 361 | 362 | /// The next bufied job 363 | Buried, 364 | 365 | /// The job with the given Id 366 | Normal(Id), 367 | } 368 | 369 | #[cfg(test)] 370 | mod test { 371 | use super::*; 372 | use pretty_assertions::assert_eq; 373 | use tracing_test::traced_test; 374 | 375 | #[traced_test] 376 | #[tokio::test] 377 | async fn it_works() { 378 | let mut bean = Beanstalkd::connect( 379 | &"127.0.0.1:11300" 380 | .parse() 381 | .expect("Unable to connect to Beanstalkd"), 382 | ) 383 | .await 384 | .unwrap(); 385 | 386 | // Let put a job in 387 | let data = &b"data"[..]; 388 | let Response::Inserted(id1) = bean.put(0, 1, 100, data).await.unwrap() else { 389 | panic!("Unexpected response for put"); 390 | }; 391 | 392 | // how about another one? 393 | bean.put(0, 1, 100, &b"more data"[..]).await.unwrap(); 394 | 395 | let job = bean.reserve().await.unwrap(); 396 | 397 | let Response::Reserved(job) = job else { 398 | panic!("Unexpected response"); 399 | }; 400 | 401 | assert_eq!(job.id, id1); 402 | assert_eq!(&job.data, data); 403 | 404 | // Let's watch a particular tube 405 | let response = bean.using("test").await.unwrap(); 406 | 407 | assert_eq!(response, Response::Using(String::from("test"))); 408 | } 409 | } 410 | -------------------------------------------------------------------------------- /src/proto/error.rs: -------------------------------------------------------------------------------- 1 | //! Error types returned by Beanstalkd 2 | 3 | use std::io; 4 | 5 | use thiserror::Error; 6 | 7 | /// Enum that helps understand what kind of a decoder error occurred. 8 | #[derive(Debug, Error)] 9 | pub(crate) enum BeanError { 10 | #[error("A protocol error occurred")] 11 | Protocol(#[from] ProtocolError), 12 | #[error("A parsing error occurred")] 13 | Parsing(#[from] ParsingError), 14 | 15 | #[error("Io error occurred")] 16 | IoError(#[from] io::Error), 17 | } 18 | 19 | #[derive(Copy, Clone, Eq, PartialEq, Debug, Error)] 20 | pub(crate) enum ParsingError { 21 | /// Represents errors when parsing an Integer value such as a Job ID or the 22 | /// number of tubes being watched 23 | #[error("ID parse error")] 24 | ParseId, 25 | 26 | /// Represents any errors which occur when converting the parsed ASCII string 27 | /// to UTF8. This should not occur as the Beanstalkd protocol only works with 28 | /// ASCII names 29 | #[error("Stringn parse error")] 30 | ParseString, 31 | 32 | /// Error occurred while parsing a number 33 | #[error("Number parse error")] 34 | ParseNumber, 35 | 36 | /// If some unknown error occurred. 37 | #[error("Unknown")] 38 | UnknownResponse, 39 | } 40 | 41 | #[derive(Copy, Clone, Eq, PartialEq, Debug, Error)] 42 | pub(crate) enum ProtocolError { 43 | #[error("BadFormat")] 44 | BadFormat, 45 | #[error("OutOfMemory")] 46 | OutOfMemory, 47 | #[error("InternalError")] 48 | InternalError, 49 | #[error("UnknownCommand")] 50 | UnknownCommand, 51 | #[error("ExpectedCRLF")] 52 | ExpectedCRLF, 53 | #[error("JobTooBig")] 54 | JobTooBig, 55 | #[error("Draining")] 56 | Draining, 57 | #[error("NotIgnored")] 58 | NotIgnored, 59 | #[error("StreamClosed")] 60 | StreamClosed, 61 | } 62 | 63 | /// Custom error type which represents all the various errors which can occur when encoding a 64 | /// request for the server. 65 | #[derive(Debug, Error)] 66 | pub(crate) enum EncodeError { 67 | #[error("Tube name too long")] 68 | TubeNameTooLong, 69 | 70 | #[error("IO error")] 71 | IoError(#[from] io::Error), 72 | } 73 | -------------------------------------------------------------------------------- /src/proto/mod.rs: -------------------------------------------------------------------------------- 1 | use tokio_util::bytes::BytesMut; 2 | use tokio_util::codec::Decoder; 3 | use tokio_util::codec::Encoder; 4 | use tracing::debug; 5 | use tracing::instrument; 6 | use tracing::trace; 7 | 8 | use std::io; 9 | use std::str; 10 | use std::str::FromStr; 11 | 12 | pub(crate) mod error; 13 | mod request; 14 | pub(crate) mod response; 15 | 16 | pub(crate) use self::request::Request; 17 | pub use self::response::*; 18 | 19 | use self::error::{BeanError, EncodeError, ParsingError, ProtocolError}; 20 | use self::response::{Job, PreJob}; 21 | 22 | /// A Tube is a way of separating different types of jobs in Beanstalkd. 23 | /// 24 | /// The clinet can use a particular tube by calling [`using`][using] and Beanstalkd will create a 25 | /// new tube if one does not already exist with that name. Workers can [`watch`][watch] particular 26 | /// tubes and receive jobs only from those tubes. 27 | /// 28 | /// [using]: struct.Beanstalkd.html#method.using 29 | /// [watch]: struct.Beanstalkd.html#method.watch 30 | pub type Tube = String; 31 | 32 | /// The ID of a job assigned by Beanstalkd 33 | #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] 34 | pub struct Id(u32); 35 | 36 | impl From for Id { 37 | fn from(value: u32) -> Self { 38 | Self(value) 39 | } 40 | } 41 | 42 | #[derive(Debug, Clone)] 43 | pub(crate) struct CommandCodec { 44 | /// Prefix of outbox that has been sent 45 | outstart: usize, 46 | } 47 | 48 | impl CommandCodec { 49 | pub(crate) fn new() -> CommandCodec { 50 | CommandCodec { outstart: 0 } 51 | } 52 | 53 | /// Helper method which handles all single word responses 54 | #[instrument] 55 | fn single_word_response(&self, list: &[&str]) -> Result { 56 | let result = match list[0] { 57 | "OUT_OF_MEMORY" => Err(BeanError::Protocol(ProtocolError::OutOfMemory))?, 58 | "INTERNAL_ERROR" => Err(BeanError::Protocol(ProtocolError::InternalError))?, 59 | "BAD_FORMAT" => Err(BeanError::Protocol(ProtocolError::BadFormat))?, 60 | "UNKNOWN_COMMAND" => Err(BeanError::Protocol(ProtocolError::UnknownCommand))?, 61 | "EXPECTED_CRLF" => Err(BeanError::Protocol(ProtocolError::ExpectedCRLF))?, 62 | "JOB_TOO_BIG" => Err(BeanError::Protocol(ProtocolError::JobTooBig))?, 63 | "DRAINING" => Err(BeanError::Protocol(ProtocolError::Draining))?, 64 | "NOT_IGNORED" => Err(BeanError::Protocol(ProtocolError::NotIgnored))?, 65 | "NOT_FOUND" => Ok(Response::NotFound), 66 | "BURIED" => Ok(Response::Buried(None)), 67 | "TOUCHED" => Ok(Response::Touched), 68 | "RELEASED" => Ok(Response::Released), 69 | "DELETED" => Ok(Response::Deleted), 70 | "KICKED" => Ok(Response::JobKicked), 71 | _ => Err(BeanError::Parsing(ParsingError::UnknownResponse))?, 72 | }; 73 | 74 | trace!(?result, "processed single word response"); 75 | result 76 | } 77 | 78 | /// Helper method which handles all two word responses 79 | #[instrument] 80 | fn two_word_response(&self, list: &[&str]) -> Result { 81 | let result = match list[0] { 82 | "INSERTED" => { 83 | let id: u32 = u32::from_str(list[1]) 84 | .map_err(|_| BeanError::Parsing(ParsingError::ParseId))?; 85 | Ok(Response::Inserted(id.into())) 86 | } 87 | "BURIED" => { 88 | let id: u32 = u32::from_str(list[1]) 89 | .map_err(|_| BeanError::Parsing(ParsingError::ParseId))?; 90 | Ok(Response::Buried(Some(id.into()))) 91 | } 92 | "WATCHING" => { 93 | let count = u32::from_str(list[1]) 94 | .map_err(|_| BeanError::Parsing(ParsingError::ParseId))?; 95 | Ok(Response::Watching(count)) 96 | } 97 | "USING" => Ok(Response::Using(String::from(list[1]))), 98 | "KICKED" => { 99 | let count: u32 = u32::from_str(list[1]) 100 | .map_err(|_| BeanError::Parsing(ParsingError::ParseNumber))?; 101 | Ok(Response::Kicked(count)) 102 | } 103 | _ => Err(BeanError::Parsing(ParsingError::UnknownResponse))?, 104 | }; 105 | 106 | trace!(?result, "Parsed two word response"); 107 | result 108 | } 109 | 110 | #[instrument] 111 | fn parse_response(&self, list: &[&str]) -> Result { 112 | debug!("Parsing response"); 113 | if list.len() == 1 { 114 | return self.single_word_response(list); 115 | } 116 | 117 | if list.len() == 2 { 118 | eprintln!("Parsing: {:?}", list[1]); 119 | return self.two_word_response(list); 120 | } 121 | 122 | if list.len() == 3 { 123 | return match list[0] { 124 | "RESERVED" => Ok(Response::Pre(parse_pre_job( 125 | &list[1..], 126 | response::PreResponse::Reserved, 127 | )?)), 128 | "FOUND" => Ok(Response::Pre(parse_pre_job( 129 | &list[1..], 130 | response::PreResponse::Peek, 131 | )?)), 132 | _ => Err(BeanError::Parsing(ParsingError::UnknownResponse))?, 133 | }; 134 | } 135 | 136 | Err(BeanError::Parsing(ParsingError::UnknownResponse))? 137 | } 138 | 139 | #[instrument(skip_all)] 140 | fn parse_job(&mut self, src: &mut BytesMut, pre: &PreJob) -> Result, BeanError> { 141 | if let Some(carriage_offset) = src.iter().position(|b| *b == b'\r') { 142 | if src[carriage_offset + 1] == b'\n' { 143 | let line = utf8(src).map_err(|_| BeanError::Parsing(ParsingError::ParseString))?; 144 | let line: Vec<&str> = line.trim().split(' ').collect(); 145 | 146 | let job = Job { 147 | id: pre.id, 148 | bytes: pre.bytes, 149 | data: line[0].as_bytes().to_vec(), 150 | }; 151 | 152 | trace!(?job); 153 | return Ok(Some(job)); 154 | } 155 | } 156 | self.outstart += src.len(); 157 | Ok(None) 158 | } 159 | 160 | #[instrument(skip(src))] 161 | fn handle_job_response( 162 | &mut self, 163 | response: Response, 164 | src: &mut BytesMut, 165 | ) -> Result, BeanError> { 166 | if let Response::Pre(pre) = response { 167 | if let Some(job) = self.parse_job(src, &pre)? { 168 | self.outstart = 0; 169 | src.clear(); 170 | Ok(Some(pre.to_anyresponse(job))) 171 | } else { 172 | Ok(None) 173 | } 174 | } else { 175 | self.outstart = 0; 176 | src.clear(); 177 | Ok(Some(response)) 178 | } 179 | } 180 | } 181 | 182 | fn parse_pre_job(list: &[&str], response_type: response::PreResponse) -> Result { 183 | let id = u32::from_str(list[0]).map_err(|_| BeanError::Parsing(ParsingError::ParseId))?; 184 | let bytes = usize::from_str(list[1]).map_err(|_| BeanError::Parsing(ParsingError::ParseId))?; 185 | Ok(PreJob { 186 | id: id.into(), 187 | bytes, 188 | response_type, 189 | }) 190 | } 191 | 192 | impl Decoder for CommandCodec { 193 | type Item = Response; 194 | type Error = BeanError; 195 | 196 | fn decode(&mut self, src: &mut BytesMut) -> Result, Self::Error> { 197 | trace!(source = ?src , "Decoding"); 198 | if let Some(carriage_offset) = src[self.outstart..].iter().position(|b| *b == b'\r') { 199 | if src[carriage_offset + 1] == b'\n' { 200 | // Afterwards src contains elements [at, len), and the returned BytesMut 201 | // contains elements [0, at), so + 1 for \r and then +1 for \n 202 | let offset = self.outstart + carriage_offset + 1 + 1; 203 | let line = src.split_to(offset); 204 | let line = 205 | utf8(&line).map_err(|_| BeanError::Parsing(ParsingError::ParseString))?; 206 | let line: Vec<&str> = line.trim().split(' ').collect(); 207 | 208 | let response = self.parse_response(&line[..])?; 209 | debug!(?response, "Got response"); 210 | 211 | // Since the actual job data is on a second line, we need additional parsing 212 | // extract it from the buffer. 213 | return self.handle_job_response(response, src); 214 | } 215 | } 216 | self.outstart = src.len(); 217 | src.clear(); 218 | 219 | Ok(None) 220 | } 221 | } 222 | 223 | impl Encoder for CommandCodec { 224 | type Error = EncodeError; 225 | 226 | #[instrument(skip_all)] 227 | fn encode(&mut self, item: Request, dst: &mut BytesMut) -> Result<(), Self::Error> { 228 | trace!(?item, "Making request"); 229 | match item { 230 | Request::Watch { tube } => { 231 | if tube.as_bytes().len() > 200 { 232 | return Err(EncodeError::TubeNameTooLong); 233 | } 234 | item.serialize(dst) 235 | } 236 | _ => item.serialize(dst), 237 | } 238 | Ok(()) 239 | } 240 | } 241 | 242 | fn utf8(buf: &[u8]) -> Result<&str, io::Error> { 243 | str::from_utf8(buf) 244 | // This should never happen since everything is ascii 245 | .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "Unable to decode input as UTF8")) 246 | } 247 | -------------------------------------------------------------------------------- /src/proto/request.rs: -------------------------------------------------------------------------------- 1 | use std::borrow::Cow; 2 | use std::fmt; 3 | 4 | use tokio_util::bytes::{BufMut, BytesMut}; 5 | 6 | /// A Request holds the data to be serialized on to the wire. 7 | #[derive(Debug)] 8 | pub(crate) enum Request { 9 | Put { 10 | priority: u32, 11 | delay: u32, 12 | ttr: u32, 13 | data: Cow<'static, [u8]>, 14 | }, 15 | Reserve, 16 | Use { 17 | tube: &'static str, 18 | }, 19 | Delete { 20 | id: super::Id, 21 | }, 22 | Release { 23 | id: super::Id, 24 | priority: u32, 25 | delay: u32, 26 | }, 27 | Bury { 28 | id: super::Id, 29 | priority: u32, 30 | }, 31 | Touch { 32 | id: super::Id, 33 | }, 34 | Watch { 35 | tube: &'static str, 36 | }, 37 | Ignore { 38 | tube: &'static str, 39 | }, 40 | Peek { 41 | id: super::Id, 42 | }, 43 | PeekReady, 44 | PeekDelay, 45 | PeekBuried, 46 | Kick { 47 | bound: u32, // Can this and other u32 values be 64 bits? 48 | }, 49 | KickJob { 50 | id: super::Id, 51 | }, 52 | } 53 | 54 | impl Request { 55 | /// The serailize method is used to serialize any request containig data. 56 | /// Since the beanstalkd protocol consists af ASCII characters, the 57 | /// actual heavy lifting is done by the `Display` implementation. 58 | pub(crate) fn serialize(&self, dst: &mut BytesMut) { 59 | let format_string = format!("{}", &self); 60 | dst.reserve(format_string.len()); 61 | dst.put(format_string.as_bytes()); 62 | match *self { 63 | Request::Put { ref data, .. } => { 64 | dst.reserve(data.len()); 65 | dst.put(&data[..]); 66 | dst.reserve(2); 67 | dst.put(&b"\r\n"[..]); 68 | } 69 | Request::Reserve 70 | | Request::PeekReady 71 | | Request::PeekDelay 72 | | Request::PeekBuried 73 | | Request::Use { .. } 74 | | Request::Delete { .. } 75 | | Request::Release { .. } 76 | | Request::Bury { .. } 77 | | Request::Touch { .. } 78 | | Request::Watch { .. } 79 | | Request::Ignore { .. } 80 | | Request::Peek { .. } 81 | | Request::Kick { .. } 82 | | Request::KickJob { .. } => {} 83 | } 84 | } 85 | } 86 | 87 | impl fmt::Display for Request { 88 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 89 | match *self { 90 | Request::Put { 91 | priority, 92 | delay, 93 | ttr, 94 | ref data, 95 | } => write!( 96 | f, 97 | "put {pri} {del} {ttr} {len}\r\n", 98 | pri = priority, 99 | del = delay, 100 | ttr = ttr, 101 | len = data.len(), 102 | ), 103 | Request::Reserve => write!(f, "reserve\r\n"), 104 | Request::Use { tube } => write!(f, "use {}\r\n", tube), 105 | Request::Delete { id } => write!(f, "delete {:?}\r\n", id), 106 | Request::Release { 107 | id, 108 | priority, 109 | delay, 110 | } => write!(f, "release {:?} {} {}\r\n", id, priority, delay), 111 | Request::Bury { id, priority } => write!(f, "bury {:?} {}\r\n", id, priority), 112 | Request::Touch { id } => write!(f, "touch {:?}\r\n", id), 113 | Request::Watch { tube } => write!(f, "watch {}\r\n", tube), 114 | Request::Ignore { tube } => write!(f, "ignore {}\r\n", tube), 115 | Request::Peek { id } => write!(f, "peek {:?}\r\n", id), 116 | Request::PeekReady => write!(f, "peek-ready\r\n"), 117 | Request::PeekDelay => write!(f, "peek-delayed\r\n"), 118 | Request::PeekBuried => write!(f, "peek-buried\r\n"), 119 | Request::Kick { bound } => write!(f, "kick {}\r\n", bound), 120 | Request::KickJob { id } => write!(f, "kick-job {:?}\r\n", id), 121 | } 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /src/proto/response.rs: -------------------------------------------------------------------------------- 1 | //! Response types returned by Beanstalkd 2 | 3 | /// [pre]: [tokio_beanstalkd::proto::response::PreJob] 4 | #[derive(Debug, PartialEq, Eq, Clone, Copy)] 5 | pub enum PreResponse { 6 | /// A response for a reserve request 7 | Reserved, 8 | 9 | /// All types of peek requests have the same response 10 | Peek, 11 | } 12 | 13 | /// This is an internal type which is not returned by tokio-beanstalkd. 14 | /// It is used when parsing the job data returned by Beanstald. 15 | #[derive(Debug, PartialEq, Eq, Clone, Copy)] 16 | pub struct PreJob { 17 | pub id: super::Id, 18 | pub bytes: usize, 19 | pub response_type: PreResponse, 20 | } 21 | 22 | /// A job according to Beanstalkd 23 | #[derive(Debug, PartialEq, Eq)] 24 | pub struct Job { 25 | /// The ID job assigned by Beanstalkd 26 | pub id: super::Id, 27 | 28 | /// The size of the payload 29 | pub bytes: usize, 30 | 31 | /// The payload 32 | pub data: Vec, 33 | } 34 | 35 | pub struct Stats {} 36 | 37 | impl PreJob { 38 | /// Simple method to match a given PreJob to the right Response 39 | pub(crate) fn to_anyresponse(self, job: Job) -> Response { 40 | match self.response_type { 41 | PreResponse::Reserved => Response::Reserved(job), 42 | PreResponse::Peek => Response::Found(job), 43 | } 44 | } 45 | } 46 | 47 | /// All possible responses that the Beanstalkd server can send. 48 | #[derive(Debug, PartialEq, Eq)] 49 | #[non_exhaustive] 50 | pub enum Response { 51 | Reserved(Job), 52 | Inserted(super::Id), 53 | 54 | /// Specifies buried state for different commands 55 | /// 56 | /// - Release: If the server ran out of memory trying to grow the priority queue data structure. 57 | /// - Bury: to indicate success 58 | /// - 59 | Buried(Option), 60 | 61 | /// Response from [`use`][super::Beanstalkd::use] 62 | /// 63 | /// - is the name of the tube now being used. 64 | Using(super::Tube), 65 | 66 | 67 | Deleted, 68 | Watching(u32), 69 | 70 | /// To indicate success for the `release` command. 71 | Released, 72 | 73 | Touched, 74 | Found(Job), 75 | Kicked(u32), 76 | JobKicked, 77 | 78 | /// If the job does not exist or is not reserved by the client 79 | NotFound, 80 | 81 | // Custom type used for reserved job response parsing. 82 | Pre(PreJob), 83 | } 84 | --------------------------------------------------------------------------------