├── .dockerignore ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── cmd └── smoothmq │ ├── server │ └── server.go │ ├── smoothmq.go │ └── tester │ └── tester.go ├── config.yaml ├── config └── config.go ├── dashboard ├── dashboard.go └── views │ ├── delete_queue.html │ ├── layout.html │ ├── message.html │ ├── queue.html │ ├── queue_settings.html │ └── queues.html ├── docs └── queue.gif ├── go.mod ├── go.sum ├── main.go ├── models ├── message.go ├── queue.go ├── stats.go └── tenant.go ├── protocols └── sqs │ ├── errors.go │ ├── models.go │ ├── sigv4.go │ └── sqs.go ├── queue └── sqlite │ └── sqlite.go └── tenants └── defaultmanager └── manager.go /.dockerignore: -------------------------------------------------------------------------------- 1 | # flyctl launch added from .gitignore 2 | # If you prefer the allow list template instead of the deny list, see community template: 3 | # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore 4 | # 5 | # Binaries for programs and plugins 6 | **/*.exe 7 | **/*.exe~ 8 | **/*.dll 9 | **/*.so 10 | **/*.dylib 11 | 12 | # Test binary, built with `go test -c` 13 | **/*.test 14 | 15 | # Output of the go coverage tool, specifically when used with LiteIDE 16 | **/*.out 17 | 18 | # Dependency directories (remove the comment below to include it) 19 | # vendor/ 20 | 21 | # Go workspace file 22 | **/go.work 23 | **/go.work.sum 24 | 25 | # env file 26 | **/.env 27 | 28 | **/scratch 29 | 30 | **/*.sqlite 31 | **/*.db 32 | fly.toml 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # If you prefer the allow list template instead of the deny list, see community template: 2 | # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore 3 | # 4 | # Binaries for programs and plugins 5 | *.exe 6 | *.exe~ 7 | *.dll 8 | *.so 9 | *.dylib 10 | 11 | # Test binary, built with `go test -c` 12 | *.test 13 | 14 | # Output of the go coverage tool, specifically when used with LiteIDE 15 | *.out 16 | 17 | # Dependency directories (remove the comment below to include it) 18 | # vendor/ 19 | 20 | # Go workspace file 21 | go.work 22 | go.work.sum 23 | 24 | # env file 25 | .env 26 | 27 | scratch/ 28 | 29 | *.sqlite 30 | *.sqlite-* 31 | *.db 32 | 33 | smoothmq 34 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG GO_VERSION=1.22 2 | FROM golang:${GO_VERSION}-bookworm as builder 3 | 4 | WORKDIR /usr/src/app 5 | COPY go.mod go.sum ./ 6 | RUN go mod download && go mod verify 7 | COPY . . 8 | RUN go build -v -o /run-app . 9 | 10 | 11 | FROM debian:bookworm 12 | 13 | COPY --from=builder /run-app /usr/local/bin/ 14 | CMD ["run-app", "server"] 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU AFFERO GENERAL PUBLIC LICENSE 2 | Version 3, 19 November 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU Affero General Public License is a free, copyleft license for 11 | software and other kinds of works, specifically designed to ensure 12 | cooperation with the community in the case of network server software. 13 | 14 | The licenses for most software and other practical works are designed 15 | to take away your freedom to share and change the works. By contrast, 16 | our General Public Licenses are intended to guarantee your freedom to 17 | share and change all versions of a program--to make sure it remains free 18 | software for all its users. 19 | 20 | When we speak of free software, we are referring to freedom, not 21 | price. Our General Public Licenses are designed to make sure that you 22 | have the freedom to distribute copies of free software (and charge for 23 | them if you wish), that you receive source code or can get it if you 24 | want it, that you can change the software or use pieces of it in new 25 | free programs, and that you know you can do these things. 26 | 27 | Developers that use our General Public Licenses protect your rights 28 | with two steps: (1) assert copyright on the software, and (2) offer 29 | you this License which gives you legal permission to copy, distribute 30 | and/or modify the software. 31 | 32 | A secondary benefit of defending all users' freedom is that 33 | improvements made in alternate versions of the program, if they 34 | receive widespread use, become available for other developers to 35 | incorporate. Many developers of free software are heartened and 36 | encouraged by the resulting cooperation. However, in the case of 37 | software used on network servers, this result may fail to come about. 38 | The GNU General Public License permits making a modified version and 39 | letting the public access it on a server without ever releasing its 40 | source code to the public. 41 | 42 | The GNU Affero General Public License is designed specifically to 43 | ensure that, in such cases, the modified source code becomes available 44 | to the community. It requires the operator of a network server to 45 | provide the source code of the modified version running there to the 46 | users of that server. Therefore, public use of a modified version, on 47 | a publicly accessible server, gives the public access to the source 48 | code of the modified version. 49 | 50 | An older license, called the Affero General Public License and 51 | published by Affero, was designed to accomplish similar goals. This is 52 | a different license, not a version of the Affero GPL, but Affero has 53 | released a new version of the Affero GPL which permits relicensing under 54 | this license. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | TERMS AND CONDITIONS 60 | 61 | 0. Definitions. 62 | 63 | "This License" refers to version 3 of the GNU Affero General Public License. 64 | 65 | "Copyright" also means copyright-like laws that apply to other kinds of 66 | works, such as semiconductor masks. 67 | 68 | "The Program" refers to any copyrightable work licensed under this 69 | License. Each licensee is addressed as "you". "Licensees" and 70 | "recipients" may be individuals or organizations. 71 | 72 | To "modify" a work means to copy from or adapt all or part of the work 73 | in a fashion requiring copyright permission, other than the making of an 74 | exact copy. The resulting work is called a "modified version" of the 75 | earlier work or a work "based on" the earlier work. 76 | 77 | A "covered work" means either the unmodified Program or a work based 78 | on the Program. 79 | 80 | To "propagate" a work means to do anything with it that, without 81 | permission, would make you directly or secondarily liable for 82 | infringement under applicable copyright law, except executing it on a 83 | computer or modifying a private copy. Propagation includes copying, 84 | distribution (with or without modification), making available to the 85 | public, and in some countries other activities as well. 86 | 87 | To "convey" a work means any kind of propagation that enables other 88 | parties to make or receive copies. Mere interaction with a user through 89 | a computer network, with no transfer of a copy, is not conveying. 90 | 91 | An interactive user interface displays "Appropriate Legal Notices" 92 | to the extent that it includes a convenient and prominently visible 93 | feature that (1) displays an appropriate copyright notice, and (2) 94 | tells the user that there is no warranty for the work (except to the 95 | extent that warranties are provided), that licensees may convey the 96 | work under this License, and how to view a copy of this License. If 97 | the interface presents a list of user commands or options, such as a 98 | menu, a prominent item in the list meets this criterion. 99 | 100 | 1. Source Code. 101 | 102 | The "source code" for a work means the preferred form of the work 103 | for making modifications to it. "Object code" means any non-source 104 | form of a work. 105 | 106 | A "Standard Interface" means an interface that either is an official 107 | standard defined by a recognized standards body, or, in the case of 108 | interfaces specified for a particular programming language, one that 109 | is widely used among developers working in that language. 110 | 111 | The "System Libraries" of an executable work include anything, other 112 | than the work as a whole, that (a) is included in the normal form of 113 | packaging a Major Component, but which is not part of that Major 114 | Component, and (b) serves only to enable use of the work with that 115 | Major Component, or to implement a Standard Interface for which an 116 | implementation is available to the public in source code form. A 117 | "Major Component", in this context, means a major essential component 118 | (kernel, window system, and so on) of the specific operating system 119 | (if any) on which the executable work runs, or a compiler used to 120 | produce the work, or an object code interpreter used to run it. 121 | 122 | The "Corresponding Source" for a work in object code form means all 123 | the source code needed to generate, install, and (for an executable 124 | work) run the object code and to modify the work, including scripts to 125 | control those activities. However, it does not include the work's 126 | System Libraries, or general-purpose tools or generally available free 127 | programs which are used unmodified in performing those activities but 128 | which are not part of the work. For example, Corresponding Source 129 | includes interface definition files associated with source files for 130 | the work, and the source code for shared libraries and dynamically 131 | linked subprograms that the work is specifically designed to require, 132 | such as by intimate data communication or control flow between those 133 | subprograms and other parts of the work. 134 | 135 | The Corresponding Source need not include anything that users 136 | can regenerate automatically from other parts of the Corresponding 137 | Source. 138 | 139 | The Corresponding Source for a work in source code form is that 140 | same work. 141 | 142 | 2. Basic Permissions. 143 | 144 | All rights granted under this License are granted for the term of 145 | copyright on the Program, and are irrevocable provided the stated 146 | conditions are met. This License explicitly affirms your unlimited 147 | permission to run the unmodified Program. The output from running a 148 | covered work is covered by this License only if the output, given its 149 | content, constitutes a covered work. This License acknowledges your 150 | rights of fair use or other equivalent, as provided by copyright law. 151 | 152 | You may make, run and propagate covered works that you do not 153 | convey, without conditions so long as your license otherwise remains 154 | in force. You may convey covered works to others for the sole purpose 155 | of having them make modifications exclusively for you, or provide you 156 | with facilities for running those works, provided that you comply with 157 | the terms of this License in conveying all material for which you do 158 | not control copyright. Those thus making or running the covered works 159 | for you must do so exclusively on your behalf, under your direction 160 | and control, on terms that prohibit them from making any copies of 161 | your copyrighted material outside their relationship with you. 162 | 163 | Conveying under any other circumstances is permitted solely under 164 | the conditions stated below. Sublicensing is not allowed; section 10 165 | makes it unnecessary. 166 | 167 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 168 | 169 | No covered work shall be deemed part of an effective technological 170 | measure under any applicable law fulfilling obligations under article 171 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 172 | similar laws prohibiting or restricting circumvention of such 173 | measures. 174 | 175 | When you convey a covered work, you waive any legal power to forbid 176 | circumvention of technological measures to the extent such circumvention 177 | is effected by exercising rights under this License with respect to 178 | the covered work, and you disclaim any intention to limit operation or 179 | modification of the work as a means of enforcing, against the work's 180 | users, your or third parties' legal rights to forbid circumvention of 181 | technological measures. 182 | 183 | 4. Conveying Verbatim Copies. 184 | 185 | You may convey verbatim copies of the Program's source code as you 186 | receive it, in any medium, provided that you conspicuously and 187 | appropriately publish on each copy an appropriate copyright notice; 188 | keep intact all notices stating that this License and any 189 | non-permissive terms added in accord with section 7 apply to the code; 190 | keep intact all notices of the absence of any warranty; and give all 191 | recipients a copy of this License along with the Program. 192 | 193 | You may charge any price or no price for each copy that you convey, 194 | and you may offer support or warranty protection for a fee. 195 | 196 | 5. Conveying Modified Source Versions. 197 | 198 | You may convey a work based on the Program, or the modifications to 199 | produce it from the Program, in the form of source code under the 200 | terms of section 4, provided that you also meet all of these conditions: 201 | 202 | a) The work must carry prominent notices stating that you modified 203 | it, and giving a relevant date. 204 | 205 | b) The work must carry prominent notices stating that it is 206 | released under this License and any conditions added under section 207 | 7. This requirement modifies the requirement in section 4 to 208 | "keep intact all notices". 209 | 210 | c) You must license the entire work, as a whole, under this 211 | License to anyone who comes into possession of a copy. This 212 | License will therefore apply, along with any applicable section 7 213 | additional terms, to the whole of the work, and all its parts, 214 | regardless of how they are packaged. This License gives no 215 | permission to license the work in any other way, but it does not 216 | invalidate such permission if you have separately received it. 217 | 218 | d) If the work has interactive user interfaces, each must display 219 | Appropriate Legal Notices; however, if the Program has interactive 220 | interfaces that do not display Appropriate Legal Notices, your 221 | work need not make them do so. 222 | 223 | A compilation of a covered work with other separate and independent 224 | works, which are not by their nature extensions of the covered work, 225 | and which are not combined with it such as to form a larger program, 226 | in or on a volume of a storage or distribution medium, is called an 227 | "aggregate" if the compilation and its resulting copyright are not 228 | used to limit the access or legal rights of the compilation's users 229 | beyond what the individual works permit. Inclusion of a covered work 230 | in an aggregate does not cause this License to apply to the other 231 | parts of the aggregate. 232 | 233 | 6. Conveying Non-Source Forms. 234 | 235 | You may convey a covered work in object code form under the terms 236 | of sections 4 and 5, provided that you also convey the 237 | machine-readable Corresponding Source under the terms of this License, 238 | in one of these ways: 239 | 240 | a) Convey the object code in, or embodied in, a physical product 241 | (including a physical distribution medium), accompanied by the 242 | Corresponding Source fixed on a durable physical medium 243 | customarily used for software interchange. 244 | 245 | b) Convey the object code in, or embodied in, a physical product 246 | (including a physical distribution medium), accompanied by a 247 | written offer, valid for at least three years and valid for as 248 | long as you offer spare parts or customer support for that product 249 | model, to give anyone who possesses the object code either (1) a 250 | copy of the Corresponding Source for all the software in the 251 | product that is covered by this License, on a durable physical 252 | medium customarily used for software interchange, for a price no 253 | more than your reasonable cost of physically performing this 254 | conveying of source, or (2) access to copy the 255 | Corresponding Source from a network server at no charge. 256 | 257 | c) Convey individual copies of the object code with a copy of the 258 | written offer to provide the Corresponding Source. This 259 | alternative is allowed only occasionally and noncommercially, and 260 | only if you received the object code with such an offer, in accord 261 | with subsection 6b. 262 | 263 | d) Convey the object code by offering access from a designated 264 | place (gratis or for a charge), and offer equivalent access to the 265 | Corresponding Source in the same way through the same place at no 266 | further charge. You need not require recipients to copy the 267 | Corresponding Source along with the object code. If the place to 268 | copy the object code is a network server, the Corresponding Source 269 | may be on a different server (operated by you or a third party) 270 | that supports equivalent copying facilities, provided you maintain 271 | clear directions next to the object code saying where to find the 272 | Corresponding Source. Regardless of what server hosts the 273 | Corresponding Source, you remain obligated to ensure that it is 274 | available for as long as needed to satisfy these requirements. 275 | 276 | e) Convey the object code using peer-to-peer transmission, provided 277 | you inform other peers where the object code and Corresponding 278 | Source of the work are being offered to the general public at no 279 | charge under subsection 6d. 280 | 281 | A separable portion of the object code, whose source code is excluded 282 | from the Corresponding Source as a System Library, need not be 283 | included in conveying the object code work. 284 | 285 | A "User Product" is either (1) a "consumer product", which means any 286 | tangible personal property which is normally used for personal, family, 287 | or household purposes, or (2) anything designed or sold for incorporation 288 | into a dwelling. In determining whether a product is a consumer product, 289 | doubtful cases shall be resolved in favor of coverage. For a particular 290 | product received by a particular user, "normally used" refers to a 291 | typical or common use of that class of product, regardless of the status 292 | of the particular user or of the way in which the particular user 293 | actually uses, or expects or is expected to use, the product. A product 294 | is a consumer product regardless of whether the product has substantial 295 | commercial, industrial or non-consumer uses, unless such uses represent 296 | the only significant mode of use of the product. 297 | 298 | "Installation Information" for a User Product means any methods, 299 | procedures, authorization keys, or other information required to install 300 | and execute modified versions of a covered work in that User Product from 301 | a modified version of its Corresponding Source. The information must 302 | suffice to ensure that the continued functioning of the modified object 303 | code is in no case prevented or interfered with solely because 304 | modification has been made. 305 | 306 | If you convey an object code work under this section in, or with, or 307 | specifically for use in, a User Product, and the conveying occurs as 308 | part of a transaction in which the right of possession and use of the 309 | User Product is transferred to the recipient in perpetuity or for a 310 | fixed term (regardless of how the transaction is characterized), the 311 | Corresponding Source conveyed under this section must be accompanied 312 | by the Installation Information. But this requirement does not apply 313 | if neither you nor any third party retains the ability to install 314 | modified object code on the User Product (for example, the work has 315 | been installed in ROM). 316 | 317 | The requirement to provide Installation Information does not include a 318 | requirement to continue to provide support service, warranty, or updates 319 | for a work that has been modified or installed by the recipient, or for 320 | the User Product in which it has been modified or installed. Access to a 321 | network may be denied when the modification itself materially and 322 | adversely affects the operation of the network or violates the rules and 323 | protocols for communication across the network. 324 | 325 | Corresponding Source conveyed, and Installation Information provided, 326 | in accord with this section must be in a format that is publicly 327 | documented (and with an implementation available to the public in 328 | source code form), and must require no special password or key for 329 | unpacking, reading or copying. 330 | 331 | 7. Additional Terms. 332 | 333 | "Additional permissions" are terms that supplement the terms of this 334 | License by making exceptions from one or more of its conditions. 335 | Additional permissions that are applicable to the entire Program shall 336 | be treated as though they were included in this License, to the extent 337 | that they are valid under applicable law. If additional permissions 338 | apply only to part of the Program, that part may be used separately 339 | under those permissions, but the entire Program remains governed by 340 | this License without regard to the additional permissions. 341 | 342 | When you convey a copy of a covered work, you may at your option 343 | remove any additional permissions from that copy, or from any part of 344 | it. (Additional permissions may be written to require their own 345 | removal in certain cases when you modify the work.) You may place 346 | additional permissions on material, added by you to a covered work, 347 | for which you have or can give appropriate copyright permission. 348 | 349 | Notwithstanding any other provision of this License, for material you 350 | add to a covered work, you may (if authorized by the copyright holders of 351 | that material) supplement the terms of this License with terms: 352 | 353 | a) Disclaiming warranty or limiting liability differently from the 354 | terms of sections 15 and 16 of this License; or 355 | 356 | b) Requiring preservation of specified reasonable legal notices or 357 | author attributions in that material or in the Appropriate Legal 358 | Notices displayed by works containing it; or 359 | 360 | c) Prohibiting misrepresentation of the origin of that material, or 361 | requiring that modified versions of such material be marked in 362 | reasonable ways as different from the original version; or 363 | 364 | d) Limiting the use for publicity purposes of names of licensors or 365 | authors of the material; or 366 | 367 | e) Declining to grant rights under trademark law for use of some 368 | trade names, trademarks, or service marks; or 369 | 370 | f) Requiring indemnification of licensors and authors of that 371 | material by anyone who conveys the material (or modified versions of 372 | it) with contractual assumptions of liability to the recipient, for 373 | any liability that these contractual assumptions directly impose on 374 | those licensors and authors. 375 | 376 | All other non-permissive additional terms are considered "further 377 | restrictions" within the meaning of section 10. If the Program as you 378 | received it, or any part of it, contains a notice stating that it is 379 | governed by this License along with a term that is a further 380 | restriction, you may remove that term. If a license document contains 381 | a further restriction but permits relicensing or conveying under this 382 | License, you may add to a covered work material governed by the terms 383 | of that license document, provided that the further restriction does 384 | not survive such relicensing or conveying. 385 | 386 | If you add terms to a covered work in accord with this section, you 387 | must place, in the relevant source files, a statement of the 388 | additional terms that apply to those files, or a notice indicating 389 | where to find the applicable terms. 390 | 391 | Additional terms, permissive or non-permissive, may be stated in the 392 | form of a separately written license, or stated as exceptions; 393 | the above requirements apply either way. 394 | 395 | 8. Termination. 396 | 397 | You may not propagate or modify a covered work except as expressly 398 | provided under this License. Any attempt otherwise to propagate or 399 | modify it is void, and will automatically terminate your rights under 400 | this License (including any patent licenses granted under the third 401 | paragraph of section 11). 402 | 403 | However, if you cease all violation of this License, then your 404 | license from a particular copyright holder is reinstated (a) 405 | provisionally, unless and until the copyright holder explicitly and 406 | finally terminates your license, and (b) permanently, if the copyright 407 | holder fails to notify you of the violation by some reasonable means 408 | prior to 60 days after the cessation. 409 | 410 | Moreover, your license from a particular copyright holder is 411 | reinstated permanently if the copyright holder notifies you of the 412 | violation by some reasonable means, this is the first time you have 413 | received notice of violation of this License (for any work) from that 414 | copyright holder, and you cure the violation prior to 30 days after 415 | your receipt of the notice. 416 | 417 | Termination of your rights under this section does not terminate the 418 | licenses of parties who have received copies or rights from you under 419 | this License. If your rights have been terminated and not permanently 420 | reinstated, you do not qualify to receive new licenses for the same 421 | material under section 10. 422 | 423 | 9. Acceptance Not Required for Having Copies. 424 | 425 | You are not required to accept this License in order to receive or 426 | run a copy of the Program. Ancillary propagation of a covered work 427 | occurring solely as a consequence of using peer-to-peer transmission 428 | to receive a copy likewise does not require acceptance. However, 429 | nothing other than this License grants you permission to propagate or 430 | modify any covered work. These actions infringe copyright if you do 431 | not accept this License. Therefore, by modifying or propagating a 432 | covered work, you indicate your acceptance of this License to do so. 433 | 434 | 10. Automatic Licensing of Downstream Recipients. 435 | 436 | Each time you convey a covered work, the recipient automatically 437 | receives a license from the original licensors, to run, modify and 438 | propagate that work, subject to this License. You are not responsible 439 | for enforcing compliance by third parties with this License. 440 | 441 | An "entity transaction" is a transaction transferring control of an 442 | organization, or substantially all assets of one, or subdividing an 443 | organization, or merging organizations. If propagation of a covered 444 | work results from an entity transaction, each party to that 445 | transaction who receives a copy of the work also receives whatever 446 | licenses to the work the party's predecessor in interest had or could 447 | give under the previous paragraph, plus a right to possession of the 448 | Corresponding Source of the work from the predecessor in interest, if 449 | the predecessor has it or can get it with reasonable efforts. 450 | 451 | You may not impose any further restrictions on the exercise of the 452 | rights granted or affirmed under this License. For example, you may 453 | not impose a license fee, royalty, or other charge for exercise of 454 | rights granted under this License, and you may not initiate litigation 455 | (including a cross-claim or counterclaim in a lawsuit) alleging that 456 | any patent claim is infringed by making, using, selling, offering for 457 | sale, or importing the Program or any portion of it. 458 | 459 | 11. Patents. 460 | 461 | A "contributor" is a copyright holder who authorizes use under this 462 | License of the Program or a work on which the Program is based. The 463 | work thus licensed is called the contributor's "contributor version". 464 | 465 | A contributor's "essential patent claims" are all patent claims 466 | owned or controlled by the contributor, whether already acquired or 467 | hereafter acquired, that would be infringed by some manner, permitted 468 | by this License, of making, using, or selling its contributor version, 469 | but do not include claims that would be infringed only as a 470 | consequence of further modification of the contributor version. For 471 | purposes of this definition, "control" includes the right to grant 472 | patent sublicenses in a manner consistent with the requirements of 473 | this License. 474 | 475 | Each contributor grants you a non-exclusive, worldwide, royalty-free 476 | patent license under the contributor's essential patent claims, to 477 | make, use, sell, offer for sale, import and otherwise run, modify and 478 | propagate the contents of its contributor version. 479 | 480 | In the following three paragraphs, a "patent license" is any express 481 | agreement or commitment, however denominated, not to enforce a patent 482 | (such as an express permission to practice a patent or covenant not to 483 | sue for patent infringement). To "grant" such a patent license to a 484 | party means to make such an agreement or commitment not to enforce a 485 | patent against the party. 486 | 487 | If you convey a covered work, knowingly relying on a patent license, 488 | and the Corresponding Source of the work is not available for anyone 489 | to copy, free of charge and under the terms of this License, through a 490 | publicly available network server or other readily accessible means, 491 | then you must either (1) cause the Corresponding Source to be so 492 | available, or (2) arrange to deprive yourself of the benefit of the 493 | patent license for this particular work, or (3) arrange, in a manner 494 | consistent with the requirements of this License, to extend the patent 495 | license to downstream recipients. "Knowingly relying" means you have 496 | actual knowledge that, but for the patent license, your conveying the 497 | covered work in a country, or your recipient's use of the covered work 498 | in a country, would infringe one or more identifiable patents in that 499 | country that you have reason to believe are valid. 500 | 501 | If, pursuant to or in connection with a single transaction or 502 | arrangement, you convey, or propagate by procuring conveyance of, a 503 | covered work, and grant a patent license to some of the parties 504 | receiving the covered work authorizing them to use, propagate, modify 505 | or convey a specific copy of the covered work, then the patent license 506 | you grant is automatically extended to all recipients of the covered 507 | work and works based on it. 508 | 509 | A patent license is "discriminatory" if it does not include within 510 | the scope of its coverage, prohibits the exercise of, or is 511 | conditioned on the non-exercise of one or more of the rights that are 512 | specifically granted under this License. You may not convey a covered 513 | work if you are a party to an arrangement with a third party that is 514 | in the business of distributing software, under which you make payment 515 | to the third party based on the extent of your activity of conveying 516 | the work, and under which the third party grants, to any of the 517 | parties who would receive the covered work from you, a discriminatory 518 | patent license (a) in connection with copies of the covered work 519 | conveyed by you (or copies made from those copies), or (b) primarily 520 | for and in connection with specific products or compilations that 521 | contain the covered work, unless you entered into that arrangement, 522 | or that patent license was granted, prior to 28 March 2007. 523 | 524 | Nothing in this License shall be construed as excluding or limiting 525 | any implied license or other defenses to infringement that may 526 | otherwise be available to you under applicable patent law. 527 | 528 | 12. No Surrender of Others' Freedom. 529 | 530 | If conditions are imposed on you (whether by court order, agreement or 531 | otherwise) that contradict the conditions of this License, they do not 532 | excuse you from the conditions of this License. If you cannot convey a 533 | covered work so as to satisfy simultaneously your obligations under this 534 | License and any other pertinent obligations, then as a consequence you may 535 | not convey it at all. For example, if you agree to terms that obligate you 536 | to collect a royalty for further conveying from those to whom you convey 537 | the Program, the only way you could satisfy both those terms and this 538 | License would be to refrain entirely from conveying the Program. 539 | 540 | 13. Remote Network Interaction; Use with the GNU General Public License. 541 | 542 | Notwithstanding any other provision of this License, if you modify the 543 | Program, your modified version must prominently offer all users 544 | interacting with it remotely through a computer network (if your version 545 | supports such interaction) an opportunity to receive the Corresponding 546 | Source of your version by providing access to the Corresponding Source 547 | from a network server at no charge, through some standard or customary 548 | means of facilitating copying of software. This Corresponding Source 549 | shall include the Corresponding Source for any work covered by version 3 550 | of the GNU General Public License that is incorporated pursuant to the 551 | following paragraph. 552 | 553 | Notwithstanding any other provision of this License, you have 554 | permission to link or combine any covered work with a work licensed 555 | under version 3 of the GNU General Public License into a single 556 | combined work, and to convey the resulting work. The terms of this 557 | License will continue to apply to the part which is the covered work, 558 | but the work with which it is combined will remain governed by version 559 | 3 of the GNU General Public License. 560 | 561 | 14. Revised Versions of this License. 562 | 563 | The Free Software Foundation may publish revised and/or new versions of 564 | the GNU Affero General Public License from time to time. Such new versions 565 | will be similar in spirit to the present version, but may differ in detail to 566 | address new problems or concerns. 567 | 568 | Each version is given a distinguishing version number. If the 569 | Program specifies that a certain numbered version of the GNU Affero General 570 | Public License "or any later version" applies to it, you have the 571 | option of following the terms and conditions either of that numbered 572 | version or of any later version published by the Free Software 573 | Foundation. If the Program does not specify a version number of the 574 | GNU Affero General Public License, you may choose any version ever published 575 | by the Free Software Foundation. 576 | 577 | If the Program specifies that a proxy can decide which future 578 | versions of the GNU Affero General Public License can be used, that proxy's 579 | public statement of acceptance of a version permanently authorizes you 580 | to choose that version for the Program. 581 | 582 | Later license versions may give you additional or different 583 | permissions. However, no additional obligations are imposed on any 584 | author or copyright holder as a result of your choosing to follow a 585 | later version. 586 | 587 | 15. Disclaimer of Warranty. 588 | 589 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 590 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 591 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 592 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 593 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 594 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 595 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 596 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 597 | 598 | 16. Limitation of Liability. 599 | 600 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 601 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 602 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 603 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 604 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 605 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 606 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 607 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 608 | SUCH DAMAGES. 609 | 610 | 17. Interpretation of Sections 15 and 16. 611 | 612 | If the disclaimer of warranty and limitation of liability provided 613 | above cannot be given local legal effect according to their terms, 614 | reviewing courts shall apply local law that most closely approximates 615 | an absolute waiver of all civil liability in connection with the 616 | Program, unless a warranty or assumption of liability accompanies a 617 | copy of the Program in return for a fee. 618 | 619 | END OF TERMS AND CONDITIONS 620 | 621 | How to Apply These Terms to Your New Programs 622 | 623 | If you develop a new program, and you want it to be of the greatest 624 | possible use to the public, the best way to achieve this is to make it 625 | free software which everyone can redistribute and change under these terms. 626 | 627 | To do so, attach the following notices to the program. It is safest 628 | to attach them to the start of each source file to most effectively 629 | state the exclusion of warranty; and each file should have at least 630 | the "copyright" line and a pointer to where the full notice is found. 631 | 632 | 633 | Copyright (C) 634 | 635 | This program is free software: you can redistribute it and/or modify 636 | it under the terms of the GNU Affero General Public License as published 637 | by the Free Software Foundation, either version 3 of the License, or 638 | (at your option) any later version. 639 | 640 | This program is distributed in the hope that it will be useful, 641 | but WITHOUT ANY WARRANTY; without even the implied warranty of 642 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 643 | GNU Affero General Public License for more details. 644 | 645 | You should have received a copy of the GNU Affero General Public License 646 | along with this program. If not, see . 647 | 648 | Also add information on how to contact you by electronic and paper mail. 649 | 650 | If your software can interact with users remotely through a computer 651 | network, you should also make sure that it provides a way for users to 652 | get its source. For example, if your program is a web application, its 653 | interface could display a "Source" link that leads users to an archive 654 | of the code. There are many ways you could offer source, and different 655 | solutions will be better for different programs; see section 13 for the 656 | specific requirements. 657 | 658 | You should also get your employer (if you work as a programmer) or school, 659 | if any, to sign a "copyright disclaimer" for the program, if necessary. 660 | For more information on this, and how to apply and follow the GNU AGPL, see 661 | . -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SmoothMQ 2 | 3 | SmoothMQ is a drop-in replacement for SQS with a much smoother developer experience. 4 | It has a functional UI, observability, tracing, message scheduling, and rate-limiting. 5 | SmoothMQ lets you run a private SQS instance on any cloud. 6 | 7 | ## Survey! 8 | I'd love your feedback on the direction of this project! https://forms.gle/m5iMjcA5Xvp685Yw8 9 | 10 | 11 | 12 | 13 | ## Getting Started 14 | 15 | SmoothMQ deploys as a single go binary and can be used by any existing SQS client. 16 | 17 | ## Running 18 | 19 | This will run a UI on `:3000` and an SQS-compatible server on `:3001`. 20 | 21 | ``` 22 | $ go run . server 23 | ``` 24 | 25 | ## Connecting 26 | 27 | This works with any SQS client in any language. 28 | 29 | ### Python 30 | 31 | ``` py 32 | import boto3 33 | 34 | # Simply change the endpoint_url 35 | sqs = boto3.client("sqs", ..., endpoint_url="http://localhost:3001") 36 | sqs.send_message(QueueUrl="...", MessageBody="hello world") 37 | ``` 38 | 39 | Celery works seamlessly: 40 | 41 | ``` py 42 | app = Celery("tasks", broker_url="sqs://...@localhost:3001") 43 | ``` 44 | 45 | ## UI 46 | 47 | The UI lets you manage queues and search individual messages. 48 | 49 | ![Dashboard UI](docs/queue.gif) 50 | -------------------------------------------------------------------------------- /cmd/smoothmq/server/server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "net/http" 7 | "os" 8 | "os/signal" 9 | "syscall" 10 | "time" 11 | 12 | "github.com/gofiber/fiber/v2" 13 | "github.com/gofiber/fiber/v2/middleware/adaptor" 14 | "github.com/poundifdef/smoothmq/config" 15 | "github.com/poundifdef/smoothmq/dashboard" 16 | "github.com/poundifdef/smoothmq/models" 17 | "github.com/poundifdef/smoothmq/protocols/sqs" 18 | "github.com/poundifdef/smoothmq/queue/sqlite" 19 | "github.com/poundifdef/smoothmq/tenants/defaultmanager" 20 | "github.com/prometheus/client_golang/prometheus/promhttp" 21 | ) 22 | 23 | func recordTelemetry(message string, disabled bool) { 24 | if disabled { 25 | return 26 | } 27 | 28 | url := "https://telemetry.fly.dev" 29 | jsonData := []byte(message) 30 | 31 | client := &http.Client{ 32 | Timeout: 100 * time.Millisecond, 33 | } 34 | 35 | req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) 36 | if err != nil { 37 | return 38 | } 39 | 40 | resp, err := client.Do(req) 41 | if err != nil { 42 | return 43 | } 44 | 45 | resp.Body.Close() 46 | } 47 | 48 | func Run(tm models.TenantManager, queue models.Queue, cfg config.ServerCommand) { 49 | recordTelemetry("start", cfg.DisableTelemetry) 50 | 51 | // Initialize default tenant manager 52 | if tm == nil { 53 | tm = defaultmanager.NewDefaultTenantManager(cfg.SQS.Keys) 54 | } 55 | 56 | // Initialize default queue implementation 57 | if queue == nil { 58 | queue = sqlite.NewSQLiteQueue(cfg.SQLite) 59 | } 60 | 61 | dashboardServer := dashboard.NewDashboard(queue, tm, cfg.Dashboard) 62 | sqsServer := sqs.NewSQS(queue, tm, cfg.SQS) 63 | 64 | c := make(chan os.Signal, 1) 65 | signal.Notify(c, os.Interrupt, syscall.SIGTERM) 66 | 67 | if !cfg.UseSinglePort { 68 | go func() { 69 | dashboardServer.Start() 70 | }() 71 | 72 | go func() { 73 | sqsServer.Start() 74 | }() 75 | 76 | if cfg.Metrics.PrometheusEnabled { 77 | fmt.Printf("Prometheus metrics: http://%s:%d%s\n", cfg.Metrics.PrometheusHost, cfg.Metrics.PrometheusPort, cfg.Metrics.PrometheusPath) 78 | go func() { 79 | http.Handle(cfg.Metrics.PrometheusPath, promhttp.Handler()) 80 | http.ListenAndServe(fmt.Sprintf("%s:%d", cfg.Metrics.PrometheusHost, cfg.Metrics.PrometheusPort), nil) 81 | }() 82 | } 83 | 84 | <-c // This blocks the main thread until an interrupt is received 85 | fmt.Println("Gracefully shutting down...") 86 | 87 | dashboardServer.Stop() 88 | sqsServer.Stop() 89 | } else { 90 | app := fiber.New(fiber.Config{ 91 | DisableStartupMessage: true, 92 | }) 93 | 94 | if cfg.Dashboard.Enabled { 95 | app.Mount("/", dashboardServer.App) 96 | fmt.Printf("Dashboard http://%s:%d\n", cfg.Host, cfg.Port) 97 | } 98 | 99 | if cfg.SQS.Enabled { 100 | app.Mount("/sqs", sqsServer.App) 101 | fmt.Printf("SQS Endpoint http://%s:%d/sqs\n", cfg.Host, cfg.Port) 102 | } 103 | 104 | if cfg.Metrics.PrometheusEnabled { 105 | app.Group("/metrics", adaptor.HTTPHandler(promhttp.Handler())) 106 | fmt.Printf("Prometheus http://%s:%d/metrics\n", cfg.Host, cfg.Port) 107 | } 108 | 109 | go func() { 110 | app.Listen(fmt.Sprintf("%s:%d", cfg.Host, cfg.Port)) 111 | }() 112 | 113 | <-c // This blocks the main thread until an interrupt is received 114 | fmt.Println("Gracefully shutting down...") 115 | 116 | app.Shutdown() 117 | } 118 | 119 | queue.Shutdown() 120 | recordTelemetry("stop", cfg.DisableTelemetry) 121 | } 122 | -------------------------------------------------------------------------------- /cmd/smoothmq/smoothmq.go: -------------------------------------------------------------------------------- 1 | package smoothmq 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "strconv" 7 | 8 | "github.com/poundifdef/smoothmq/cmd/smoothmq/server" 9 | "github.com/poundifdef/smoothmq/cmd/smoothmq/tester" 10 | "github.com/poundifdef/smoothmq/config" 11 | "github.com/poundifdef/smoothmq/models" 12 | "github.com/rs/zerolog" 13 | "github.com/rs/zerolog/log" 14 | ) 15 | 16 | func Run(command string, cfg *config.CLI, tenantManager models.TenantManager, queue models.Queue) { 17 | zerolog.CallerMarshalFunc = func(pc uintptr, file string, line int) string { 18 | return filepath.Base(file) + ":" + strconv.Itoa(line) 19 | } 20 | 21 | // TODO: read log level and format from config 22 | logLevel, _ := zerolog.ParseLevel(cfg.Log.Level) 23 | log.Logger = zerolog.New(os.Stderr).With().Timestamp().Caller().Logger().Level(logLevel) 24 | if cfg.Log.Pretty { 25 | log.Logger = log.Logger.Output(zerolog.ConsoleWriter{Out: os.Stderr}) 26 | } 27 | 28 | switch command { 29 | case "tester": 30 | tester.Run(cfg.Tester) 31 | default: 32 | server.Run(tenantManager, queue, cfg.Server) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /cmd/smoothmq/tester/tester.go: -------------------------------------------------------------------------------- 1 | package tester 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/rs/zerolog/log" 8 | 9 | "math/rand" 10 | "os" 11 | "sync" 12 | "time" 13 | 14 | smoothCfg "github.com/poundifdef/smoothmq/config" 15 | 16 | "github.com/aws/aws-sdk-go-v2/aws" 17 | "github.com/aws/aws-sdk-go-v2/config" 18 | "github.com/aws/aws-sdk-go-v2/credentials" 19 | "github.com/aws/aws-sdk-go-v2/service/sqs" 20 | "github.com/aws/aws-sdk-go-v2/service/sqs/types" 21 | ) 22 | 23 | func Run(c smoothCfg.TesterCommand) { 24 | var sentMessages, receivedMessages int 25 | 26 | queueUrl := "https://sqs.us-east-1.amazonaws.com/123/test-queue" 27 | 28 | // Load the AWS configuration with hardcoded credentials 29 | cfg, err := config.LoadDefaultConfig(context.TODO(), 30 | config.WithRegion("us-east-1"), 31 | config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, "")), 32 | ) 33 | if err != nil { 34 | log.Fatal().Msgf("unable to load SDK config, %v", err) 35 | } 36 | // cfg.RateLimiter = NoOpRateLimit{} 37 | 38 | sqsClient := sqs.NewFromConfig(cfg, func(o *sqs.Options) { 39 | o.BaseEndpoint = aws.String(c.SqsEndpoint) 40 | }) 41 | 42 | var wg sync.WaitGroup 43 | 44 | var ch chan int 45 | 46 | for i := 0; i < c.Senders; i++ { 47 | wg.Add(1) 48 | go func(id int) { 49 | defer wg.Done() 50 | for j := 0; j < c.Messages; j += c.BatchSize { 51 | sendMessage(sqsClient, queueUrl, id, j, c.BatchSize, c.DelaySeconds) 52 | sentMessages += 1 53 | } 54 | }(i) 55 | } 56 | 57 | for i := 0; i < c.Receivers; i++ { 58 | go func(id int) { 59 | for { 60 | msgs := receiveMessage(sqsClient, queueUrl, id) 61 | receivedMessages += msgs 62 | // time.Sleep(1 * time.Second) 63 | } 64 | }(i) 65 | } 66 | 67 | go func() { 68 | pct := 0.0 69 | for { 70 | if sentMessages > 0 { 71 | pct = float64(receivedMessages) / float64(sentMessages) 72 | } 73 | log.Info().Msg(fmt.Sprintf("sent: %d, received: %d, pct: %f", sentMessages, receivedMessages, pct)) 74 | time.Sleep(1 * time.Second) 75 | } 76 | }() 77 | 78 | wg.Wait() 79 | 80 | if c.Senders > 0 { 81 | log.Info().Msg("All messages sent") 82 | if c.Receivers == 0 { 83 | os.Exit(0) 84 | } 85 | } 86 | 87 | <-ch 88 | } 89 | 90 | func GenerateRandomString(n int) string { 91 | const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" 92 | b := make([]byte, n) 93 | for i := range b { 94 | b[i] = charset[rand.Intn(len(charset))] 95 | } 96 | return string(b) 97 | } 98 | 99 | func sendMessage(client *sqs.Client, queueUrl string, goroutineID, requestID, batchSize, delaySeconds int) { 100 | 101 | if batchSize > 1 { 102 | input := &sqs.SendMessageBatchInput{ 103 | QueueUrl: aws.String(queueUrl), 104 | } 105 | 106 | for i := range batchSize { 107 | messageBody := fmt.Sprintf("Message from goroutine %d, request %d, batchId %d %s", goroutineID, requestID, i, GenerateRandomString(2000)) 108 | input.Entries = append(input.Entries, types.SendMessageBatchRequestEntry{ 109 | Id: aws.String(fmt.Sprintf("%d", i)), 110 | MessageBody: &messageBody, 111 | MessageAttributes: map[string]types.MessageAttributeValue{ 112 | "a": { 113 | DataType: aws.String("String"), 114 | StringValue: aws.String("abc"), 115 | }, 116 | "b": { 117 | DataType: aws.String("Binary"), 118 | BinaryValue: []byte("xyz"), 119 | }, 120 | }, 121 | }) 122 | } 123 | 124 | _, err := client.SendMessageBatch(context.TODO(), input) 125 | 126 | if err != nil { 127 | log.Printf("Failed to send message from goroutine %d, request %d: %v", goroutineID, requestID, err) 128 | } 129 | 130 | } else { 131 | messageBody := fmt.Sprintf("Message from goroutine %d, request %d %s", goroutineID, requestID, GenerateRandomString(5)) 132 | input := &sqs.SendMessageInput{ 133 | QueueUrl: aws.String(queueUrl), 134 | MessageBody: aws.String(messageBody), 135 | DelaySeconds: *aws.Int32(int32(delaySeconds)), 136 | MessageAttributes: map[string]types.MessageAttributeValue{ 137 | "a": { 138 | DataType: aws.String("String"), 139 | StringValue: aws.String("abc"), 140 | }, 141 | "b": { 142 | DataType: aws.String("Binary"), 143 | BinaryValue: []byte("xyz"), 144 | }, 145 | }, 146 | } 147 | _, err := client.SendMessage(context.TODO(), input) 148 | 149 | if err != nil { 150 | log.Printf("Failed to send message from goroutine %d, request %d: %v", goroutineID, requestID, err) 151 | } 152 | } 153 | 154 | // time.Sleep(100 * time.Millisecond) 155 | } 156 | 157 | func receiveMessage(client *sqs.Client, queueUrl string, goroutineID int) int { 158 | i := &sqs.ReceiveMessageInput{ 159 | QueueUrl: aws.String(queueUrl), 160 | MaxNumberOfMessages: 1, 161 | // VisibilityTimeout: *aws.Int32(5), 162 | MessageAttributeNames: []string{ 163 | "All", 164 | }, 165 | } 166 | msgs, err := client.ReceiveMessage(context.TODO(), i) 167 | if err != nil { 168 | log.Error().Err(err).Send() 169 | } 170 | 171 | for _, msg := range msgs.Messages { 172 | log.Trace().Interface("message", msg).Msg("Received message") 173 | delInput := &sqs.DeleteMessageInput{ 174 | QueueUrl: aws.String(queueUrl), 175 | ReceiptHandle: msg.ReceiptHandle, 176 | } 177 | _, delerr := client.DeleteMessage(context.TODO(), delInput) 178 | if delerr != nil { 179 | log.Printf("Failed to delete message from goroutine %d, request %d: %v", goroutineID, msg.ReceiptHandle, delerr) 180 | } 181 | } 182 | 183 | // time.Sleep(1 * time.Second) 184 | return len(msgs.Messages) 185 | 186 | } 187 | -------------------------------------------------------------------------------- /config.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | server: 3 | sqs: 4 | enabled: true 5 | port: 3001 6 | keys: 7 | - accesskey: DEV_ACCESS_KEY_ID 8 | secretkey: DEV_SECRET_ACCESS_KEY 9 | 10 | dashboard: 11 | enabled: true 12 | port: 3000 13 | 14 | sqlite: 15 | path: smoothmq.sqlite -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "errors" 5 | "strings" 6 | 7 | "github.com/alecthomas/kong" 8 | kongyaml "github.com/alecthomas/kong-yaml" 9 | ) 10 | 11 | type CLI struct { 12 | Server ServerCommand `cmd:"server" help:"Run queue server"` 13 | Tester TesterCommand `cmd:"tester" help:"Run queue test tool"` 14 | 15 | Config kong.ConfigFlag `name:"config" help:"Configuration file"` 16 | Log LogConfig `embed:"" prefix:"log-" name:"log" envprefix:"LOG_"` 17 | } 18 | 19 | type TesterCommand struct { 20 | SqsEndpoint string `help:"SQS endpoint" name:"endpoint" default:"http://localhost:3001"` 21 | Senders int `help:"" default:"0"` 22 | Receivers int `help:"" default:"0"` 23 | Messages int `help:"" default:"0"` 24 | BatchSize int `help:"" default:"1"` 25 | DelaySeconds int `help:"" default:"0"` 26 | AccessKey string `help:"" default:"DEV_ACCESS_KEY_ID"` 27 | SecretKey string `help:"" default:"DEV_SECRET_ACCESS_KEY"` 28 | } 29 | 30 | type ServerCommand struct { 31 | SQS SQSConfig `embed:"" prefix:"sqs-" envprefix:"Q_SQS_"` 32 | Dashboard DashboardConfig `embed:"" prefix:"dashboard-" envprefix:"Q_DASHBOARD_"` 33 | SQLite SQLiteConfig `embed:"" prefix:"sqlite-" envprefix:"Q_SQLITE_"` 34 | Metrics MetricsConfig `embed:"" prefix:"metrics-" name:"metrics" envprefix:"Q_METRICS_"` 35 | 36 | DisableTelemetry bool `name:"disable-telemetry" default:"false" env:"DISABLE_TELEMETRY"` 37 | UseSinglePort bool `name:"use-single-port" default:"false" env:"Q_SERVER_USE_SINGLE_PORT" help:"Enables having all HTTP services run on a single port with different endpoints"` 38 | Port int `name:"port" default:"8080" env:"PORT" help:"If use-single-port is enabled, this is the port number for the server"` 39 | Host string `name:"host" env:"HOST" help:"If use-single-port is enabled, this is the hostname or ip address for the server"` 40 | } 41 | 42 | type LogConfig struct { 43 | Pretty bool `name:"pretty" default:"true" env:"PRETTY"` 44 | Level string `name:"level" enum:"trace,debug,info,warn,error,fatal,panic" default:"info" help:"Log level" env:"LEVEL"` 45 | } 46 | 47 | type MetricsConfig struct { 48 | PrometheusEnabled bool `name:"prometheus-enabled" default:"true" env:"PROMETHEUS_ENABLED"` 49 | PrometheusHost string `name:"prometheus-host" default:"localhost" env:"PROMETHEUS_HOST"` 50 | PrometheusPort int `name:"prometheus-port" default:"2112" env:"PROMETHEUS_PORT"` 51 | PrometheusPath string `name:"prometheus-path" default:"/metrics" env:"PROMETHEUS_PATH"` 52 | } 53 | 54 | type SQLiteConfig struct { 55 | Path string `name:"path" help:"Path of SQLite file" default:"smoothmq.sqlite" env:"PATH"` 56 | } 57 | 58 | type SQSConfig struct { 59 | Enabled bool `name:"enabled" default:"true" help:"Enable SQS protocol for queue" env:"ENABLED"` 60 | Host string `name:"host" default:"localhost" help:"hostname or ip address for SQS protocol" env:"HOST"` 61 | Port int `name:"port" default:"3001" help:"HTTP port for SQS protocol" env:"PORT"` 62 | Keys []AWSKey `name:"keys" default:"DEV_ACCESS_KEY_ID:DEV_SECRET_ACCESS_KEY" env:"KEYS"` 63 | ParseCelery bool `name:"parse-celery" default:"true" env:"PARSE_CELERY" help:"Parse Celery messages. Lets you search by celery message ID and task type."` 64 | MaxRequestSize int `name:"max-request-size" default:"1048576" env:"MAX_REQUEST_SIZE" help:"Max size of SQS request in bytes"` 65 | MaxDelaySeconds int `name:"max-delay-seconds" default:"30" env:"MAX_DELAY_SECONDS" help:"Max allowed wait time for long polling"` 66 | DelayRetryMillis int `name:"delay-retry-millis" default:"1000" env:"DELAY_RETRY_MILLIS" help:"When long polling, how often to request new items"` 67 | EndpointOverride string `name:"endpoint-override" default:"" env:"ENDPOINT_OVERRIDE" help:"Endpoint to advertise in queue URLs. Defaults to HTTP hostname."` 68 | } 69 | 70 | type AWSKey struct { 71 | AccessKey string `name:"accesskey"` 72 | SecretKey string `name:"secretkey"` 73 | } 74 | 75 | func (k *AWSKey) Decode(ctx *kong.DecodeContext) error { 76 | var val string 77 | 78 | err := ctx.Scan.PopValueInto("string", &val) 79 | if err != nil { 80 | return err 81 | } 82 | 83 | tokens := strings.Split(val, ":") 84 | if len(tokens) != 2 { 85 | return errors.New("AWS SQS key should be of the form access_key:secret_key") 86 | } 87 | 88 | k.AccessKey = tokens[0] 89 | k.SecretKey = tokens[1] 90 | 91 | return nil 92 | } 93 | 94 | type DashboardConfig struct { 95 | Enabled bool `name:"enabled" help:"Enable web dashboard" default:"true" env:"ENABLED"` 96 | Host string `name:"host" help:"hostname or ip address for dashboard" default:"localhost" env:"HOST"` 97 | Port int `name:"port" help:"HTTP port for dashboard" default:"3000" env:"PORT"` 98 | Dev bool `name:"dev" help:"Run dashboard in dev mode, refresh templates from local" default:"false" env:"DEV"` 99 | User string `name:"user" help:"Username for auth" default:"" env:"USER"` 100 | Pass string `name:"pass" help:"Pass for auth" default:"" env:"PASS"` 101 | } 102 | 103 | func Load() (string, *CLI, error) { 104 | cli := &CLI{} 105 | c := kong.Parse(cli, kong.Configuration(kongyaml.Loader)) 106 | 107 | return c.Command(), cli, c.Error 108 | } 109 | -------------------------------------------------------------------------------- /dashboard/dashboard.go: -------------------------------------------------------------------------------- 1 | package dashboard 2 | 3 | import ( 4 | "embed" 5 | "encoding/base64" 6 | "encoding/json" 7 | "errors" 8 | "fmt" 9 | "io/fs" 10 | "net/http" 11 | "strconv" 12 | "strings" 13 | 14 | "github.com/poundifdef/smoothmq/config" 15 | "github.com/poundifdef/smoothmq/models" 16 | 17 | "github.com/rs/zerolog/log" 18 | 19 | "github.com/gofiber/fiber/v2" 20 | "github.com/gofiber/fiber/v2/middleware/adaptor" 21 | "github.com/gofiber/fiber/v2/middleware/basicauth" 22 | "github.com/gofiber/template/html/v2" 23 | ) 24 | 25 | //go:embed views/* 26 | var viewsfs embed.FS 27 | 28 | type Dashboard struct { 29 | App *fiber.App 30 | 31 | queue models.Queue 32 | tenantManager models.TenantManager 33 | 34 | cfg config.DashboardConfig 35 | } 36 | 37 | func NewDashboard(queue models.Queue, tenantManager models.TenantManager, cfg config.DashboardConfig) *Dashboard { 38 | var engine *html.Engine 39 | 40 | if cfg.Dev { 41 | engine = html.New("./dashboard/views", ".html") 42 | engine.Reload(true) 43 | engine.Debug(true) 44 | } else { 45 | http.FS(viewsfs) 46 | fs2, err := fs.Sub(viewsfs, "views") 47 | if err != nil { 48 | log.Fatal().Err(err).Send() 49 | } 50 | engine = html.NewFileSystem(http.FS(fs2), ".html") 51 | } 52 | 53 | engine.AddFunc("b64Json", func(s []byte) string { 54 | decodedStr, err := base64.StdEncoding.DecodeString(string(s)) 55 | if err == nil { 56 | if json.Valid(decodedStr) { 57 | return string(decodedStr) 58 | } 59 | } 60 | 61 | return string(s) 62 | }) 63 | 64 | app := fiber.New(fiber.Config{ 65 | Views: engine, 66 | DisableStartupMessage: true, 67 | }) 68 | 69 | if cfg.User != "" && cfg.Pass != "" { 70 | app.Use(basicauth.New(basicauth.Config{ 71 | Users: map[string]string{ 72 | cfg.User: cfg.Pass, 73 | }, 74 | })) 75 | } 76 | 77 | d := &Dashboard{ 78 | App: app, 79 | queue: queue, 80 | tenantManager: tenantManager, 81 | cfg: cfg, 82 | } 83 | 84 | app.Get("/", d.Queues) 85 | app.Post("/queues", d.NewQueue) 86 | app.Get("/queues/:queue", d.Queue) 87 | app.Get("/queues/:queue/settings", d.QueueSettings) 88 | app.Post("/queues/:queue/settings", d.SaveQueueSettings) 89 | app.Get("/queues/:queue/delete", d.DeleteQueueConfirm) 90 | app.Post("/queues/:queue/delete", d.DeleteQueue) 91 | app.Get("/queues/:queue/messages/:message", d.Message) 92 | 93 | return d 94 | } 95 | 96 | func (d *Dashboard) Start() error { 97 | if !d.cfg.Enabled { 98 | return nil 99 | } 100 | 101 | fmt.Printf("Dashboard: http://%s:%d\n", d.cfg.Host, d.cfg.Port) 102 | return d.App.Listen(fmt.Sprintf("%s:%d", d.cfg.Host, d.cfg.Port)) 103 | } 104 | 105 | func (d *Dashboard) Stop() error { 106 | if d.cfg.Enabled { 107 | return d.App.Shutdown() 108 | } 109 | 110 | return nil 111 | } 112 | 113 | func (d *Dashboard) Queues(c *fiber.Ctx) error { 114 | r, err := adaptor.ConvertRequest(c, false) 115 | if err != nil { 116 | return err 117 | } 118 | 119 | tenantId, err := d.tenantManager.GetTenant(r) 120 | if err != nil { 121 | return err 122 | } 123 | 124 | type QueueDetails struct { 125 | Name string 126 | Stats models.QueueStats 127 | Count int 128 | } 129 | 130 | queues, err := d.queue.ListQueues(tenantId) 131 | 132 | queueDetails := make([]QueueDetails, len(queues)) 133 | for i, queue := range queues { 134 | queueStats := d.queue.Stats(tenantId, queue) 135 | 136 | totalMessages := 0 137 | for _, v := range queueStats.Counts { 138 | totalMessages += v 139 | } 140 | 141 | queueDetails[i] = QueueDetails{ 142 | Name: queue, 143 | Stats: queueStats, 144 | Count: totalMessages, 145 | } 146 | } 147 | 148 | return c.Render("queues", fiber.Map{"Queues": queueDetails, "Err": err}, "layout") 149 | } 150 | 151 | func (d *Dashboard) Queue(c *fiber.Ctx) error { 152 | queueName := c.Params("queue") 153 | 154 | r, err := adaptor.ConvertRequest(c, false) 155 | if err != nil { 156 | return err 157 | } 158 | 159 | tenantId, err := d.tenantManager.GetTenant(r) 160 | if err != nil { 161 | return err 162 | } 163 | 164 | queueStats := d.queue.Stats(tenantId, queueName) 165 | 166 | filterCriteria := models.FilterCriteria{ 167 | KV: make(map[string]string), 168 | } 169 | filterString := c.Query("filter") 170 | 171 | filterFields := strings.Fields(filterString) 172 | for _, field := range filterFields { 173 | maybeMessageID, err := strconv.ParseInt(field, 10, 64) 174 | if err == nil { 175 | filterCriteria.MessageID = maybeMessageID 176 | } 177 | 178 | if strings.Contains(field, "=") { 179 | tokens := strings.Split(field, "=") 180 | filterCriteria.KV[strings.TrimSpace(tokens[0])] = strings.TrimSpace(tokens[1]) 181 | } 182 | 183 | } 184 | 185 | filteredMessageIDs := d.queue.Filter(tenantId, queueName, filterCriteria) 186 | 187 | messages := make([]*models.Message, 0) 188 | for _, messageId := range filteredMessageIDs { 189 | message := d.queue.Peek(tenantId, queueName, messageId) 190 | if message != nil { 191 | messages = append(messages, message) 192 | } 193 | } 194 | 195 | return c.Render("queue", fiber.Map{"Queue": queueName, "Stats": queueStats, "Messages": messages, "Filter": filterString}, "layout") 196 | } 197 | 198 | func (d *Dashboard) DeleteQueueConfirm(c *fiber.Ctx) error { 199 | queueName := c.Params("queue") 200 | 201 | r, err := adaptor.ConvertRequest(c, false) 202 | if err != nil { 203 | return err 204 | } 205 | 206 | _, err = d.tenantManager.GetTenant(r) 207 | if err != nil { 208 | return err 209 | } 210 | 211 | return c.Render("delete_queue", fiber.Map{"Queue": queueName}, "layout") 212 | } 213 | 214 | func (d *Dashboard) QueueSettings(c *fiber.Ctx) error { 215 | queueName := c.Params("queue") 216 | 217 | r, err := adaptor.ConvertRequest(c, false) 218 | if err != nil { 219 | return err 220 | } 221 | 222 | tenantId, err := d.tenantManager.GetTenant(r) 223 | if err != nil { 224 | return err 225 | } 226 | 227 | queue, err := d.queue.GetQueue(tenantId, queueName) 228 | if err != nil { 229 | return err 230 | } 231 | 232 | return c.Render("queue_settings", fiber.Map{"Queue": queue}, "layout") 233 | } 234 | 235 | func (d *Dashboard) SaveQueueSettings(c *fiber.Ctx) error { 236 | queueName := c.Params("queue") 237 | 238 | r, err := adaptor.ConvertRequest(c, false) 239 | if err != nil { 240 | return err 241 | } 242 | 243 | tenantId, err := d.tenantManager.GetTenant(r) 244 | if err != nil { 245 | return err 246 | } 247 | 248 | queue, err := d.queue.GetQueue(tenantId, queueName) 249 | if err != nil { 250 | return err 251 | } 252 | 253 | rateLimit, err := strconv.ParseFloat(c.FormValue("rate_limit"), 64) 254 | if err != nil { 255 | return err 256 | } 257 | 258 | maxRetries, err := strconv.ParseInt(c.FormValue("max_retries"), 10, 32) 259 | if err != nil { 260 | return err 261 | } 262 | 263 | visibilityTimeout, err := strconv.ParseInt(c.FormValue("visibility_timeout"), 10, 32) 264 | if err != nil { 265 | return err 266 | } 267 | 268 | queue.RateLimit = rateLimit 269 | queue.MaxRetries = int(maxRetries) 270 | queue.VisibilityTimeout = int(visibilityTimeout) 271 | 272 | err = d.queue.UpdateQueue(tenantId, queueName, queue) 273 | if err != nil { 274 | return err 275 | } 276 | 277 | return c.Redirect("/queues/" + queueName + "/settings") 278 | // return c.Render("queue_settings", fiber.Map{"Queue": queue}, "layout") 279 | } 280 | 281 | func (d *Dashboard) Message(c *fiber.Ctx) error { 282 | queueName := c.Params("queue") 283 | messageID := c.Params("message") 284 | r, err := adaptor.ConvertRequest(c, false) 285 | if err != nil { 286 | return err 287 | } 288 | 289 | tenantId, err := d.tenantManager.GetTenant(r) 290 | if err != nil { 291 | return err 292 | } 293 | 294 | // TODO: check for errors 295 | messageIdInt, err := strconv.ParseInt(messageID, 10, 64) 296 | if err != nil { 297 | return err 298 | } 299 | 300 | message := d.queue.Peek(tenantId, queueName, messageIdInt) 301 | if message == nil { 302 | return errors.New("Message not found") 303 | } 304 | 305 | return c.Render("message", fiber.Map{"Queue": queueName, "Message": message}, "layout") 306 | } 307 | 308 | func (d *Dashboard) NewQueue(c *fiber.Ctx) error { 309 | queueName := c.FormValue("queue") 310 | 311 | r, err := adaptor.ConvertRequest(c, false) 312 | if err != nil { 313 | return err 314 | } 315 | 316 | tenantId, err := d.tenantManager.GetTenant(r) 317 | if err != nil { 318 | return err 319 | } 320 | 321 | properties := models.QueueProperties{ 322 | Name: queueName, 323 | RateLimit: -1, 324 | MaxRetries: -1, 325 | VisibilityTimeout: 30, 326 | } 327 | err = d.queue.CreateQueue(tenantId, properties) 328 | 329 | if err != nil { 330 | return err 331 | } 332 | 333 | return c.Redirect("/") 334 | } 335 | 336 | func (d *Dashboard) DeleteQueue(c *fiber.Ctx) error { 337 | queueName := c.Params("queue") 338 | r, err := adaptor.ConvertRequest(c, false) 339 | if err != nil { 340 | return err 341 | } 342 | 343 | tenantId, err := d.tenantManager.GetTenant(r) 344 | if err != nil { 345 | return err 346 | } 347 | 348 | err = d.queue.DeleteQueue(tenantId, queueName) 349 | if err != nil { 350 | return err 351 | } 352 | 353 | return c.Redirect("/") 354 | } 355 | -------------------------------------------------------------------------------- /dashboard/views/delete_queue.html: -------------------------------------------------------------------------------- 1 |

2 | Are you sure you want to delete queue {{.Queue}}? 3 | This will permanently delete all messages. 4 |

5 | 6 |
7 | 8 |
-------------------------------------------------------------------------------- /dashboard/views/layout.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 |
11 | 12 | {{if .Err}} 13 |
14 |
{{.Err}}
15 |
16 | {{end}} 17 | 18 | {{embed}} 19 | 20 | 21 | 22 |
23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /dashboard/views/message.html: -------------------------------------------------------------------------------- 1 | 2 |
3 | Queues 4 |
/
5 | {{.Queue}} 6 |
/
7 | {{.Message.ID}} 8 |
9 | 10 |
11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 47 | 48 | 49 | {{ if .Message.IsB64 }} 50 | 51 | 52 | 53 | 54 | {{end}} 55 | 56 | 57 | 58 |
Status{{.Message.Status}}
k/v 25 | {{range $k, $v := .Message.KeyValues}} 26 | {{$k}}: {{$v}} 27 | {{end}} 28 |
Deliver At{{.Message.DeliverAt}}
Last Delivered{{.Message.DeliveredAt}}
Delivery Attempts{{.Message.Tries}}
Message 45 | {{printf "%s" .Message.Message}} 46 |
Base64 Decoded {{printf "%s" .Message.Base64Decode}}
59 | -------------------------------------------------------------------------------- /dashboard/views/queue.html: -------------------------------------------------------------------------------- 1 | 2 |
3 | Queues 4 |
/
5 | {{.Queue}} 6 |
7 | 8 |
9 | 10 |
11 | 12 |
13 |
14 | Messages 15 | Settings 16 |
17 |
18 | 19 |
20 |
21 |
22 |
23 | {{index $.Stats.Counts 1}} 24 |
25 |
26 | Queued 27 |
28 |
29 |
30 |
31 | {{index $.Stats.Counts 2}} 32 |
33 |
34 | Processing 35 |
36 |
37 |
38 |
39 | {{index $.Stats.Counts 3}} 40 |
41 |
42 | Failed 43 |
44 |
45 |
46 |
47 | 48 |
49 |

Search for a specific message by ID, or filter based on key/value pairs.

50 |
51 |
52 | 53 | 54 |
55 |
56 |
57 |
58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | {{range .Messages}} 67 | 68 | 71 | 72 | 77 | 83 | 84 | {{end}} 85 | 86 |
IDStatusk/vMessage
69 | {{.ID}} 70 | {{.Status}} 73 | {{range $k, $v := .KeyValues}} 74 | {{$k}}: {{$v}} 75 | {{end}} 76 | 78 | {{ if .IsB64 }} 79 |
b64
80 | {{end}} 81 | {{printf "%s" .Base64Decode}} 82 |
87 |
88 |
89 | -------------------------------------------------------------------------------- /dashboard/views/queue_settings.html: -------------------------------------------------------------------------------- 1 | 2 |
3 | Queues 4 |
/
5 | {{.Queue.Name}} 6 |
7 | 8 |
9 | 10 |
11 | 12 |
13 |
14 | Messages 15 | Settings 16 |
17 |
18 | 19 |
20 | 21 |
22 |
23 | 24 | 25 |
26 |
27 | 28 | 29 |
30 |
31 | 32 | 33 |
34 | 35 |
36 | 37 |
38 |
39 | -------------------------------------------------------------------------------- /dashboard/views/queues.html: -------------------------------------------------------------------------------- 1 | 2 |
3 | Queues 4 |
5 | 6 |
7 | 8 |

9 | Select a queue to view messages. 10 |

11 | 12 |
13 |
14 | 15 | 16 |
17 |
18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | {{range .Queues}} 30 | 31 | 32 | 33 | 41 | 42 | {{end}} 43 | 44 |
NameMessages 
{{.Name}} {{.Count}} 34 | Settings 35 | Delete 36 | 37 | 40 |
-------------------------------------------------------------------------------- /docs/queue.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/poundifdef/smoothmq/a6094b12b2f13f3a0f6349ac12cccc2f4715a47b/docs/queue.gif -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/poundifdef/smoothmq 2 | 3 | go 1.22.2 4 | 5 | require ( 6 | github.com/alecthomas/kong v0.9.0 7 | github.com/alecthomas/kong-yaml v0.2.0 8 | github.com/aws/aws-sdk-go-v2 v1.30.1 9 | github.com/aws/aws-sdk-go-v2/config v1.27.23 10 | github.com/aws/aws-sdk-go-v2/credentials v1.17.23 11 | github.com/aws/aws-sdk-go-v2/service/sqs v1.34.1 12 | github.com/bwmarrin/snowflake v0.3.0 13 | github.com/gofiber/contrib/fiberzerolog v1.0.1 14 | github.com/gofiber/fiber/v2 v2.52.5 15 | github.com/gofiber/template/html/v2 v2.1.1 16 | github.com/mattn/go-sqlite3 v1.14.22 17 | github.com/prometheus/client_golang v1.19.1 18 | github.com/rs/zerolog v1.33.0 19 | github.com/tidwall/gjson v1.17.1 20 | github.com/valyala/fasthttp v1.51.0 21 | gorm.io/driver/sqlite v1.5.6 22 | gorm.io/gorm v1.25.11 23 | ) 24 | 25 | require ( 26 | github.com/andybalholm/brotli v1.0.5 // indirect 27 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9 // indirect 28 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13 // indirect 29 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13 // indirect 30 | github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect 31 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect 32 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15 // indirect 33 | github.com/aws/aws-sdk-go-v2/service/sso v1.22.1 // indirect 34 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.1 // indirect 35 | github.com/aws/aws-sdk-go-v2/service/sts v1.30.1 // indirect 36 | github.com/aws/smithy-go v1.20.3 // indirect 37 | github.com/beorn7/perks v1.0.1 // indirect 38 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 39 | github.com/gofiber/template v1.8.3 // indirect 40 | github.com/gofiber/utils v1.1.0 // indirect 41 | github.com/google/uuid v1.5.0 // indirect 42 | github.com/jinzhu/inflection v1.0.0 // indirect 43 | github.com/jinzhu/now v1.1.5 // indirect 44 | github.com/klauspost/compress v1.17.9 // indirect 45 | github.com/kr/text v0.2.0 // indirect 46 | github.com/mattn/go-colorable v0.1.13 // indirect 47 | github.com/mattn/go-isatty v0.0.20 // indirect 48 | github.com/mattn/go-runewidth v0.0.15 // indirect 49 | github.com/prometheus/client_model v0.5.0 // indirect 50 | github.com/prometheus/common v0.48.0 // indirect 51 | github.com/prometheus/procfs v0.12.0 // indirect 52 | github.com/rivo/uniseg v0.2.0 // indirect 53 | github.com/stretchr/testify v1.9.0 // indirect 54 | github.com/tidwall/match v1.1.1 // indirect 55 | github.com/tidwall/pretty v1.2.0 // indirect 56 | github.com/valyala/bytebufferpool v1.0.0 // indirect 57 | github.com/valyala/tcplisten v1.0.0 // indirect 58 | golang.org/x/sys v0.22.0 // indirect 59 | golang.org/x/text v0.16.0 // indirect 60 | google.golang.org/protobuf v1.33.0 // indirect 61 | gopkg.in/yaml.v3 v3.0.1 // indirect 62 | ) 63 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/alecthomas/assert/v2 v2.6.0 h1:o3WJwILtexrEUk3cUVal3oiQY2tfgr/FHWiz/v2n4FU= 2 | github.com/alecthomas/assert/v2 v2.6.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= 3 | github.com/alecthomas/kong v0.9.0 h1:G5diXxc85KvoV2f0ZRVuMsi45IrBgx9zDNGNj165aPA= 4 | github.com/alecthomas/kong v0.9.0/go.mod h1:Y47y5gKfHp1hDc7CH7OeXgLIpp+Q2m1Ni0L5s3bI8Os= 5 | github.com/alecthomas/kong-yaml v0.2.0 h1:iiVVqVttmOsHKawlaW/TljPsjaEv1O4ODx6dloSA58Y= 6 | github.com/alecthomas/kong-yaml v0.2.0/go.mod h1:vMvOIy+wpB49MCZ0TA3KMts38Mu9YfRP03Q1StN69/g= 7 | github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= 8 | github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= 9 | github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= 10 | github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= 11 | github.com/aws/aws-sdk-go-v2 v1.30.1 h1:4y/5Dvfrhd1MxRDD77SrfsDaj8kUkkljU7XE83NPV+o= 12 | github.com/aws/aws-sdk-go-v2 v1.30.1/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc= 13 | github.com/aws/aws-sdk-go-v2/config v1.27.23 h1:Cr/gJEa9NAS7CDAjbnB7tHYb3aLZI2gVggfmSAasDac= 14 | github.com/aws/aws-sdk-go-v2/config v1.27.23/go.mod h1:WMMYHqLCFu5LH05mFOF5tsq1PGEMfKbu083VKqLCd0o= 15 | github.com/aws/aws-sdk-go-v2/credentials v1.17.23 h1:G1CfmLVoO2TdQ8z9dW+JBc/r8+MqyPQhXCafNZcXVZo= 16 | github.com/aws/aws-sdk-go-v2/credentials v1.17.23/go.mod h1:V/DvSURn6kKgcuKEk4qwSwb/fZ2d++FFARtWSbXnLqY= 17 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9 h1:Aznqksmd6Rfv2HQN9cpqIV/lQRMaIpJkLLaJ1ZI76no= 18 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9/go.mod h1:WQr3MY7AxGNxaqAtsDWn+fBxmd4XvLkzeqQ8P1VM0/w= 19 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13 h1:5SAoZ4jYpGH4721ZNoS1znQrhOfZinOhc4XuTXx/nVc= 20 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13/go.mod h1:+rdA6ZLpaSeM7tSg/B0IEDinCIBJGmW8rKDFkYpP04g= 21 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13 h1:WIijqeaAO7TYFLbhsZmi2rgLEAtWOC1LhxCAVTJlSKw= 22 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13/go.mod h1:i+kbfa76PQbWw/ULoWnp51EYVWH4ENln76fLQE3lXT8= 23 | github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= 24 | github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= 25 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8= 26 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI= 27 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15 h1:I9zMeF107l0rJrpnHpjEiiTSCKYAIw8mALiXcPsGBiA= 28 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15/go.mod h1:9xWJ3Q/S6Ojusz1UIkfycgD1mGirJfLLKqq3LPT7WN8= 29 | github.com/aws/aws-sdk-go-v2/service/sqs v1.34.1 h1:Tp1oKSfWHE8fTz0H+DuD05cXPJ96Z6Rko0W/dAp7wJ0= 30 | github.com/aws/aws-sdk-go-v2/service/sqs v1.34.1/go.mod h1:5gGM2xv51W5Hkyr3vj7JTEf/b5oOCb7rXcEVbXrcTAU= 31 | github.com/aws/aws-sdk-go-v2/service/sso v1.22.1 h1:p1GahKIjyMDZtiKoIn0/jAj/TkMzfzndDv5+zi2Mhgc= 32 | github.com/aws/aws-sdk-go-v2/service/sso v1.22.1/go.mod h1:/vWdhoIoYA5hYoPZ6fm7Sv4d8701PiG5VKe8/pPJL60= 33 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.1 h1:lCEv9f8f+zJ8kcFeAjRZsekLd/x5SAm96Cva+VbUdo8= 34 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.1/go.mod h1:xyFHA4zGxgYkdD73VeezHt3vSKEG9EmFnGwoKlP00u4= 35 | github.com/aws/aws-sdk-go-v2/service/sts v1.30.1 h1:+woJ607dllHJQtsnJLi52ycuqHMwlW+Wqm2Ppsfp4nQ= 36 | github.com/aws/aws-sdk-go-v2/service/sts v1.30.1/go.mod h1:jiNR3JqT15Dm+QWq2SRgh0x0bCNSRP2L25+CqPNpJlQ= 37 | github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE= 38 | github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= 39 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 40 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 41 | github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0= 42 | github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE= 43 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 44 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 45 | github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= 46 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 47 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 48 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 49 | github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= 50 | github.com/gofiber/contrib/fiberzerolog v1.0.1 h1:kKIlD1v72Iy9/PW3z5nyZ1XeAAO1ZNIl8oKGen0BiV4= 51 | github.com/gofiber/contrib/fiberzerolog v1.0.1/go.mod h1:bDsXnPPGUVTETnl9KfzoTgLABYFxDdIjSabLqU763Do= 52 | github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yGMo= 53 | github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ= 54 | github.com/gofiber/template v1.8.3 h1:hzHdvMwMo/T2kouz2pPCA0zGiLCeMnoGsQZBTSYgZxc= 55 | github.com/gofiber/template v1.8.3/go.mod h1:bs/2n0pSNPOkRa5VJ8zTIvedcI/lEYxzV3+YPXdBvq8= 56 | github.com/gofiber/template/html/v2 v2.1.1 h1:QEy3O3EBkvwDthy5bXVGUseOyO6ldJoiDxlF4+MJiV8= 57 | github.com/gofiber/template/html/v2 v2.1.1/go.mod h1:2G0GHHOUx70C1LDncoBpe4T6maQbNa4x1CVNFW0wju0= 58 | github.com/gofiber/utils v1.1.0 h1:vdEBpn7AzIUJRhe+CiTOJdUcTg4Q9RK+pEa0KPbLdrM= 59 | github.com/gofiber/utils v1.1.0/go.mod h1:poZpsnhBykfnY1Mc0KeEa6mSHrS3dV0+oBWyeQmb2e0= 60 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 61 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 62 | github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= 63 | github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 64 | github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= 65 | github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= 66 | github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= 67 | github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= 68 | github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= 69 | github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= 70 | github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= 71 | github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= 72 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 73 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 74 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 75 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 76 | github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 77 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 78 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 79 | github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 80 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 81 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 82 | github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= 83 | github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= 84 | github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= 85 | github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= 86 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 87 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 88 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 89 | github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= 90 | github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= 91 | github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= 92 | github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= 93 | github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= 94 | github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= 95 | github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= 96 | github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= 97 | github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= 98 | github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 99 | github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= 100 | github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= 101 | github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= 102 | github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= 103 | github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= 104 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 105 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 106 | github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= 107 | github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= 108 | github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= 109 | github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= 110 | github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= 111 | github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= 112 | github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= 113 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 114 | github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA= 115 | github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g= 116 | github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= 117 | github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= 118 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 119 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 120 | golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 121 | golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= 122 | golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 123 | golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= 124 | golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= 125 | google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= 126 | google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= 127 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 128 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 129 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 130 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 131 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 132 | gorm.io/driver/sqlite v1.5.6 h1:fO/X46qn5NUEEOZtnjJRWRzZMe8nqJiQ9E+0hi+hKQE= 133 | gorm.io/driver/sqlite v1.5.6/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4= 134 | gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg= 135 | gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= 136 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | 6 | "github.com/poundifdef/smoothmq/cmd/smoothmq" 7 | "github.com/poundifdef/smoothmq/config" 8 | ) 9 | 10 | func main() { 11 | log.SetFlags(log.LstdFlags | log.Lshortfile) 12 | 13 | command, cli, err := config.Load() 14 | if err != nil { 15 | panic(err) 16 | } 17 | 18 | smoothmq.Run(command, cli, nil, nil) 19 | } 20 | -------------------------------------------------------------------------------- /models/message.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/json" 6 | "fmt" 7 | "time" 8 | ) 9 | 10 | type MessageStatus uint8 11 | 12 | const ( 13 | MessageStatusQueued MessageStatus = 1 14 | MessageStatusDequeued MessageStatus = 2 15 | MessageStatusFailed MessageStatus = 3 16 | // MessageStatusPaused MessageStatus = 3 17 | // MessageStatusDeleted MessageStatus = 4 18 | ) 19 | 20 | func (s MessageStatus) String() string { 21 | switch s { 22 | case MessageStatusQueued: 23 | return "Queued" 24 | case MessageStatusDequeued: 25 | return "Dequeued" 26 | case MessageStatusFailed: 27 | return "Failed" 28 | } 29 | 30 | return fmt.Sprintf("%d", s) 31 | } 32 | 33 | type Message struct { 34 | ID int64 `db:"id"` 35 | TenantID int64 `db:"tenant_id"` 36 | QueueID int64 `db:"queue_id"` 37 | 38 | DeliverAt int `db:"deliver_at"` 39 | DeliveredAt int `db:"delivered_at"` 40 | Tries int `db:"tries"` 41 | MaxTries int `db:"max_tries"` 42 | // RequeueIn int `db:"requeue_in"` 43 | 44 | // Status MessageStatus `db:"status"` 45 | 46 | Message []byte `db:"message"` 47 | KeyValues map[string]string 48 | } 49 | 50 | func (m *Message) Status() MessageStatus { 51 | now := time.Now().UTC().Unix() 52 | 53 | if m.Tries == m.MaxTries && now > int64(m.DeliverAt) { 54 | return MessageStatusFailed 55 | } 56 | 57 | if now >= int64(m.DeliveredAt) && now < int64(m.DeliverAt) { 58 | return MessageStatusDequeued 59 | } 60 | 61 | return MessageStatusQueued 62 | } 63 | 64 | func (m *Message) IsB64() bool { 65 | _, err := base64.StdEncoding.DecodeString(string(m.Message)) 66 | return err == nil 67 | } 68 | 69 | func (m *Message) Base64Decode() []byte { 70 | data, err := base64.StdEncoding.DecodeString(string(m.Message)) 71 | if err != nil { 72 | return m.Message 73 | } 74 | 75 | return data 76 | } 77 | 78 | func (m *Message) IsJSON() bool { 79 | return json.Valid(m.Message) 80 | } 81 | -------------------------------------------------------------------------------- /models/queue.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import "errors" 4 | 5 | var ErrInvalidQueueName = errors.New("Invalid queue name") 6 | var ErrQueueExists = errors.New("Queue already exists") 7 | 8 | type FilterCriteria struct { 9 | MessageID int64 10 | 11 | // 0 means unbounded 12 | DeliverAtStart int 13 | DeliverAtEnd int 14 | 15 | // status is an OR filter 16 | Status []MessageStatus 17 | 18 | // kv is an AND filter 19 | KV map[string]string 20 | 21 | // Smallest message ID to return. Message IDs are Snowflake IDs 22 | MinMessageID int64 23 | 24 | // How many message IDs to return 25 | Limit int 26 | } 27 | 28 | type QueueProperties struct { 29 | Name string 30 | RateLimit float64 31 | MaxRetries int 32 | VisibilityTimeout int 33 | } 34 | 35 | type Queue interface { 36 | GetQueue(tenantId int64, queueName string) (QueueProperties, error) 37 | CreateQueue(tenantId int64, properties QueueProperties) error 38 | UpdateQueue(tenantId int64, queue string, properties QueueProperties) error 39 | DeleteQueue(tenantId int64, queue string) error 40 | ListQueues(tenantId int64) ([]string, error) 41 | 42 | Enqueue(tenantId int64, queue string, message string, kv map[string]string, delay int) (int64, error) 43 | Dequeue(tenantId int64, queue string, numToDequeue int, requeueIn int) ([]*Message, error) 44 | UpdateMessage(tenantId int64, queue string, messageId int64, m *Message) error 45 | 46 | Peek(tenantId int64, queue string, messageId int64) *Message 47 | Stats(tenantId int64, queue string) QueueStats 48 | Filter(tenantId int64, queue string, filterCriteria FilterCriteria) []int64 49 | 50 | Delete(tenantId int64, queue string, messageId int64) error 51 | 52 | Shutdown() error 53 | } 54 | -------------------------------------------------------------------------------- /models/stats.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | type QueueStats struct { 4 | Counts map[MessageStatus]int 5 | TotalMessages int 6 | 7 | // TODO: Sliding window of queue rates 8 | // ProduceRate float64 9 | // ConsumeRate float64 10 | } 11 | -------------------------------------------------------------------------------- /models/tenant.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import "net/http" 4 | 5 | type TenantManager interface { 6 | GetTenant(r *http.Request) (int64, error) 7 | GetAWSSecretKey(accessKey string, region string) (tenantId int64, secretKey string, err error) 8 | } 9 | -------------------------------------------------------------------------------- /protocols/sqs/errors.go: -------------------------------------------------------------------------------- 1 | package sqs 2 | 3 | import "fmt" 4 | 5 | // SQS error response format: https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-json-api-responses.html#sqs-api-error-response-structure 6 | // Common errors across all requests: https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/CommonErrors.html 7 | // Individual methods have errors: https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_GetQueueUrl.html#API_GetQueueUrl_Errors 8 | 9 | type SQSError struct { 10 | Code int `json:"-"` 11 | Type string `json:"__type"` 12 | Message string `json:"message"` 13 | } 14 | 15 | func (e *SQSError) Error() string { 16 | return fmt.Sprintf("%s: %s", e.Type, e.Message) 17 | } 18 | 19 | func NewSQSError(code int, errType string, message string) *SQSError { 20 | return &SQSError{Code: code, Type: errType, Message: message} 21 | } 22 | 23 | var ErrIncompleteSignature = NewSQSError(400, "IncompleteSignature", "The request signature does not conform to AWS standards.") 24 | var ErrInvalidClientTokenId = NewSQSError(403, "InvalidClientTokenId", "The security token included in the request is invalid") 25 | var ErrQueueDoesNotExist = NewSQSError(400, "QueueDoesNotExist", "Queue does not exist") 26 | var ErrValidationError = NewSQSError(400, "ValidationError", "Invalid request payload") 27 | var ErrQueueNameExists = NewSQSError(400, "QueueNameExists", "Queue already exists") 28 | -------------------------------------------------------------------------------- /protocols/sqs/models.go: -------------------------------------------------------------------------------- 1 | package sqs 2 | 3 | type ChangeMessageVisibilityRequest struct { 4 | QueueUrl string `json:"QueueUrl"` 5 | ReceiptHandle string `json:"ReceiptHandle"` 6 | VisibilityTimeout int `json:"VisibilityTimeout"` 7 | } 8 | 9 | type ChangeMessageVisibilityResponse struct { 10 | } 11 | 12 | type SendMessagePayload struct { 13 | QueueUrl string `json:"QueueUrl"` 14 | MessageBody string `json:"MessageBody"` 15 | DelaySeconds int `json:"DelaySeconds,omitempty"` 16 | MessageAttributes map[string]MessageAttributeValue `json:"MessageAttributes,omitempty"` 17 | MessageDeduplicationId string `json:"MessageDeduplicationId,omitempty"` 18 | MessageGroupId string `json:"MessageGroupId,omitempty"` 19 | } 20 | 21 | type MessageAttributeValue struct { 22 | StringValue string `json:"StringValue,omitempty"` 23 | BinaryValue string `json:"BinaryValue,omitempty"` 24 | StringListValue []string `json:"StringListValue,omitempty"` 25 | BinaryListValue [][]byte `json:"BinaryListValue,omitempty"` 26 | DataType string `json:"DataType"` 27 | } 28 | 29 | type SendMessageResponse struct { 30 | MD5OfMessageBody string `json:"MD5OfMessageBody"` 31 | MD5OfMessageAttributes string `json:"MD5OfMessageAttributes"` 32 | MessageId string `json:"MessageId"` 33 | SequenceNumber string `json:"SequenceNumber,omitempty"` 34 | } 35 | 36 | type ReceiveMessageRequest struct { 37 | QueueUrl string `json:"QueueUrl"` 38 | AttributeNames []string `json:"AttributeNames,omitempty"` 39 | MessageAttributeNames []string `json:"MessageAttributeNames,omitempty"` 40 | MaxNumberOfMessages int `json:"MaxNumberOfMessages,omitempty"` 41 | VisibilityTimeout *int `json:"VisibilityTimeout,omitempty"` 42 | WaitTimeSeconds int `json:"WaitTimeSeconds,omitempty"` 43 | ReceiveRequestAttemptId string `json:"ReceiveRequestAttemptId,omitempty"` 44 | } 45 | 46 | type ReceiveMessageResponse struct { 47 | Messages []Message `json:"Messages"` 48 | } 49 | 50 | type Message struct { 51 | MessageId string `json:"MessageId"` 52 | ReceiptHandle string `json:"ReceiptHandle"` 53 | MD5OfBody string `json:"MD5OfBody"` 54 | Body string `json:"Body"` 55 | Attributes map[string]string `json:"Attributes"` 56 | MD5OfMessageAttributes string `json:"MD5OfMessageAttributes,omitempty"` 57 | MessageAttributes map[string]MessageAttribute `json:"MessageAttributes,omitempty"` 58 | } 59 | 60 | type MessageAttribute struct { 61 | StringValue string `json:"StringValue,omitempty"` 62 | BinaryValue []byte `json:"BinaryValue,omitempty"` 63 | StringListValues []string `json:"StringListValues,omitempty"` 64 | BinaryListValues [][]byte `json:"BinaryListValues,omitempty"` 65 | DataType string `json:"DataType"` 66 | } 67 | 68 | type DeleteMessageRequest struct { 69 | QueueUrl string `json:"QueueUrl"` 70 | ReceiptHandle string `json:"ReceiptHandle"` 71 | } 72 | 73 | type ListQueuesRequest struct { 74 | QueueNamePrefix string `json:"QueueNamePrefix,omitempty"` 75 | } 76 | 77 | type ListQueuesResponse struct { 78 | QueueUrls []string `json:"QueueUrls"` 79 | } 80 | 81 | type CreateQueueRequest struct { 82 | QueueName string `json:"QueueName"` 83 | Attributes map[string]string `json:"Attributes,omitempty"` 84 | Tags map[string]string `json:"Tags,omitempty"` 85 | } 86 | 87 | type CreateQueueResponse struct { 88 | QueueUrl string `json:"QueueUrl"` 89 | } 90 | 91 | type GetQueueAttributesRequest struct { 92 | QueueUrl string `json:"QueueUrl"` 93 | AttributeNames []string `json:"AttributeNames,omitempty"` 94 | } 95 | 96 | type GetQueueAttributesResponse struct { 97 | Attributes map[string]string `json:"Attributes"` 98 | } 99 | 100 | type PurgeQueueRequest struct { 101 | QueueUrl string `json:"QueueUrl"` 102 | } 103 | 104 | type PurgeQueueResponse struct { 105 | Success bool `json:"Success"` 106 | } 107 | 108 | type GetQueueURLRequest struct { 109 | QueueName string `json:"QueueName"` 110 | QueueOwnerAWSAccountId string `json:"QueueOwnerAWSAccountId"` 111 | } 112 | 113 | type GetQueueURLResponse struct { 114 | QueueURL string `json:"QueueUrl"` 115 | } 116 | 117 | // SendMessageBatchRequest represents the input for the SendMessageBatch operation. 118 | type SendMessageBatchRequest struct { 119 | QueueUrl string `json:"QueueUrl"` 120 | Entries []SendMessageBatchRequestEntry `json:"Entries"` 121 | } 122 | 123 | // SendMessageBatchRequestEntry represents an entry in the SendMessageBatch operation. 124 | type SendMessageBatchRequestEntry struct { 125 | ID string `json:"Id"` 126 | MessageBody string `json:"MessageBody"` 127 | DelaySeconds int `json:"DelaySeconds,omitempty"` 128 | MessageAttributes map[string]MessageAttributeValue `json:"MessageAttributes,omitempty"` 129 | MessageDeduplicationId string `json:"MessageDeduplicationId,omitempty"` 130 | MessageGroupId string `json:"MessageGroupId,omitempty"` 131 | } 132 | 133 | // SendMessageBatchResponse represents the output for the SendMessageBatch operation. 134 | type SendMessageBatchResponse struct { 135 | Successful []SendMessageBatchResultEntry `json:"Successful"` 136 | Failed []BatchResultErrorEntry `json:"Failed"` 137 | } 138 | 139 | // SendMessageBatchResultEntry represents a successful entry in the SendMessageBatch operation. 140 | type SendMessageBatchResultEntry struct { 141 | ID string `json:"Id"` 142 | MessageId string `json:"MessageId"` 143 | MD5OfMessageBody string `json:"MD5OfMessageBody"` 144 | MD5OfMessageAttributes string `json:"MD5OfMessageAttributes,omitempty"` 145 | SequenceNumber string `json:"SequenceNumber,omitempty"` 146 | } 147 | 148 | // BatchResultErrorEntry represents a failed entry in the SendMessageBatch operation. 149 | type BatchResultErrorEntry struct { 150 | ID string `json:"Id"` 151 | SenderFault bool `json:"SenderFault"` 152 | Code string `json:"Code"` 153 | Message string `json:"Message"` 154 | } 155 | -------------------------------------------------------------------------------- /protocols/sqs/sigv4.go: -------------------------------------------------------------------------------- 1 | package sqs 2 | 3 | import ( 4 | "crypto/hmac" 5 | "crypto/sha256" 6 | "encoding/hex" 7 | "errors" 8 | "fmt" 9 | "io" 10 | "net/http" 11 | "strings" 12 | ) 13 | 14 | type AuthHeader struct { 15 | Algorithm string 16 | AccessKey string 17 | Date string 18 | SignedHeaders []string 19 | Signature string 20 | Region string 21 | Service string 22 | } 23 | 24 | func ParseAuthorizationHeader(r *http.Request) (AuthHeader, error) { 25 | 26 | authHeader := r.Header.Get("Authorization") 27 | if authHeader == "" { 28 | return AuthHeader{}, errors.New("Authorization header is missing") 29 | } 30 | 31 | parts := strings.SplitN(authHeader, " ", 2) 32 | if len(parts) != 2 { 33 | return AuthHeader{}, errors.New("invalid Authorization header format") 34 | } 35 | 36 | rc := AuthHeader{} 37 | rc.Algorithm = parts[0] 38 | 39 | componentsString := parts[1] 40 | 41 | components := strings.Split(componentsString, ",") 42 | for _, componentString := range components { 43 | kv := strings.Split(strings.TrimSpace(componentString), "=") 44 | 45 | if strings.EqualFold(kv[0], "Credential") { 46 | tokens := strings.Split(kv[1], "/") 47 | rc.AccessKey = tokens[0] 48 | rc.Date = tokens[1] 49 | rc.Region = tokens[2] 50 | rc.Service = tokens[3] 51 | } else if strings.EqualFold(kv[0], "SignedHeaders") { 52 | rc.SignedHeaders = strings.Split(kv[1], ";") 53 | } else if strings.EqualFold(kv[0], "Signature") { 54 | rc.Signature = kv[1] 55 | } 56 | 57 | } 58 | 59 | return rc, nil 60 | } 61 | 62 | func getSignatureKey(key, dateStamp, region, service string) []byte { 63 | kSecret := []byte(fmt.Sprintf("AWS4%s", key)) 64 | kDate := sign(kSecret, dateStamp) 65 | kRegion := sign(kDate, region) 66 | kService := sign(kRegion, service) 67 | kSigning := sign(kService, "aws4_request") 68 | return kSigning 69 | } 70 | 71 | func sign(key []byte, message string) []byte { 72 | h := hmac.New(sha256.New, key) 73 | h.Write([]byte(message)) 74 | return h.Sum(nil) 75 | } 76 | 77 | func hashAndEncode(s string) string { 78 | h := sha256.New() 79 | h.Write([]byte(s)) 80 | return hex.EncodeToString(h.Sum(nil)) 81 | } 82 | 83 | func ValidateAWSRequest(awsCreds AuthHeader, secretKey string, r *http.Request) error { 84 | body, err := io.ReadAll(r.Body) 85 | if err != nil { 86 | return err 87 | } 88 | 89 | defer r.Body.Close() 90 | 91 | payloadHash := hashAndEncode(string(body)) 92 | 93 | canonicalHeaders := strings.Builder{} 94 | for _, header := range awsCreds.SignedHeaders { 95 | canonicalHeaders.WriteString(header) 96 | canonicalHeaders.WriteString(":") 97 | canonicalHeaders.WriteString(r.Header.Get(header)) 98 | canonicalHeaders.WriteString("\n") 99 | } 100 | 101 | signedHeaders := strings.Join(awsCreds.SignedHeaders, ";") 102 | 103 | canonicalRequest := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", 104 | strings.ToUpper(r.Method), r.URL, "", canonicalHeaders.String(), signedHeaders, payloadHash) 105 | 106 | canonicalRequestHash := hashAndEncode(canonicalRequest) 107 | credentialScope := fmt.Sprintf("%s/%s/sqs/aws4_request", 108 | awsCreds.Date, awsCreds.Region) 109 | 110 | stringToSign := fmt.Sprintf("%s\n%s\n%s\n%s", 111 | awsCreds.Algorithm, r.Header.Get("x-amz-date"), credentialScope, canonicalRequestHash) 112 | 113 | signingKey := getSignatureKey(secretKey, awsCreds.Date, awsCreds.Region, "sqs") 114 | 115 | signatureSHA := hmac.New(sha256.New, signingKey) 116 | signatureSHA.Write([]byte(stringToSign)) 117 | signatureString := hex.EncodeToString(signatureSHA.Sum(nil)) 118 | 119 | // authorizationHeader := fmt.Sprintf("%s Credential=%s/%s, SignedHeaders=%s, Signature=%s", awsCreds.Algorithm, awsCreds.AccessKey, credentialScope, signedHeaders, signatureString) 120 | 121 | if signatureString == awsCreds.Signature { 122 | return nil 123 | } 124 | 125 | return errors.New("invalid signature") 126 | } 127 | -------------------------------------------------------------------------------- /protocols/sqs/sqs.go: -------------------------------------------------------------------------------- 1 | package sqs 2 | 3 | /* 4 | Docs: 5 | https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_ReceiveMessage.html 6 | https://docs.aws.amazon.com/cli/latest/reference/sqs/delete-message.html 7 | 8 | Testing: 9 | AWS_ACCESS_KEY_ID=DEV_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=DEV_SECRET_ACCESS_KEY aws sqs ... 10 | aws sqs list-queues --endpoint-url http://localhost:3001 11 | aws sqs send-message --queue-url https://sqs.us-east-1.amazonaws.com/1/a --message-body "hello world" --endpoint-url http://localhost:3001 12 | aws sqs receive-message --queue-url https://sqs.us-east-1.amazonaws.com/1/a --endpoint-url http://localhost:3001 13 | aws sqs delete-message --receipt-handle x --queue-url https://sqs.us-east-1.amazonaws.com/1/a --endpoint-url http://localhost:3001 14 | aws sqs create-queue --queue-name b --endpoint-url http://localhost:3001 15 | aws sqs get-queue-attributes --queue-url https://sqs.us-east-1.amazonaws.com/1/a --endpoint-url http://localhost:3001 16 | aws sqs get-queue-url --debug --queue-name test-queue --endpoint-url http://localhost:3001 17 | */ 18 | 19 | import ( 20 | "crypto/md5" 21 | "encoding/base64" 22 | "encoding/hex" 23 | "encoding/json" 24 | "errors" 25 | "fmt" 26 | "regexp" 27 | "time" 28 | 29 | "net/http" 30 | "strconv" 31 | "strings" 32 | 33 | "github.com/poundifdef/smoothmq/config" 34 | "github.com/poundifdef/smoothmq/models" 35 | "github.com/prometheus/client_golang/prometheus" 36 | "github.com/prometheus/client_golang/prometheus/promauto" 37 | "github.com/tidwall/gjson" 38 | 39 | "github.com/gofiber/contrib/fiberzerolog" 40 | "github.com/gofiber/fiber/v2" 41 | "github.com/gofiber/fiber/v2/middleware/adaptor" 42 | "github.com/gofiber/fiber/v2/utils" 43 | "github.com/rs/zerolog" 44 | "github.com/rs/zerolog/log" 45 | "github.com/valyala/fasthttp/fasthttpadaptor" 46 | ) 47 | 48 | type SQS struct { 49 | App *fiber.App 50 | queue models.Queue 51 | tenantManager models.TenantManager 52 | 53 | cfg config.SQSConfig 54 | } 55 | 56 | var requestLatency = promauto.NewHistogramVec( 57 | prometheus.HistogramOpts{ 58 | Name: "sqs_request_latency", 59 | Help: "Latency of SQS requests", 60 | Buckets: prometheus.ExponentialBucketsRange(0.05, 1, 5), 61 | }, 62 | []string{"tenant_id", "aws_method"}, 63 | ) 64 | 65 | var requestStatus = promauto.NewCounterVec( 66 | prometheus.CounterOpts{ 67 | Name: "sqs_request_status", 68 | Help: "Status SQS requests", 69 | }, 70 | []string{"tenant_id", "aws_method", "status"}, 71 | ) 72 | 73 | func NewSQS(queue models.Queue, tenantManager models.TenantManager, cfg config.SQSConfig) *SQS { 74 | s := &SQS{ 75 | queue: queue, 76 | tenantManager: tenantManager, 77 | cfg: cfg, 78 | } 79 | 80 | app := fiber.New(fiber.Config{ 81 | DisableStartupMessage: true, 82 | ErrorHandler: s.errorHandler, 83 | BodyLimit: cfg.MaxRequestSize, 84 | }) 85 | 86 | app.Use(fiberzerolog.New(fiberzerolog.Config{ 87 | Logger: &log.Logger, 88 | Levels: []zerolog.Level{zerolog.ErrorLevel, zerolog.WarnLevel, zerolog.TraceLevel}, 89 | })) 90 | 91 | app.Use(s.authMiddleware) 92 | app.Post("/*", s.Action) 93 | 94 | s.App = app 95 | 96 | return s 97 | } 98 | 99 | func (s *SQS) errorHandler(c *fiber.Ctx, err error) error { 100 | sqsErr, ok := err.(*SQSError) 101 | if !ok { 102 | sqsErr = NewSQSError(500, "InternalFailure", err.Error()) 103 | } 104 | 105 | return c.Status(sqsErr.Code).JSON(sqsErr) 106 | } 107 | 108 | func (s *SQS) authMiddleware(c *fiber.Ctx) error { 109 | r, err := adaptor.ConvertRequest(c, false) 110 | if err != nil { 111 | return ErrIncompleteSignature 112 | } 113 | 114 | awsHeader, err := ParseAuthorizationHeader(r) 115 | if err != nil { 116 | return ErrIncompleteSignature 117 | } 118 | 119 | tenantId, secretKey, err := s.tenantManager.GetAWSSecretKey(awsHeader.AccessKey, awsHeader.Region) 120 | if err != nil { 121 | return ErrInvalidClientTokenId 122 | } 123 | 124 | err = ValidateAWSRequest(awsHeader, secretKey, r) 125 | if err != nil { 126 | return ErrIncompleteSignature 127 | } 128 | 129 | c.Locals("tenantId", tenantId) 130 | return c.Next() 131 | } 132 | 133 | func (s *SQS) Start() error { 134 | if !s.cfg.Enabled { 135 | return nil 136 | } 137 | 138 | fmt.Printf("SQS Endpoint: http://%s:%d\n", s.cfg.Host, s.cfg.Port) 139 | return s.App.Listen(fmt.Sprintf("%s:%d", s.cfg.Host, s.cfg.Port)) 140 | } 141 | 142 | func (s *SQS) Stop() error { 143 | if s.cfg.Enabled { 144 | return s.App.Shutdown() 145 | } 146 | return nil 147 | } 148 | 149 | func (s *SQS) Action(c *fiber.Ctx) error { 150 | log.Trace().Interface("headers", c.GetReqHeaders()).Bytes("body", c.Body()).Send() 151 | start := time.Now() 152 | 153 | awsMethodHeader, ok := c.GetReqHeaders()["X-Amz-Target"] 154 | if !ok { 155 | return errors.New("X-Amz-Target header not found") 156 | } 157 | awsMethod := awsMethodHeader[0] 158 | 159 | var r *http.Request = &http.Request{} 160 | fasthttpadaptor.ConvertRequest(c.Context(), r, false) 161 | 162 | tenantId := c.Locals("tenantId").(int64) 163 | 164 | defer func() { 165 | requestLatency.WithLabelValues(fmt.Sprintf("%d", tenantId), utils.CopyString(awsMethod)).Observe(time.Since(start).Seconds()) 166 | }() 167 | 168 | var rc error 169 | switch awsMethod { 170 | case "AmazonSQS.SendMessage": 171 | rc = s.SendMessage(c, tenantId) 172 | case "AmazonSQS.SendMessageBatch": 173 | rc = s.SendMessageBatch(c, tenantId) 174 | case "AmazonSQS.ReceiveMessage": 175 | rc = s.ReceiveMessage(c, tenantId) 176 | case "AmazonSQS.DeleteMessage": 177 | rc = s.DeleteMessage(c, tenantId) 178 | case "AmazonSQS.ListQueues": 179 | rc = s.ListQueues(c, tenantId) 180 | case "AmazonSQS.GetQueueUrl": 181 | rc = s.GetQueueURL(c, tenantId) 182 | case "AmazonSQS.CreateQueue": 183 | rc = s.CreateQueue(c, tenantId) 184 | case "AmazonSQS.GetQueueAttributes": 185 | rc = s.GetQueueAttributes(c, tenantId) 186 | case "AmazonSQS.PurgeQueue": 187 | rc = s.PurgeQueue(c, tenantId) 188 | case "AmazonSQS.ChangeMessageVisibility": 189 | rc = s.ChangeMessageVisibility(c, tenantId) 190 | default: 191 | rc = NewSQSError(400, "UnsupportedOperation", fmt.Sprintf("SQS method %s not implemented", awsMethod)) 192 | } 193 | 194 | status := "ok" 195 | if rc != nil { 196 | status = "error" 197 | } 198 | requestStatus.WithLabelValues(fmt.Sprintf("%d", tenantId), utils.CopyString(awsMethod), status).Inc() 199 | 200 | return rc 201 | } 202 | 203 | func (s *SQS) PurgeQueue(c *fiber.Ctx, tenantId int64) error { 204 | req := &PurgeQueueRequest{} 205 | 206 | err := json.Unmarshal(c.Body(), req) 207 | if err != nil { 208 | return err 209 | } 210 | 211 | tokens := strings.Split(req.QueueUrl, "/") 212 | queue := tokens[len(tokens)-1] 213 | 214 | messages := s.queue.Filter(tenantId, queue, models.FilterCriteria{}) 215 | for _, msg := range messages { 216 | s.queue.Delete(tenantId, queue, msg) 217 | } 218 | 219 | rc := PurgeQueueResponse{ 220 | Success: true, 221 | } 222 | 223 | return c.JSON(rc) 224 | } 225 | 226 | func (s *SQS) GetQueueAttributes(c *fiber.Ctx, tenantId int64) error { 227 | req := &GetQueueAttributesRequest{} 228 | 229 | err := json.Unmarshal(c.Body(), req) 230 | if err != nil { 231 | return err 232 | } 233 | 234 | tokens := strings.Split(req.QueueUrl, "/") 235 | queue := tokens[len(tokens)-1] 236 | 237 | stats := s.queue.Stats(tenantId, queue) 238 | 239 | rc := GetQueueAttributesResponse{ 240 | Attributes: map[string]string{ 241 | "ApproximateNumberOfMessages": fmt.Sprintf("%d", stats.TotalMessages), 242 | }, 243 | } 244 | 245 | return c.JSON(rc) 246 | } 247 | 248 | func (s *SQS) CreateQueue(c *fiber.Ctx, tenantId int64) error { 249 | req := &CreateQueueRequest{} 250 | 251 | err := json.Unmarshal(c.Body(), req) 252 | if err != nil { 253 | return err 254 | } 255 | 256 | if len(req.QueueName) > 80 { 257 | return ErrValidationError 258 | } 259 | 260 | regex, err := regexp.Compile(`^[a-zA-Z0-9-_]+$`) 261 | if err != nil { 262 | return err 263 | } 264 | 265 | if !regex.MatchString(req.QueueName) { 266 | return ErrValidationError 267 | } 268 | 269 | properties := models.QueueProperties{ 270 | Name: req.QueueName, 271 | RateLimit: -1, 272 | MaxRetries: -1, 273 | VisibilityTimeout: 30, 274 | } 275 | err = s.queue.CreateQueue(tenantId, properties) 276 | 277 | if errors.Is(err, models.ErrQueueExists) { 278 | return ErrQueueNameExists 279 | } 280 | 281 | if err != nil { 282 | return err 283 | } 284 | 285 | rc := CreateQueueResponse{ 286 | QueueUrl: s.queueURL(c, tenantId, req.QueueName), 287 | } 288 | 289 | return c.JSON(rc) 290 | } 291 | 292 | func (s *SQS) queueURL(c *fiber.Ctx, tenantId int64, queue string) string { 293 | host := fmt.Sprintf("%s://%s", c.Protocol(), c.Hostname()) 294 | 295 | if s.cfg.EndpointOverride != "" { 296 | host = s.cfg.EndpointOverride 297 | } 298 | 299 | return fmt.Sprintf("%s/%d/%s", host, tenantId, queue) 300 | 301 | } 302 | func (s *SQS) ListQueues(c *fiber.Ctx, tenantId int64) error { 303 | req := &ListQueuesRequest{} 304 | err := json.Unmarshal(c.Body(), req) 305 | if err != nil { 306 | return err 307 | } 308 | 309 | queues, err := s.queue.ListQueues(tenantId) 310 | if err != nil { 311 | return err 312 | } 313 | 314 | queueUrls := make([]string, 0) 315 | 316 | for _, queue := range queues { 317 | if req.QueueNamePrefix != "" && !strings.HasPrefix(queue, req.QueueNamePrefix) { 318 | continue 319 | } 320 | queueUrls = append(queueUrls, s.queueURL(c, tenantId, queue)) 321 | } 322 | 323 | rc := ListQueuesResponse{ 324 | QueueUrls: queueUrls, 325 | } 326 | 327 | return c.JSON(rc) 328 | } 329 | 330 | func (s *SQS) GetQueueURL(c *fiber.Ctx, tenantId int64) error { 331 | req := &GetQueueURLRequest{} 332 | 333 | err := json.Unmarshal(c.Body(), req) 334 | if err != nil { 335 | return ErrValidationError 336 | } 337 | 338 | queues, err := s.queue.ListQueues(tenantId) 339 | if err != nil { 340 | return err 341 | } 342 | 343 | for _, q := range queues { 344 | if q == req.QueueName { 345 | response := GetQueueURLResponse{ 346 | QueueURL: s.queueURL(c, tenantId, q), 347 | } 348 | return c.JSON(response) 349 | } 350 | } 351 | 352 | return ErrQueueDoesNotExist 353 | } 354 | 355 | func (s *SQS) SendMessage(c *fiber.Ctx, tenantId int64) error { 356 | req := &SendMessagePayload{} 357 | 358 | err := json.Unmarshal(c.Body(), req) 359 | if err != nil { 360 | return err 361 | } 362 | 363 | tokens := strings.Split(req.QueueUrl, "/") 364 | queue := tokens[len(tokens)-1] 365 | 366 | kv := make(map[string]string) 367 | for k, v := range req.MessageAttributes { 368 | kv[k+"_DataType"] = v.DataType 369 | if v.DataType == "String" { 370 | kv[k] = v.StringValue 371 | } else if v.DataType == "Number" { 372 | kv[k] = v.StringValue 373 | } else if v.DataType == "Binary" { 374 | kv[k] = v.BinaryValue 375 | } 376 | } 377 | 378 | // Try to parse celery task and ID 379 | if s.cfg.ParseCelery { 380 | // Is our message a JSON string? 381 | if strings.HasPrefix(req.MessageBody, "ey") { 382 | jsonStr, err := base64.StdEncoding.DecodeString(req.MessageBody) 383 | 384 | if err == nil { 385 | res := gjson.GetBytes(jsonStr, "headers.task") 386 | if res.Exists() { 387 | kv["celery_task"] = res.Str 388 | } 389 | 390 | res = gjson.GetBytes(jsonStr, "headers.id") 391 | if res.Exists() { 392 | kv["celery_id"] = res.Str 393 | } 394 | } 395 | } 396 | } 397 | 398 | messageId, err := s.queue.Enqueue(tenantId, queue, req.MessageBody, kv, req.DelaySeconds) 399 | if err != nil { 400 | return err 401 | } 402 | 403 | hasher := md5.New() 404 | hasher.Write([]byte(req.MessageBody)) 405 | 406 | response := SendMessageResponse{ 407 | MessageId: fmt.Sprintf("%d", messageId), 408 | MD5OfMessageBody: hex.EncodeToString(hasher.Sum(nil)), 409 | } 410 | 411 | return c.JSON(response) 412 | } 413 | 414 | func (s *SQS) SendMessageBatch(c *fiber.Ctx, tenantId int64) error { 415 | batchReq := &SendMessageBatchRequest{} 416 | 417 | err := json.Unmarshal(c.Body(), batchReq) 418 | if err != nil { 419 | return err 420 | } 421 | 422 | tokens := strings.Split(batchReq.QueueUrl, "/") 423 | queue := tokens[len(tokens)-1] 424 | 425 | response := &SendMessageBatchResponse{} 426 | 427 | for _, req := range batchReq.Entries { 428 | 429 | kv := make(map[string]string) 430 | for k, v := range req.MessageAttributes { 431 | kv[k+"_DataType"] = v.DataType 432 | if v.DataType == "String" { 433 | kv[k] = v.StringValue 434 | } else if v.DataType == "Number" { 435 | kv[k] = v.StringValue 436 | } else if v.DataType == "Binary" { 437 | kv[k] = v.BinaryValue 438 | } 439 | } 440 | 441 | // Try to parse celery task and ID 442 | if s.cfg.ParseCelery { 443 | // Is our message a JSON string? 444 | if strings.HasPrefix(req.MessageBody, "ey") { 445 | jsonStr, err := base64.StdEncoding.DecodeString(req.MessageBody) 446 | 447 | if err == nil { 448 | res := gjson.GetBytes(jsonStr, "headers.task") 449 | if res.Exists() { 450 | kv["celery_task"] = res.Str 451 | } 452 | 453 | res = gjson.GetBytes(jsonStr, "headers.id") 454 | if res.Exists() { 455 | kv["celery_id"] = res.Str 456 | } 457 | } 458 | } 459 | } 460 | 461 | messageId, err := s.queue.Enqueue(tenantId, queue, req.MessageBody, kv, req.DelaySeconds) 462 | 463 | if err == nil { 464 | hasher := md5.New() 465 | hasher.Write([]byte(req.MessageBody)) 466 | 467 | response.Successful = append(response.Successful, SendMessageBatchResultEntry{ 468 | ID: req.ID, 469 | MessageId: fmt.Sprintf("%d", messageId), 470 | MD5OfMessageBody: hex.EncodeToString(hasher.Sum(nil)), 471 | }) 472 | } else { 473 | response.Failed = append(response.Failed, BatchResultErrorEntry{ 474 | ID: req.ID, 475 | Code: "InternalFailure", 476 | Message: err.Error(), 477 | }) 478 | } 479 | } 480 | 481 | return c.JSON(response) 482 | } 483 | 484 | func (s *SQS) ReceiveMessage(c *fiber.Ctx, tenantId int64) error { 485 | req := &ReceiveMessageRequest{} 486 | 487 | err := json.Unmarshal(c.Body(), req) 488 | if err != nil { 489 | return err 490 | } 491 | 492 | maxNumberOfMessages := req.MaxNumberOfMessages 493 | if maxNumberOfMessages == 0 { 494 | maxNumberOfMessages = 1 495 | } 496 | 497 | // log.Println(req) 498 | 499 | tokens := strings.Split(req.QueueUrl, "/") 500 | queue := tokens[len(tokens)-1] 501 | 502 | visibilityTimeout := -1 503 | if req.VisibilityTimeout != nil { 504 | visibilityTimeout = *req.VisibilityTimeout 505 | } 506 | 507 | var messages []*models.Message 508 | 509 | waitTime := min(req.WaitTimeSeconds, s.cfg.MaxDelaySeconds) 510 | maxRequestTime := time.Now().Add(time.Duration(waitTime) * time.Second) 511 | sleepIncrement := time.Duration(s.cfg.DelayRetryMillis) * time.Millisecond 512 | 513 | for { 514 | messages, err = s.queue.Dequeue(tenantId, queue, maxNumberOfMessages, visibilityTimeout) 515 | if err != nil { 516 | return err 517 | } 518 | 519 | if len(messages) > 0 { 520 | break 521 | } 522 | 523 | if time.Now().Equal(maxRequestTime) || time.Now().After(maxRequestTime) { 524 | break 525 | } 526 | 527 | time.Sleep(sleepIncrement) 528 | } 529 | 530 | response := ReceiveMessageResponse{ 531 | Messages: make([]Message, len(messages)), 532 | } 533 | 534 | hasher := md5.New() 535 | 536 | for i, message := range messages { 537 | hasher.Reset() 538 | hasher.Write(message.Message) 539 | 540 | response.Messages[i] = Message{ 541 | MessageId: fmt.Sprintf("%d", message.ID), 542 | ReceiptHandle: fmt.Sprintf("%d", message.ID), 543 | Body: string(message.Message), 544 | MessageAttributes: make(map[string]MessageAttribute), 545 | MD5OfBody: hex.EncodeToString(hasher.Sum(nil)), 546 | } 547 | 548 | for k, v := range message.KeyValues { 549 | if strings.HasSuffix(k, "_DataType") { 550 | continue 551 | } 552 | attr := MessageAttribute{ 553 | DataType: message.KeyValues[k+"_DataType"], 554 | } 555 | if attr.DataType == "String" { 556 | attr.StringValue = v 557 | } else if attr.DataType == "Number" { 558 | attr.StringValue = v 559 | } else if attr.DataType == "Binary" { 560 | data, err := base64.StdEncoding.DecodeString(v) 561 | if err != nil { 562 | log.Trace().Int64("message_id", message.ID).Err(err).Msg("Unable to decode binary SQS attribute") 563 | } else { 564 | attr.BinaryValue = data 565 | } 566 | } 567 | 568 | response.Messages[i].MessageAttributes[k] = attr 569 | } 570 | } 571 | 572 | return c.JSON(response) 573 | } 574 | 575 | func (s *SQS) DeleteMessage(c *fiber.Ctx, tenantId int64) error { 576 | req := &DeleteMessageRequest{} 577 | 578 | err := json.Unmarshal(c.Body(), req) 579 | if err != nil { 580 | return err 581 | } 582 | 583 | tokens := strings.Split(req.QueueUrl, "/") 584 | queue := tokens[len(tokens)-1] 585 | 586 | messageId, err := strconv.ParseInt(req.ReceiptHandle, 10, 64) 587 | if err != nil { 588 | return err 589 | } 590 | 591 | err = s.queue.Delete(tenantId, queue, messageId) 592 | if err != nil { 593 | return err 594 | } 595 | 596 | return nil 597 | } 598 | 599 | func (s *SQS) ChangeMessageVisibility(c *fiber.Ctx, tenantId int64) error { 600 | req := &ChangeMessageVisibilityRequest{} 601 | 602 | err := json.Unmarshal(c.Body(), req) 603 | if err != nil { 604 | return err 605 | } 606 | 607 | tokens := strings.Split(req.QueueUrl, "/") 608 | queue := tokens[len(tokens)-1] 609 | 610 | messageId, err := strconv.ParseInt(req.ReceiptHandle, 10, 64) 611 | if err != nil { 612 | return err 613 | } 614 | 615 | // Fetch the message 616 | message := s.queue.Peek(tenantId, queue, messageId) 617 | if message == nil { 618 | return NewSQSError(400, "InvalidAddress", "The specified message does not exist.") 619 | } 620 | 621 | if message.Status() != models.MessageStatusDequeued { 622 | return NewSQSError(400, "MessageNotInFlight", "The message is not in flight") 623 | } 624 | 625 | // Calculate the new delivery time in Unix timestamp (seconds) 626 | newDeliverAt := time.Now().Add(time.Duration(req.VisibilityTimeout) * time.Second).Unix() 627 | 628 | // Update the message's DeliverAt time 629 | message.DeliverAt = int(newDeliverAt) 630 | 631 | // Update the message in the queue 632 | err = s.queue.UpdateMessage(tenantId, queue, messageId, message) 633 | if err != nil { 634 | return err 635 | } 636 | 637 | // Return an empty response as per SQS specification 638 | return c.JSON(ChangeMessageVisibilityResponse{}) 639 | } 640 | -------------------------------------------------------------------------------- /queue/sqlite/sqlite.go: -------------------------------------------------------------------------------- 1 | package sqlite 2 | 3 | import ( 4 | "errors" 5 | "os" 6 | "regexp" 7 | "strings" 8 | "sync" 9 | "time" 10 | 11 | "github.com/poundifdef/smoothmq/config" 12 | "github.com/poundifdef/smoothmq/models" 13 | 14 | "github.com/rs/zerolog/log" 15 | 16 | "github.com/prometheus/client_golang/prometheus" 17 | "github.com/prometheus/client_golang/prometheus/promauto" 18 | 19 | "github.com/bwmarrin/snowflake" 20 | _ "github.com/mattn/go-sqlite3" 21 | 22 | "gorm.io/driver/sqlite" 23 | "gorm.io/gorm" 24 | "gorm.io/gorm/clause" 25 | ) 26 | 27 | type SQLiteQueue struct { 28 | Filename string 29 | DBG *gorm.DB 30 | Mu *sync.Mutex 31 | snow *snowflake.Node 32 | ticker *time.Ticker 33 | } 34 | 35 | var queueDiskSize = promauto.NewGauge( 36 | prometheus.GaugeOpts{ 37 | Name: "queue_disk_size", 38 | Help: "Size of queue data on disk", 39 | }, 40 | ) 41 | 42 | type Queue struct { 43 | ID int64 `gorm:"primaryKey;autoIncrement:false"` 44 | TenantID int64 `gorm:"not null;index:idx_queue_name,priority:1,unique"` 45 | Name string `gorm:"not null;index:idx_queue_name,priority:2"` 46 | RateLimit float64 `gorm:"not null"` 47 | MaxRetries int `gorm:"not null"` 48 | VisibilityTimeout int `gorm:"not null"` 49 | 50 | Messages []Message `gorm:"foreignKey:QueueID;references:ID"` 51 | } 52 | 53 | type Message struct { 54 | ID int64 `gorm:"primaryKey;autoIncrement:false"` 55 | TenantID int64 `gorm:"not null;index:idx_message,priority:1;not null"` 56 | QueueID int64 `gorm:"not null;index:idx_message,priority:2;not null"` 57 | 58 | DeliverAt int64 `gorm:"not null;index:idx_message,priority:3;not null"` 59 | DeliveredAt int64 `gorm:"not null;index:idx_message,priority:4;not null"` 60 | Tries int `gorm:"not null;index:idx_message,priority:5;not null"` 61 | MaxTries int `gorm:"not null;index:idx_message,priority:6;not null"` 62 | 63 | Message string `gorm:"not null"` 64 | 65 | KV []KV `gorm:"foreignKey:TenantID,QueueID,MessageID;references:TenantID,QueueID,ID"` 66 | } 67 | 68 | func (message *Message) ToModel() *models.Message { 69 | rc := &models.Message{ 70 | ID: message.ID, 71 | TenantID: message.TenantID, 72 | QueueID: message.QueueID, 73 | 74 | DeliverAt: int(message.DeliverAt), 75 | DeliveredAt: int(message.DeliveredAt), 76 | Tries: message.Tries, 77 | MaxTries: message.MaxTries, 78 | 79 | Message: []byte(message.Message), 80 | KeyValues: make(map[string]string), 81 | } 82 | 83 | for _, kv := range message.KV { 84 | rc.KeyValues[kv.K] = kv.V 85 | } 86 | 87 | return rc 88 | } 89 | 90 | type KV struct { 91 | TenantID int64 `gorm:"not null;index:idx_kv,priority:1"` 92 | QueueID int64 `gorm:"not null;index:idx_kv,priority:2"` 93 | MessageID int64 `gorm:"not null;index:idx_kv,priority:3"` 94 | K string `gorm:"not null"` 95 | V string `gorm:"not null"` 96 | } 97 | 98 | type RateLimit struct { 99 | TenantID int64 `gorm:"not null;index:idx_ratelimit,priority:1,unique"` 100 | QueueID int64 `gorm:"not null;index:idx_ratelimit,priority:2"` 101 | Ts int64 `gorm:"not null;index:idx_ratelimit,priority:3"` 102 | N int `gorm:"not null;default:0"` 103 | } 104 | 105 | func NewSQLiteQueue(cfg config.SQLiteConfig) *SQLiteQueue { 106 | snow, err := snowflake.NewNode(1) 107 | if err != nil { 108 | log.Fatal().Err(err).Send() 109 | } 110 | 111 | db, err := gorm.Open(sqlite.Open(cfg.Path+"?_journal_mode=WAL&_foreign_keys=off&_auto_vacuum=full"), &gorm.Config{TranslateError: true}) 112 | if err != nil { 113 | log.Fatal().Err(err).Send() 114 | } 115 | 116 | err = db.AutoMigrate(&Queue{}) 117 | if err != nil { 118 | log.Fatal().Err(err).Send() 119 | } 120 | 121 | err = db.AutoMigrate(&Message{}) 122 | if err != nil { 123 | log.Fatal().Err(err).Send() 124 | } 125 | 126 | err = db.AutoMigrate(&KV{}) 127 | if err != nil { 128 | log.Fatal().Err(err).Send() 129 | } 130 | 131 | err = db.AutoMigrate(&RateLimit{}) 132 | if err != nil { 133 | log.Fatal().Err(err).Send() 134 | } 135 | 136 | rc := &SQLiteQueue{ 137 | Filename: cfg.Path, 138 | DBG: db, 139 | Mu: &sync.Mutex{}, 140 | snow: snow, 141 | ticker: time.NewTicker(1 * time.Second), 142 | } 143 | 144 | go func() { 145 | for { 146 | select { 147 | case <-rc.ticker.C: 148 | stat, err := os.Stat(rc.Filename) 149 | if err == nil { 150 | queueDiskSize.Set(float64(stat.Size())) 151 | } 152 | } 153 | } 154 | }() 155 | 156 | return rc 157 | } 158 | 159 | func (q *SQLiteQueue) CreateQueue(tenantId int64, properties models.QueueProperties) error { 160 | q.Mu.Lock() 161 | defer q.Mu.Unlock() 162 | 163 | if len(properties.Name) > 80 { 164 | return models.ErrInvalidQueueName 165 | } 166 | 167 | regex, err := regexp.Compile(`^[a-zA-Z0-9-_]+$`) 168 | if err != nil { 169 | return err 170 | } 171 | 172 | var nameTrimmed = strings.TrimSpace(properties.Name) 173 | if !regex.MatchString(nameTrimmed) { 174 | return models.ErrInvalidQueueName 175 | } 176 | 177 | qId := q.snow.Generate() 178 | 179 | res := q.DBG.Create(&Queue{ 180 | ID: qId.Int64(), 181 | TenantID: tenantId, 182 | Name: nameTrimmed, 183 | RateLimit: properties.RateLimit, 184 | MaxRetries: properties.MaxRetries, 185 | VisibilityTimeout: properties.VisibilityTimeout, 186 | }) 187 | 188 | if errors.Is(res.Error, gorm.ErrDuplicatedKey) { 189 | return models.ErrQueueExists 190 | } 191 | 192 | return res.Error 193 | } 194 | 195 | func (q *SQLiteQueue) UpdateQueue(tenantId int64, queueName string, properties models.QueueProperties) error { 196 | q.Mu.Lock() 197 | defer q.Mu.Unlock() 198 | 199 | // TODO: validate, trim queue names. ensure length and valid characters. 200 | 201 | queue, err := q.getQueue(tenantId, queueName) 202 | if err != nil { 203 | return err 204 | } 205 | 206 | queue.RateLimit = properties.RateLimit 207 | queue.MaxRetries = properties.MaxRetries 208 | queue.VisibilityTimeout = properties.VisibilityTimeout 209 | 210 | // TODO: should we reset the rate limit table if the queue's rate limit changes? 211 | 212 | res := q.DBG.Save(queue) 213 | 214 | return res.Error 215 | } 216 | 217 | func (q *SQLiteQueue) GetQueue(tenantId int64, queueName string) (models.QueueProperties, error) { 218 | queue := models.QueueProperties{} 219 | 220 | properties, err := q.getQueue(tenantId, queueName) 221 | if err != nil { 222 | return queue, err 223 | } 224 | 225 | queue.Name = properties.Name 226 | queue.RateLimit = properties.RateLimit 227 | queue.MaxRetries = properties.MaxRetries 228 | queue.VisibilityTimeout = properties.VisibilityTimeout 229 | 230 | return queue, nil 231 | } 232 | 233 | func (q *SQLiteQueue) DeleteQueue(tenantId int64, queueName string) error { 234 | // Delete all messages with the queue, and then the queue itself 235 | 236 | queue, err := q.getQueue(tenantId, queueName) 237 | if err != nil { 238 | return err 239 | } 240 | 241 | q.Mu.Lock() 242 | defer q.Mu.Unlock() 243 | 244 | rc := q.DBG.Transaction(func(tx *gorm.DB) error { 245 | if err := tx.Where("tenant_id = ? AND queue_id = ?", tenantId, queue.ID).Delete(&KV{}).Error; err != nil { 246 | return err 247 | } 248 | 249 | if err := tx.Where("tenant_id = ? AND queue_id = ?", tenantId, queue.ID).Delete(&Message{}).Error; err != nil { 250 | return err 251 | } 252 | 253 | if err := tx.Where("tenant_id = ? AND id = ?", tenantId, queue.ID).Delete(&Queue{}).Error; err != nil { 254 | return err 255 | } 256 | 257 | return nil 258 | }) 259 | 260 | return rc 261 | } 262 | 263 | func (q *SQLiteQueue) ListQueues(tenantId int64) ([]string, error) { 264 | var queues []Queue 265 | res := q.DBG.Where("tenant_id = ?", tenantId).Select("name").Find(&queues) 266 | if res.Error != nil { 267 | return nil, res.Error 268 | } 269 | 270 | rc := make([]string, len(queues)) 271 | for i, queue := range queues { 272 | rc[i] = queue.Name 273 | } 274 | 275 | return rc, nil 276 | } 277 | 278 | func (q *SQLiteQueue) getQueue(tenantId int64, queueName string) (*Queue, error) { 279 | rc := &Queue{} 280 | res := q.DBG.Where("tenant_id = ? AND name = ?", tenantId, queueName).First(rc) 281 | if res.RowsAffected != 1 { 282 | return nil, errors.New("Queue not found") 283 | } 284 | return rc, res.Error 285 | } 286 | 287 | func (q *SQLiteQueue) Enqueue(tenantId int64, queueName string, message string, kv map[string]string, delay int) (int64, error) { 288 | messageSnow := q.snow.Generate() 289 | messageId := messageSnow.Int64() 290 | 291 | queue, err := q.getQueue(tenantId, queueName) 292 | if err != nil { 293 | return 0, err 294 | } 295 | 296 | now := time.Now().UTC().Unix() 297 | deliverAt := now + int64(delay) 298 | 299 | newMessage := &Message{ 300 | ID: messageId, 301 | TenantID: tenantId, 302 | QueueID: queue.ID, 303 | DeliverAt: deliverAt, 304 | DeliveredAt: 0, 305 | MaxTries: queue.MaxRetries, 306 | Message: message, 307 | 308 | KV: make([]KV, 0), 309 | } 310 | 311 | for k, v := range kv { 312 | newKv := KV{ 313 | TenantID: tenantId, 314 | MessageID: messageId, 315 | QueueID: queue.ID, 316 | K: k, 317 | V: v, 318 | } 319 | newMessage.KV = append(newMessage.KV, newKv) 320 | } 321 | 322 | q.Mu.Lock() 323 | defer q.Mu.Unlock() 324 | 325 | if err := q.DBG.Create(newMessage).Error; err != nil { 326 | return 0, err 327 | } 328 | 329 | log.Debug().Int64("message_id", messageId).Msg("Enqueued message") 330 | 331 | return messageId, nil 332 | } 333 | 334 | func (q *SQLiteQueue) UpdateMessage(tenantId int64, queue string, messageId int64, m *models.Message) error { 335 | q.Mu.Lock() 336 | defer q.Mu.Unlock() 337 | 338 | result := q.DBG.Model(&Message{}). 339 | Where("tenant_id = ? AND queue_id = (SELECT id FROM queues WHERE tenant_id = ? AND name = ?) AND id = ?", tenantId, tenantId, queue, messageId). 340 | Update("deliver_at", m.DeliverAt) 341 | 342 | if result.Error != nil { 343 | return result.Error 344 | } 345 | 346 | if result.RowsAffected == 0 { 347 | return errors.New("message not found") 348 | } 349 | return nil 350 | } 351 | 352 | // Calculate how many messages to allow the user to dequeue based on queue's rate limit 353 | func (q *SQLiteQueue) calculateRateLimit(queue *Queue, now int64, numToDequeue int) (int, error) { 354 | maxToDequeue := numToDequeue 355 | bucketToCheck := now 356 | 357 | // If the rate limit is less than 1 per second, meaning 1 message every n seconds 358 | if queue.RateLimit > 0 && queue.RateLimit < 1 { 359 | // Go back n seconds, check how many messages have been sent in that time period 360 | 361 | limitReciprocal := 1.0 / queue.RateLimit 362 | earliestBucket := now - int64(limitReciprocal) + 1 363 | 364 | bucketToCheck = earliestBucket 365 | 366 | sql := ` 367 | SELECT coalesce(cast(sum(n) as real) / (? - min(ts) + 1),0) FROM rate_limits 368 | WHERE tenant_id = ? AND queue_id = ? AND ts >= ? 369 | ` 370 | 371 | res := q.DBG.Raw(sql, now, queue.TenantID, queue.ID, earliestBucket) 372 | if res.Error != nil { 373 | return 0, res.Error 374 | } 375 | 376 | var messageRate float64 377 | 378 | rateRes := res.Scan(&messageRate) 379 | if rateRes.Error != nil { 380 | return 0, rateRes.Error 381 | } 382 | 383 | if messageRate >= queue.RateLimit { 384 | return 0, nil 385 | } 386 | 387 | maxToDequeue = 1 388 | } else if queue.RateLimit >= 1 { 389 | // Check how many messages were sent in current bucket 390 | bucket := &RateLimit{} 391 | res := q.DBG.Where("tenant_id = ? AND queue_id = ? AND ts = ?", queue.TenantID, queue.ID, now).FirstOrCreate(bucket) 392 | 393 | if res.Error != nil && !errors.Is(res.Error, gorm.ErrDuplicatedKey) { 394 | return 0, res.Error 395 | } 396 | 397 | if bucket.N >= int(queue.RateLimit) { 398 | return 0, nil 399 | } 400 | 401 | allowedToDequeue := int(queue.RateLimit) - bucket.N 402 | if allowedToDequeue < 0 { 403 | allowedToDequeue = 0 404 | } 405 | 406 | maxToDequeue = min(numToDequeue, allowedToDequeue) 407 | if maxToDequeue <= 0 { 408 | return 0, nil 409 | } 410 | } 411 | 412 | if queue.RateLimit > 0 { 413 | res := q.DBG.Where("tenant_id = ? AND queue_id = ? AND ts < ? - 1", queue.TenantID, queue.ID, bucketToCheck).Delete(&RateLimit{}) 414 | if res.Error != nil { 415 | log.Warn().Int64("tenant_id", queue.TenantID).Int64("queue_id", queue.ID).Int64("ts", bucketToCheck).Msg("Unable to clear previous rate limits") 416 | } 417 | } 418 | 419 | return maxToDequeue, nil 420 | } 421 | 422 | func (q *SQLiteQueue) Dequeue(tenantId int64, queueName string, numToDequeue int, requeueIn int) ([]*models.Message, error) { 423 | queue, err := q.getQueue(tenantId, queueName) 424 | if err != nil { 425 | return nil, err 426 | } 427 | 428 | // Queue is "paused" 429 | if queue.RateLimit == 0 { 430 | return nil, nil 431 | } 432 | 433 | visibilityTimeout := queue.VisibilityTimeout 434 | if requeueIn > -1 { 435 | visibilityTimeout = requeueIn 436 | } 437 | 438 | now := time.Now().UTC().Unix() 439 | 440 | q.Mu.Lock() 441 | defer q.Mu.Unlock() 442 | 443 | maxToDequeue, err := q.calculateRateLimit(queue, now, numToDequeue) 444 | if err != nil { 445 | return nil, err 446 | } 447 | 448 | var messages []Message 449 | 450 | res := q.DBG.Preload("KV").Where( 451 | "deliver_at <= ? AND delivered_at <= ? AND (tries < max_tries OR max_tries = -1) AND tenant_id = ? AND queue_id = ?", 452 | now, now, tenantId, queue.ID). 453 | Limit(maxToDequeue). 454 | Find(&messages) 455 | 456 | if res.Error != nil { 457 | return nil, err 458 | } 459 | 460 | if len(messages) == 0 { 461 | return nil, nil 462 | } 463 | 464 | rc := make([]*models.Message, len(messages)) 465 | 466 | for i, message := range messages { 467 | rc[i] = message.ToModel() 468 | } 469 | 470 | messageIDs := make([]int64, len(rc)) 471 | for i, message := range rc { 472 | messageIDs[i] = message.ID 473 | } 474 | 475 | err = q.DBG.Transaction(func(tx *gorm.DB) error { 476 | res = tx.Model(&Message{}).Where("tenant_id = ? AND queue_id = ? AND id in ?", tenantId, queue.ID, messageIDs). 477 | UpdateColumns(map[string]any{ 478 | "tries": gorm.Expr("tries+1"), 479 | "delivered_at": now, 480 | "deliver_at": gorm.Expr("?", now+int64(visibilityTimeout)), 481 | }) 482 | 483 | if res.Error != nil { 484 | return res.Error 485 | } 486 | 487 | if queue.RateLimit > 0 { 488 | bucket := RateLimit{ 489 | TenantID: tenantId, 490 | QueueID: queue.ID, 491 | Ts: now, 492 | N: len(messageIDs), 493 | } 494 | res = tx.Clauses(clause.OnConflict{ 495 | DoUpdates: clause.Assignments(map[string]interface{}{"n": gorm.Expr("n + ?", len(messageIDs))}), 496 | }).Create(&bucket) 497 | 498 | if res.Error != nil && !errors.Is(res.Error, gorm.ErrDuplicatedKey) { 499 | return res.Error 500 | } 501 | } 502 | 503 | return nil 504 | }) 505 | 506 | if err != nil { 507 | return nil, err 508 | } 509 | 510 | for _, messageId := range messageIDs { 511 | log.Debug().Int64("message_id", messageId).Msg("Dequeued message") 512 | } 513 | 514 | return rc, nil 515 | } 516 | 517 | func (q *SQLiteQueue) Peek(tenantId int64, queueName string, messageId int64) *models.Message { 518 | queue, err := q.getQueue(tenantId, queueName) 519 | if err != nil { 520 | return nil 521 | } 522 | 523 | message := &Message{} 524 | 525 | res := q.DBG.Preload("KV").Where( 526 | "tenant_id = ? AND queue_id = ? AND id = ?", 527 | tenantId, queue.ID, messageId). 528 | First(message) 529 | 530 | if res.Error != nil { 531 | return nil 532 | } 533 | 534 | return message.ToModel() 535 | } 536 | 537 | func (q *SQLiteQueue) Stats(tenantId int64, queueName string) models.QueueStats { 538 | queue, err := q.getQueue(tenantId, queueName) 539 | if err != nil { 540 | return models.QueueStats{} 541 | } 542 | 543 | now := time.Now().UTC().Unix() 544 | 545 | res := q.DBG.Raw(` 546 | SELECT 547 | CASE 548 | WHEN tries = max_tries AND deliver_at < ? THEN 3 549 | WHEN delivered_at <= ? AND deliver_at > ? THEN 2 550 | ELSE 1 551 | END AS s, 552 | count(*) FROM messages WHERE queue_id=? AND tenant_id=? GROUP BY s 553 | `, now, now, now, 554 | queue.ID, tenantId, 555 | ) 556 | 557 | if res.Error != nil { 558 | return models.QueueStats{} 559 | } 560 | 561 | rows, err := res.Rows() 562 | if err != nil { 563 | return models.QueueStats{} 564 | } 565 | 566 | stats := models.QueueStats{ 567 | Counts: make(map[models.MessageStatus]int), 568 | TotalMessages: 0, 569 | } 570 | 571 | for rows.Next() { 572 | var statusType models.MessageStatus 573 | var count int 574 | 575 | rows.Scan(&statusType, &count) 576 | 577 | stats.TotalMessages += count 578 | stats.Counts[statusType] = count 579 | } 580 | 581 | rows.Close() 582 | 583 | return stats 584 | } 585 | 586 | func (q *SQLiteQueue) Filter(tenantId int64, queueName string, filterCriteria models.FilterCriteria) []int64 { 587 | var rc []int64 588 | 589 | queue, err := q.getQueue(tenantId, queueName) 590 | if err != nil { 591 | return nil 592 | } 593 | 594 | args := make([]any, 0) 595 | args = append(args, tenantId) 596 | args = append(args, queue.ID) 597 | 598 | sql := "SELECT id FROM messages WHERE tenant_id=? AND queue_id=? " 599 | 600 | if filterCriteria.MessageID > 0 { 601 | sql += " AND id = ? " 602 | args = append(args, filterCriteria.MessageID) 603 | } 604 | 605 | if len(filterCriteria.KV) > 0 { 606 | sql += " AND " 607 | sql += " id IN (SELECT message_id FROM kvs WHERE (" 608 | 609 | for i := range len(filterCriteria.KV) { 610 | sql += "(k=? AND v=? and tenant_id=? and queue_id=?)" 611 | 612 | if i < len(filterCriteria.KV)-1 { 613 | sql += " OR " 614 | } 615 | } 616 | 617 | sql += " ) GROUP BY message_id HAVING count(*) = ? LIMIT 10" 618 | sql += " ) " 619 | 620 | } 621 | 622 | for k, v := range filterCriteria.KV { 623 | args = append(args, k, v, tenantId, queue.ID) 624 | } 625 | 626 | args = append(args, len(filterCriteria.KV)) 627 | 628 | sql += "LIMIT 10" 629 | 630 | res := q.DBG.Raw(sql, args...).Scan(&rc) 631 | if res.Error != nil { 632 | log.Error().Err(res.Error).Msg("Unable to filter") 633 | } 634 | 635 | return rc 636 | } 637 | 638 | func (q *SQLiteQueue) Delete(tenantId int64, queueName string, messageId int64) error { 639 | queue, err := q.getQueue(tenantId, queueName) 640 | if err != nil { 641 | return err 642 | } 643 | 644 | q.Mu.Lock() 645 | defer q.Mu.Unlock() 646 | 647 | err = q.DBG.Transaction(func(tx *gorm.DB) error { 648 | if err := tx.Where("tenant_id = ? AND queue_id = ? AND message_id = ?", tenantId, queue.ID, messageId).Delete(&KV{}).Error; err != nil { 649 | return err 650 | } 651 | 652 | if err := tx.Where("tenant_id = ? AND queue_id = ? AND id = ?", tenantId, queue.ID, messageId).Delete(&Message{}).Error; err != nil { 653 | return err 654 | } 655 | 656 | return nil 657 | }) 658 | 659 | if err == nil { 660 | log.Debug().Int64("message_id", messageId).Msg("Deleted message") 661 | } 662 | 663 | return err 664 | } 665 | 666 | func (q *SQLiteQueue) Shutdown() error { 667 | db, err := q.DBG.DB() 668 | if err != nil { 669 | return err 670 | } 671 | 672 | return db.Close() 673 | } 674 | -------------------------------------------------------------------------------- /tenants/defaultmanager/manager.go: -------------------------------------------------------------------------------- 1 | package defaultmanager 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "net/http" 7 | 8 | "github.com/poundifdef/smoothmq/config" 9 | "github.com/poundifdef/smoothmq/models" 10 | ) 11 | 12 | type DefaultTenantManager struct { 13 | keys map[string]string 14 | } 15 | 16 | func (tm *DefaultTenantManager) GetTenant(r *http.Request) (int64, error) { 17 | return 1, nil 18 | } 19 | 20 | func (tm *DefaultTenantManager) GetAWSSecretKey(accessKey string, region string) (int64, string, error) { 21 | secretKey, ok := tm.keys[accessKey] 22 | if !ok { 23 | return 0, "", errors.New("invalid key") 24 | } 25 | 26 | return int64(1), secretKey, nil 27 | } 28 | 29 | func NewDefaultTenantManager(cfg []config.AWSKey) models.TenantManager { 30 | keys := make(map[string]string) 31 | for _, key := range cfg { 32 | keys[key.AccessKey] = key.SecretKey 33 | 34 | if key.AccessKey == "DEV_ACCESS_KEY_ID" { 35 | fmt.Println() 36 | fmt.Println("Development SQS credentials:") 37 | fmt.Println(" Access Key: " + key.AccessKey) 38 | fmt.Println(" Secret Key: " + key.SecretKey) 39 | fmt.Println() 40 | } 41 | } 42 | 43 | rc := &DefaultTenantManager{ 44 | keys: keys, 45 | } 46 | return rc 47 | } 48 | --------------------------------------------------------------------------------