├── .env.example ├── .github ├── dependabot.yaml └── workflows │ └── ci.yaml ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── cassandra_init_scripts └── keyspace.cql ├── certs └── .gitkeep ├── custom-mysql-settings.cnf ├── docker-compose-ci.yml ├── docker-compose.yml ├── oracles.env ├── readme-docker-header.png ├── streamr-docker-dev ├── bin.sh └── help_scripts.sh └── yamllint-config.yaml /.env.example: -------------------------------------------------------------------------------- 1 | ## Necessary for AWS 2 | AWS_ACCESS_KEY_ID=xxx 3 | AWS_SECRET_ACCESS_KEY=xxxxx 4 | FILEUPLOAD_S3_BUCKET=bucketname -------------------------------------------------------------------------------- /.github/dependabot.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: 2 3 | updates: 4 | # Maintain dependencies for GitHub Actions 5 | - package-ecosystem: "github-actions" 6 | directory: "/" 7 | schedule: 8 | interval: "daily" 9 | assignees: 10 | - DaisyDomergue 11 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Build 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | branches: 9 | - master 10 | concurrency: 11 | group: ci-${{ github.head_ref }} 12 | cancel-in-progress: true 13 | defaults: 14 | run: 15 | shell: bash 16 | jobs: 17 | lint: 18 | name: Lint 19 | runs-on: ubuntu-latest 20 | steps: 21 | - name: Check out the repo 22 | uses: actions/checkout@v4.0.0 23 | - name: Run YAML Lint 24 | uses: ibiqlik/action-yamllint@v3.1 25 | with: 26 | file_or_dir: . 27 | strict: true 28 | no_warnings: false 29 | config_file: yamllint-config.yaml 30 | - name: Run shellcheck 31 | run: shellcheck --color=never --check-sourced --shell=bash streamr-docker-dev/bin.sh streamr-docker-dev/help_scripts.sh 32 | - name: Run smoke tests 33 | run: make test-smoke 34 | build: 35 | needs: 36 | - lint 37 | name: ${{ matrix.os }} 38 | runs-on: ${{ matrix.os }} 39 | continue-on-error: false 40 | strategy: 41 | fail-fast: false 42 | matrix: 43 | os: 44 | - ubuntu-22.04 45 | - ubuntu-24.04 46 | steps: 47 | - name: Check out the repo 48 | uses: actions/checkout@v4.0.0 49 | - name: Start Streamr Docker Stack 50 | run: ./streamr-docker-dev/bin.sh start --wait --timeout 600 51 | - name: Collect docker logs on failure 52 | if: failure() 53 | uses: jwalton/gh-docker-logs@v2.2.1 54 | with: 55 | dest: 'logs' 56 | - name: Upload logs to GitHub 57 | if: failure() 58 | uses: actions/upload-artifact@v4 59 | with: 60 | name: docker-logs-${{ matrix.os }}-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt}} 61 | path: 'logs' 62 | - name: Stop Streamr Docker Stack 63 | if: always() 64 | run: | 65 | docker kill $(docker ps -q) 66 | docker rm $(docker ps -a -q) 67 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | data/ 2 | .idea 3 | .env 4 | certs/ 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | 635 | Copyright (C) 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | Copyright (C) 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # https://makefiletutorial.com 3 | # https://clarkgrubb.com/makefile-style-guide 4 | # 5 | 6 | LANG := en_US.UTF-8 7 | SHELL := /bin/bash 8 | .SHELLFLAGS := -eu -o pipefail -c # run '/bin/bash ... -c /bin/cmd' 9 | .DEFAULT_GOAL := test-smoke 10 | 11 | YAMLLINT := /usr/local/bin/yamllint 12 | YAMLLINTFLAGS := --config-file yamllint-config.yaml 13 | 14 | $(YAMLLINT): 15 | pip3 install --user yamllint 16 | 17 | .PHONY: lint-yaml 18 | lint-yaml: $(YAMLLINT) 19 | $(YAMLLINT) $(YAMLLINTFLAGS) . 20 | 21 | .PHONY: test-smoke 22 | test-smoke: 23 | docker compose -f docker-compose.yml -f docker-compose-ci.yml config 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | 3 | Streamr 4 | 5 |

6 |

7 | 8 | # Streamr development environment 9 | 10 | This repository contains Docker compose files and command line tool `streamr_docker_dev` for setting up a local Streamr development environment. The environment consists of Streamr services and supporting (3rd party) services. The data of the services is persisted on the local disk. 11 | 12 | ## Setting up 13 | 14 | ### OS X 15 | 16 | 1. Download, install, and run [Docker Community Edition for Mac](https://store.docker.com/editions/community/docker-ce-desktop-mac). 17 | 18 | 2. Increase the amount of memory allocated to Docker VM from 2GB to something 19 | like 5GB or more (Docker icon -> Preferences -> Advanced). Click Apply & Restart. 20 | 21 | 3. Clone this repo: `git clone git@github.com:streamr-dev/streamr-docker-dev.git`, change into that directory `cd streamr-docker-dev` 22 | 23 | 4. Create a symlink to `streamr-docker-dev` into a suitable directory in your PATH (run from repository root): 24 | ``` 25 | ln -sf $(pwd)/streamr-docker-dev/bin.sh /usr/local/bin/streamr-docker-dev 26 | ``` 27 | 28 | ### Linux 29 | 30 | 1. Install and start Docker service. 31 | 32 | 2. Clone this repo: `git clone git@github.com:streamr-dev/streamr-docker-dev.git`, change into that directory `cd streamr-docker-dev` 33 | 34 | 3. Add `streamr-docker-dev` into a suitable directory in your PATH (run from repository root), e.g.: 35 | ``` 36 | ln -sf $(pwd)/streamr-docker-dev/bin.sh /usr/local/bin/streamr-docker-dev 37 | ``` 38 | 39 | ### Windows 40 | 41 | The docker stack has not been tested in a Windows environment and is not recommended at this time. 42 | 43 | If you know what services you need, you don't need to use the `bin.sh`, you can just use `docker-compose` directly, like so: 44 | 45 | 1. Install and start Docker service. 46 | 47 | 2. Clone this repo: `git clone git@github.com:streamr-dev/streamr-docker-dev.git`, change into that directory `cd streamr-docker-dev` 48 | 49 | 3. Bind the loopback interface to 10.200.10.1: `netsh int ip add address "Loopback" 10.200.10.1` 50 | 51 | ## Quickstart 52 | 53 | **Option 1** 54 | 55 | `streamr-docker-dev start --wait ` 56 | 57 | ### Interacting with the local blockchain 58 | 59 | The local Streamr Stack is configured to interact with the local Ethereum node. Transactions should be near instant. 60 | 61 | The recommended way to interact with the blockchain is through Metamask. Here is the network configuration to add: 62 | - Network Name: Streamr Local 63 | - RPC URL: http://localhost:8545 (mainchain) or http://localhost:8546 (sidechain) 64 | - ChainID: 8995 (mainchain) or 8997 (sidechain) 65 | 66 | If you use `StreamrClient`, please configure MetaMask with the sidechain values. 67 | 68 | ### Getting ETH on the local blockchain 69 | 70 | There are pre-filled wallets with ETH & DATA on this chain to use. 71 | 72 | #### Available Accounts 73 | 74 | - (0) 0xa3d1F77ACfF0060F7213D7BF3c7fEC78df847De1 (100 ETH) 75 | - (1) 0x4178baBE9E5148c6D5fd431cD72884B07Ad855a0 (100 ETH) 76 | - (2) 0xdC353aA3d81fC3d67Eb49F443df258029B01D8aB (100 ETH) 77 | - (3) 0x7986b71C27B6eAAB3120a984F26511B2dcfe3Fb4 (100 ETH) 78 | - (4) 0xa6743286b55F36AFA5F4e7e35B6a80039C452dBD (100 ETH) 79 | - (5) 0x7B556228B0D887CfC8d895cCe27CbC79d3e55b3C (100 ETH) 80 | - (6) 0x795063367EbFEB994445d810b94461274E4f109A (100 ETH) 81 | - (7) 0xcA9b39e7A7063cDb845483426D4f12F1f4A44A19 (100 ETH) 82 | - (8) 0x505D48552Ac17FfD0845FFA3783C2799fd4aaD78 (100 ETH) 83 | - (9) 0x65416CBeF822290d9A2FC319Eb6c7f6D9Cd4a541 (100 ETH) 84 | 85 | #### Private Keys 86 | 87 | - (0) 0x5e98cce00cff5dea6b454889f359a4ec06b9fa6b88e9d69b86de8e1c81887da0 88 | - (1) 0xe5af7834455b7239881b85be89d905d6881dcb4751063897f12be1b0dd546bdb 89 | - (2) 0x4059de411f15511a85ce332e7a428f36492ab4e87c7830099dadbf130f1896ae 90 | - (3) 0x633a182fb8975f22aaad41e9008cb49a432e9fdfef37f151e9e7c54e96258ef9 91 | - (4) 0x957a8212980a9a39bf7c03dcbeea3c722d66f2b359c669feceb0e3ba8209a297 92 | - (5) 0xfe1d528b7e204a5bdfb7668a1ed3adfee45b4b96960a175c9ef0ad16dd58d728 93 | - (6) 0xd7609ae3a29375768fac8bc0f8c2f6ac81c5f2ffca2b981e6cf15460f01efe14 94 | - (7) 0xb1abdb742d3924a45b0a54f780f0f21b9d9283b231a0a0b35ce5e455fa5375e7 95 | - (8) 0x2cd9855d17e01ce041953829398af7e48b24ece04ff9d0e183414de54dc52285 96 | - (9) 0x2c326a4c139eced39709b235fffa1fde7c252f3f7b505103f7b251586c35d543 97 | 98 | 99 | #### Local DATA token contract address 100 | 0xbAA81A0179015bE47Ad439566374F2Bae098686F 101 | 102 | ### Troubleshooting 103 | 104 | Metamask can often get stuck when transacting on many different networks. A handy trick is to "Reset Account" in Settings/Advanced. This will remove the transaction history which helps it get unstuck. 105 | 106 | ## Commands 107 | 108 | ### `help` 109 | 110 | List available commands: `streamr-docker-dev` 111 | 112 | Help about particular command: `streamr-docker-dev help [command]` 113 | 114 | ### `start` 115 | 116 | Start all services: `streamr-docker-dev start` 117 | 118 | Start particular services: `streamr-docker-dev start [services]` 119 | 120 | Start all services except particular ones: `streamr-docker-dev start --except [service]` 121 | 122 | Start and wait for services to become healthy: `streamr-docker-dev start --wait` 123 | 124 | ### `stop` 125 | 126 | Stop all services: `streamr-docker-dev stop` 127 | 128 | Stop particular services: `streamr-docker-dev stop [services]` 129 | 130 | ### `restart` 131 | 132 | Restart all services: `streamr-docker-dev restart` 133 | 134 | Restart particular services: `streamr-docker-dev restart [services]` 135 | 136 | ### `wait` 137 | 138 | Wait up to 5 min for pending health checks to pass: `streamr-docker-dev wait --timeout 300` 139 | 140 | ### `ps` 141 | 142 | View process list: `streamr-docker-dev ps [services]` 143 | 144 | ### `log` 145 | 146 | View all logs: `streamr-docker-dev log [-f]` 147 | 148 | View logs of certain services: `streamr-docker-dev log [-f] [services]` 149 | 150 | ### `shell` 151 | 152 | Open an interactive shell into a container: `streamr-docker-dev shell [service]` 153 | 154 | ### `pull` 155 | 156 | Pull latest images of all services: `streamr-docker-dev pull` 157 | 158 | Pull latest images of certain services: `streamr-docker-dev pull [services]` 159 | 160 | ### `update` 161 | 162 | Update the tool to the latest version: `streamr-docker-dev update` 163 | 164 | ### `wipe` 165 | 166 | Wipe the persisted data of all services: `streamr-docker-dev wipe` 167 | 168 | ### `factory-reset` 169 | 170 | "Factory reset" the docker environment by removing all images and persisted data: `streamr-docker-dev factory-reset`. 171 | **Warning: this will delete all your docker images and their state, not just ones related to the Streamr stack.** 172 | 173 | ## Accessing the Core app and Streamr API 174 | 175 | Once the services are running, browse to [http://localhost](http://localhost) to use Core. 176 | 177 | The API root is at `http://localhost/api/v2`. 178 | 179 | ## Usage in Streamr development 180 | 181 | When you're developing one of the Streamr components, you'll want to use the `streamr-docker-dev` tool with the `--except` flag to exclude the service you're developing: 182 | 183 | ``` 184 | streamr-docker-dev start --except [service-under-development] 185 | ``` 186 | 187 | ## Usage in testing and CI 188 | 189 | In CI, the `--wait` flag can be used with `start` to block until all health checks are passing. This ensures that the services are up and running before starting your tests. 190 | 191 | For integration tests, use the `--except` option to exclude the service under test: 192 | 193 | ``` 194 | streamr-docker-dev start --except [service-under-test] --wait 195 | ``` 196 | 197 | When testing the SDKs or running other end-to-end tests, just start the full stack before your tests: 198 | 199 | ``` 200 | streamr-docker-dev start --wait 201 | ``` 202 | 203 | ## Included services 204 | 205 | ### Streamr services 206 | - 3 x [Broker](https://github.com/streamr-dev/broker) nodes 207 | - 2 broker nodes + 1 storage Streamr network nodes. This creates a local and private Streamr Network. 208 | - 3 x [Tracker](https://github.com/streamr-dev/broker) 209 | - Helps node discovery in the Strearm Network 210 | - 1 x [TheGraph node](https://github.com/streamr-dev/network-contracts) 211 | - GraphQL queries at http://localhost:8000/subgraphs/name/githubname/subgraphname 212 | - GUI to past GraphQL queries: http://192.168.0.8:8000/subgraphs/name/githubname/subgraphname/graphql 213 | - An example query is: 214 | ``` 215 | { 216 | streams { 217 | id, 218 | metadata, 219 | permissions { 220 | id, 221 | user, 222 | edit, 223 | canDelete, 224 | publish, 225 | subscribed, 226 | share, 227 | } 228 | } 229 | } 230 | ``` 231 | ### Supporting services 232 | - 1 x MySQL instance 233 | - 1 x Apache Cassandra instance with `streamr_dev` keyspace 234 | - 1 x Ethereum node 235 | - 1 x Postgres DB for TheGraph 236 | - 1 x ipfs for TheGraph 237 | - 1 x adapter for ENS queries from sidechain to mainchain 238 | 239 | ## Troubleshooting 240 | 241 | ### Metamask transactions are stuck 242 | 243 | This is a common problem on Metamask when switching inbetween chains. Clearing the transaction history in Metamask usually fixes this issue: 244 | Click, 'Settings', then 'Advanced', then, 'Reset Account'. 245 | 246 | ### Making image uploads to AWS work 247 | 248 | Uploading images to AWS needs credentials to be able to access the target S3 bucket. 249 | 250 | 1. copy `.env.example` as `.env` and change the values to correct ones 251 | (Note: `.env` file is in `.gitignore`) 252 | 253 | 2. Restart `engine-and-editor` and S3 services should work. You can debug any 254 | potential issues with `streamr-docker-dev log -f engine-and-editor`. 255 | 256 | ### Error response from daemon 257 | `Get https://registry-1.docker.io/v2/: net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)` 258 | 259 | This is a connection issue; could be DNS settings, could be firewall. For me, changing to another wlan helped. 260 | 261 | I received this error only during `docker login`, after login I could resume using office wlan (`docker pull` works somehow differently). 262 | 263 | ## Directory structure 264 | 265 | | File | Description | 266 | |:-----------------------------|:----------------------------------------------------------------| 267 | | docker-compose.yml | Supporting services stack (MySQL, Cassandra, Redis, etc.) | 268 | | docker-compose.override.yml | Streamr service stack | 269 | | custom-mysql-settings.cnf | Custom MySQL settings | 270 | | mysql\_init\_scripts | Database dumps used to initalize MySQL databases | 271 | | keyspace.cql | Keyspace definitions and demo data for Cassandra | 272 | | data | Persistance of container data | 273 | 274 | ## License 275 | 276 | This software is open source and licensed under [GPLv3](https://www.gnu.org/licenses/gpl.html). 277 | 278 | -------------------------------------------------------------------------------- /cassandra_init_scripts/keyspace.cql: -------------------------------------------------------------------------------- 1 | CREATE KEYSPACE IF NOT EXISTS streamr_dev WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }; 2 | USE streamr_dev; 3 | 4 | CREATE TABLE IF NOT EXISTS stream_data ( 5 | id varchar, 6 | partition int, 7 | ts timestamp, 8 | sequence_no bigint, 9 | publisher_id varchar, 10 | msg_chain_id varchar, 11 | payload blob, 12 | PRIMARY KEY ((id, partition), ts, sequence_no, publisher_id, msg_chain_id) 13 | ); 14 | 15 | -- ### NEW KEYSPACE ### -- 16 | CREATE KEYSPACE IF NOT EXISTS streamr_dev_v2 WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }; 17 | USE streamr_dev_v2; 18 | 19 | CREATE TABLE IF NOT EXISTS bucket 20 | ( 21 | stream_id text, 22 | partition int, 23 | date_create timestamp, 24 | id timeuuid, 25 | records int, 26 | size int, 27 | primary key ((stream_id, partition), date_create) 28 | ) WITH CLUSTERING ORDER BY (date_create desc); 29 | 30 | CREATE TABLE IF NOT EXISTS stream_data 31 | ( 32 | stream_id text, 33 | partition int, 34 | bucket_id timeuuid, 35 | ts timestamp, 36 | sequence_no int, 37 | publisher_id text, 38 | msg_chain_id text, 39 | payload blob, 40 | primary key ((stream_id, partition, bucket_id), ts, sequence_no, publisher_id, msg_chain_id) 41 | ) WITH CLUSTERING ORDER BY (ts ASC, sequence_no ASC); 42 | 43 | 44 | -------------------------------------------------------------------------------- /certs/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/streamr-dev/streamr-docker-dev/e5ccd029ef1732dbe01825d9f97befe07ad05a07/certs/.gitkeep -------------------------------------------------------------------------------- /custom-mysql-settings.cnf: -------------------------------------------------------------------------------- 1 | [mysqld] 2 | max_allowed_packet=32M 3 | sql_mode=ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION 4 | lower_case_table_names=1 5 | -------------------------------------------------------------------------------- /docker-compose-ci.yml: -------------------------------------------------------------------------------- 1 | --- 2 | services: 3 | cassandra: 4 | deploy: 5 | resources: 6 | limits: 7 | cpus: '2.0' 8 | memory: 3G 9 | reservations: 10 | memory: 1.7G 11 | init-keyspace: 12 | deploy: 13 | resources: 14 | limits: 15 | cpus: '1.0' 16 | memory: 100M 17 | reservations: 18 | memory: 6M 19 | mysql: 20 | deploy: 21 | resources: 22 | limits: 23 | cpus: '2.0' 24 | memory: 500M 25 | reservations: 26 | memory: 250M 27 | broker-node-storage-1: 28 | deploy: 29 | resources: 30 | limits: 31 | cpus: '2.0' 32 | memory: 200M 33 | reservations: 34 | memory: 150M 35 | broker-node-no-storage-1: 36 | deploy: 37 | resources: 38 | limits: 39 | cpus: '2.0' 40 | memory: 200M 41 | reservations: 42 | memory: 150M 43 | broker-node-no-storage-2: 44 | deploy: 45 | resources: 46 | limits: 47 | cpus: '2.0' 48 | memory: 200M 49 | reservations: 50 | memory: 150M 51 | network-explorer: 52 | deploy: 53 | resources: 54 | limits: 55 | cpus: '1.0' 56 | memory: 50M 57 | reservations: 58 | memory: 10M 59 | ipfs: 60 | deploy: 61 | resources: 62 | limits: 63 | cpus: '1.0' 64 | memory: 250M 65 | reservations: 66 | memory: 20M 67 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | services: 3 | cassandra: 4 | container_name: streamr-dev-cassandra 5 | image: cassandra:3.11.5 6 | init: true 7 | networks: 8 | - streamr-network 9 | ports: 10 | # - "7000:7000" # Conflicts with "Control Centre" service on MacOS. Cassandra uses it for cluster communication, which isn't necessary in a single-node cluster. 11 | - "7001:7001" 12 | - "7199:7199" 13 | - "9042:9042" 14 | - "9160:9160" 15 | volumes: 16 | - type: volume 17 | source: data-cassandra 18 | target: /var/lib/cassandra 19 | volume: 20 | nocopy: true 21 | healthcheck: 22 | test: ["CMD-SHELL", "[ $$(nodetool statusgossip) = running ]"] 23 | interval: 5s 24 | timeout: 10s 25 | retries: 10 26 | restart: unless-stopped 27 | init-keyspace: 28 | container_name: streamr-dev-init-keyspace 29 | image: cassandra:3.11.5 30 | init: true 31 | networks: 32 | - streamr-network 33 | command: bash -c "sleep 5 && cqlsh cassandra -f /init_scripts/keyspace.cql && echo keyspace initialized" 34 | restart: on-failure # exits on success 35 | volumes: 36 | - type: bind 37 | source: ./cassandra_init_scripts 38 | target: /init_scripts 39 | read_only: true 40 | bind: 41 | propagation: rprivate 42 | depends_on: 43 | - cassandra 44 | mysql: 45 | container_name: streamr-dev-mysql 46 | image: mysql:9.1.0 47 | init: true 48 | networks: 49 | - streamr-network 50 | ports: 51 | - "3306:3306" 52 | volumes: 53 | - type: volume 54 | source: data-mysql 55 | target: /var/lib/mysql 56 | volume: 57 | nocopy: true 58 | - type: bind 59 | source: ./custom-mysql-settings.cnf 60 | target: /etc/mysql/conf.d/custom-mysql-settings.cnf 61 | read_only: true 62 | bind: 63 | propagation: rprivate 64 | restart: unless-stopped 65 | environment: 66 | MYSQL_ROOT_PASSWORD: password 67 | healthcheck: 68 | test: ["CMD", "mysqladmin", "ping", "-h", "localhost"] 69 | interval: 5s 70 | timeout: 10s 71 | retries: 10 72 | entry-point: 73 | container_name: streamr-dev-entry-point 74 | image: streamr/node:dev 75 | init: true 76 | networks: 77 | - streamr-network 78 | restart: unless-stopped 79 | ports: 80 | - "40500:40500" 81 | command: npm exec -c entry-point 82 | healthcheck: 83 | test: ["CMD", "curl", "http:/10.200.10.1:40500"] 84 | interval: 1s 85 | timeout: 5s 86 | retries: 10 87 | broker-node-storage-1: 88 | container_name: streamr-dev-broker-node-storage-1 89 | image: streamr/node:dev 90 | init: true 91 | networks: 92 | - streamr-network 93 | restart: unless-stopped 94 | ports: 95 | - "8890:8890" 96 | - "8891:8891" 97 | - "9000:9000" 98 | - "40401:40401" 99 | depends_on: 100 | entry-point: 101 | condition: service_healthy 102 | cassandra: 103 | condition: service_healthy 104 | init-keyspace: 105 | condition: service_completed_successfully 106 | deploy-network-subgraphs-fastchain: 107 | condition: service_completed_successfully 108 | environment: 109 | STREAMR_URL: "10.200.10.1" 110 | CASSANDRA_HOST: 10.200.10.1:9042 111 | command: npm exec -c "streamr-node configs/docker-1.env.json" 112 | healthcheck: 113 | test: ["CMD", "curl", "--fail", "--silent", "--show-error", "--max-time", "9", "http://localhost:8891/info"] 114 | interval: 30s 115 | timeout: 10s 116 | retries: 20 117 | broker-node-no-storage-1: 118 | container_name: streamr-dev-broker-node-no-storage-1 119 | image: streamr/node:dev 120 | init: true 121 | networks: 122 | - streamr-network 123 | restart: unless-stopped 124 | ports: 125 | - "8790:8790" 126 | - "8791:8791" 127 | - "9100:9100" 128 | - "40402:40402" 129 | depends_on: 130 | entry-point: 131 | condition: service_healthy 132 | deploy-network-subgraphs-fastchain: 133 | condition: service_completed_successfully 134 | environment: 135 | STREAMR_URL: "10.200.10.1" 136 | command: npm exec -c "streamr-node configs/docker-2.env.json" 137 | healthcheck: 138 | test: ["CMD", "curl", "--fail", "--silent", "--show-error", "--max-time", "9", "http://localhost:8791/info"] 139 | interval: 30s 140 | timeout: 10s 141 | retries: 20 142 | broker-node-no-storage-2: 143 | container_name: streamr-dev-broker-node-no-storage-2 144 | image: streamr/node:dev 145 | init: true 146 | networks: 147 | - streamr-network 148 | restart: unless-stopped 149 | ports: 150 | - "8690:8690" 151 | - "8691:8691" 152 | - "9200:9200" 153 | - "40403:40403" 154 | depends_on: 155 | entry-point: 156 | condition: service_healthy 157 | deploy-network-subgraphs-fastchain: 158 | condition: service_completed_successfully 159 | environment: 160 | STREAMR_URL: "10.200.10.1" 161 | command: npm exec -c "streamr-node configs/docker-3.env.json" 162 | healthcheck: 163 | test: ["CMD", "curl", "--fail", "--silent", "--show-error", "--max-time", "9", "http://localhost:8691/info"] 164 | interval: 30s 165 | timeout: 10s 166 | retries: 20 167 | network-explorer: 168 | container_name: streamr-dev-network-explorer 169 | image: streamr/network-explorer:dev 170 | networks: 171 | - streamr-network 172 | ports: 173 | - "3334:80" 174 | depends_on: 175 | - entry-point 176 | healthcheck: 177 | test: ["CMD", "curl", "--fail", "--silent", "--show-error", "--max-time", "9", "http://localhost"] 178 | interval: 10s 179 | timeout: 10s 180 | retries: 60 181 | graph-deploy-dataunion-subgraph-fastchain: 182 | container_name: streamr-dev-graph-deploy-dataunion-subgraph-fastchain 183 | image: streamr/graph-deploy-dataunion-subgraph:dev-fastchain 184 | restart: on-failure # exits on success 185 | networks: 186 | - streamr-network 187 | depends_on: 188 | - graph-node-fastchain 189 | volumes: 190 | - type: volume 191 | source: data-graph-deploy-dataunion-fastchain 192 | target: /firstrun 193 | volume: 194 | nocopy: false 195 | ipfs: 196 | container_name: streamr-dev-ipfs 197 | image: ipfs/kubo:v0.22.0 198 | networks: 199 | - streamr-network 200 | ports: 201 | - '5001:5001' 202 | - '6695:8080' 203 | volumes: 204 | - type: volume 205 | source: data-ipfs 206 | target: /data/ipfs 207 | volume: 208 | nocopy: true 209 | environment: 210 | IPFS_PROFILE: server # sets the server profile 211 | healthcheck: 212 | test: ["CMD-SHELL", "ipfs --api=/ip4/127.0.0.1/tcp/5001 dag stat /ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn || exit 1"] # from official image 213 | interval: 10s 214 | timeout: 3s 215 | retries: 10 216 | dev-chain-fast: 217 | container_name: streamr-dev-chain-fast 218 | image: streamr/dev-chain-fast:dev 219 | networks: 220 | - streamr-network 221 | ports: 222 | - "8547:8545" 223 | restart: unless-stopped 224 | healthcheck: 225 | test: ["CMD", "curl", "--fail", "--silent", "--show-error", "--max-time", "9", "--header", "Content-Type: application/json", "--data", '[{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1},{"jsonrpc":"2.0","method":"eth_syncing","params":[],"id":1}]', "http://localhost:8545/api/health"] 226 | interval: 10s 227 | timeout: 1s 228 | retries: 10 229 | graph-node-fastchain: 230 | container_name: streamr-dev-thegraph-node-fastchain 231 | image: graphprotocol/graph-node:v0.32.0 232 | restart: unless-stopped 233 | networks: 234 | - streamr-network 235 | ports: 236 | - '8800:8000' 237 | # - '8001:8001' 238 | - '8820:8020' 239 | # - '8030:8030' 240 | # - '8040:8040' 241 | depends_on: 242 | - ipfs 243 | - postgres-fastchain 244 | - dev-chain-fast 245 | environment: 246 | postgres_host: postgres-fastchain 247 | postgres_user: streamr 248 | postgres_pass: let-me-in 249 | postgres_db: streamr 250 | ipfs: 'streamr-dev-ipfs:5001' 251 | ethereum: 'xDai:http://streamr-dev-chain-fast:8545' 252 | RUST_LOG: info 253 | GRAPH_ALLOW_NON_DETERMINISTIC_FULLTEXT_SEARCH: "true" 254 | healthcheck: 255 | test: ["CMD", "nc", "-z", "localhost", "8000"] 256 | interval: 5s 257 | timeout: 10s 258 | retries: 10 259 | deploy-network-subgraphs-fastchain: 260 | container_name: streamr-dev-deploy-network-subgraphs-fastchain 261 | image: streamr/deploy-network-subgraphs:dev-fastchain 262 | restart: on-failure # exits on success 263 | networks: 264 | - streamr-network 265 | depends_on: 266 | - graph-node-fastchain 267 | volumes: 268 | - type: volume 269 | source: data-graph-deploy-fastchain 270 | target: /firstrun 271 | volume: 272 | nocopy: false 273 | postgres-fastchain: 274 | container_name: streamr-dev-postgres-fastchain 275 | image: postgres 276 | restart: unless-stopped 277 | networks: 278 | - streamr-network 279 | # ports: 280 | # - '5432:5432' 281 | command: ["postgres", "-cshared_preload_libraries=pg_stat_statements"] 282 | environment: 283 | POSTGRES_USER: streamr 284 | POSTGRES_PASSWORD: let-me-in 285 | POSTGRES_DB: streamr 286 | POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C" 287 | volumes: 288 | - type: volume 289 | source: data-postgres-fastchain 290 | target: /var/lib/postgresql/data 291 | volume: 292 | nocopy: true 293 | healthcheck: 294 | test: ["CMD-SHELL", "pg_isready -U streamr -d streamr"] 295 | interval: 5s 296 | timeout: 10s 297 | retries: 10 298 | stream-metrics-index: 299 | container_name: streamr-dev-stream-metrics-index 300 | image: streamr/stream-metrics-index 301 | networks: 302 | - streamr-network 303 | restart: unless-stopped 304 | ports: 305 | - "4001:4001" 306 | depends_on: 307 | - mysql 308 | - deploy-network-subgraphs-fastchain 309 | - entry-point 310 | ens-sync-script-fastchain: 311 | container_name: ens-sync-script-fastchain 312 | image: streamr/ens-sync-script:dev-fastchain 313 | networks: 314 | - streamr-network 315 | depends_on: 316 | - dev-chain-fast 317 | healthcheck: 318 | test: ["CMD", "/home/node/app/script/healthcheck.sh"] 319 | interval: 10s 320 | timeout: 10s 321 | retries: 60 322 | 323 | networks: 324 | streamr-network: 325 | driver: bridge 326 | 327 | volumes: 328 | cassandra_init_scripts: 329 | data-mysql: 330 | data-cassandra: 331 | data-ipfs: 332 | data-postgres-fastchain: 333 | data-graph-deploy-fastchain: 334 | data-graph-deploy-dataunion-fastchain: 335 | -------------------------------------------------------------------------------- /oracles.env: -------------------------------------------------------------------------------- 1 | NODE_ENV=production 2 | ORACLE_BRIDGE_MODE=ARBITRARY_MESSAGE 3 | ORACLE_LOG_LEVEL=debug 4 | 5 | 6 | ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY=5e98cce00cff5dea6b454889f359a4ec06b9fa6b88e9d69b86de8e1c81887da0 7 | ORACLE_VALIDATOR_ADDRESS=0xa3d1F77ACfF0060F7213D7BF3c7fEC78df847De1 8 | COMMON_HOME_BRIDGE_ADDRESS=0xaFA0dc5Ad21796C9106a36D68f69aAD69994BB64 9 | COMMON_FOREIGN_BRIDGE_ADDRESS=0xaFA0dc5Ad21796C9106a36D68f69aAD69994BB64 10 | 11 | 12 | COMMON_HOME_RPC_URL=http://10.200.10.1:8546 13 | #COMMON_HOME_BRIDGE_ADDRESS=0xA9A988fAd795CAFF275Cc054e94283BBb953a386 14 | ORACLE_HOME_RPC_POLLING_INTERVAL=5000 15 | 16 | COMMON_FOREIGN_RPC_URL=http://10.200.10.1:8545 17 | #COMMON_FOREIGN_BRIDGE_ADDRESS=0xE4eA76e830a659282368cA2e7E4d18C4AE52D8B3 18 | ORACLE_FOREIGN_RPC_POLLING_INTERVAL=5000 19 | 20 | COMMON_HOME_GAS_PRICE_FALLBACK=1000000000 21 | COMMON_HOME_GAS_PRICE_FACTOR=1 22 | ORACLE_HOME_GAS_PRICE_UPDATE_INTERVAL=600000 23 | 24 | # COMMON_FOREIGN_GAS_PRICE_SUPPLIER_URL="https://gasprice.poa.network/" 25 | # COMMON_FOREIGN_GAS_PRICE_SPEED_TYPE="fast" 26 | ORACLE_ALLOW_HTTP_FOR_RPC=yes 27 | ORACLE_QUEUE_URL=amqp://bridge-rabbitmq 28 | ORACLE_REDIS_URL=redis://redis 29 | 30 | COMMON_FOREIGN_GAS_PRICE_FALLBACK=10000000000 31 | COMMON_FOREIGN_GAS_PRICE_FACTOR=1 32 | ORACLE_FOREIGN_GAS_PRICE_UPDATE_INTERVAL=600000 33 | 34 | UI_NATIVE_TOKEN_DISPLAY_NAME=nt 35 | UI_FOREIGN_TOKEN_DISPLAY_NAME=ft 36 | UI_HOME_NETWORK_DISPLAY_NAME=dev-mainchain 37 | UI_FOREIGN_NETWORK_DISPLAY_NAME=dev-sidechain 38 | 39 | UI_HOME_WITHOUT_EVENTS=false 40 | UI_FOREIGN_WITHOUT_EVENTS=false 41 | 42 | UI_HOME_EXPLORER_TX_TEMPLATE=https://blockscout.com/poa/sokol/tx/%s 43 | UI_FOREIGN_EXPLORER_TX_TEMPLATE=https://blockscout.com/eth/kovan/tx/%s 44 | UI_HOME_EXPLORER_ADDRESS_TEMPLATE=https://blockscout.com/poa/sokol/address/%s 45 | UI_FOREIGN_EXPLORER_ADDRESS_TEMPLATE=https://blockscout.com/eth/kovan/address/%s 46 | 47 | UI_HOME_GAS_PRICE_UPDATE_INTERVAL=15000 48 | UI_FOREIGN_GAS_PRICE_UPDATE_INTERVAL=15000 49 | 50 | UI_PORT=3000 51 | 52 | UI_APP_STYLES=classic 53 | 54 | -------------------------------------------------------------------------------- /readme-docker-header.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/streamr-dev/streamr-docker-dev/e5ccd029ef1732dbe01825d9f97befe07ad05a07/readme-docker-header.png -------------------------------------------------------------------------------- /streamr-docker-dev/bin.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ORIG_FILENAME="$(readlink "$0" -f)" 4 | ORIG_DIRNAME=$(dirname "$ORIG_FILENAME") 5 | ROOT_DIR="$ORIG_DIRNAME/.." 6 | CONTAINER_PREFIX="streamr-dev-" 7 | 8 | OPERATION= 9 | COMMANDS_TO_RUN=() 10 | 11 | SERVICES="" 12 | EXCEPT_SERVICES=() 13 | FLAGS="" 14 | DETACHED=1 15 | DRY_RUN=0 16 | FOLLOW=0 17 | WAIT=0 18 | WAIT_TIMEOUT=500 # seconds 19 | ADDITIONAL_COMPOSE_OPTS="" 20 | if [ -n "${CI-}" ]; then # Apply CI override when running on CI server 21 | ADDITIONAL_COMPOSE_OPTS="$DOCKER_COMPOSE -f docker-compose-ci.yml" 22 | fi 23 | DOCKER_COMPOSE="docker compose --ansi never -f docker-compose.yml $ADDITIONAL_COMPOSE_OPTS" 24 | 25 | # don't start these services unless explicitly started 26 | EXCEPT_SERVICES_DEFAULT=() # array of string e.g. ("a" "b") 27 | 28 | # Service Aliases 29 | NODE_NO_STORAGE='broker-node-no-storage-1 broker-node-no-storage-2' 30 | NODE_STORAGE='broker-node-storage-1' 31 | NODES="$NODE_NO_STORAGE $NODE_STORAGE" 32 | 33 | # swap aliases for full names (e.g. NODES -> broker-node-storage-1 broker-node-no-storage-1 broker-node-no-storage-2) 34 | # feel free to add more, just make sure you don't end up using actual service 35 | # names, or part-thereof as alias names 36 | expandServiceAliases() { 37 | local names=$1 38 | # aliases should be plural to reduce possibility of conflicts with service names 39 | names="${names//no-storage-nodes/$NODE_NO_STORAGE}" 40 | names="${names//storage-nodes/$NODE_STORAGE}" 41 | names="${names//brokers/$NODES}" 42 | names="${names//nodes/$NODES}" # brokers/nodes sort of interchangeable 43 | echo "$names" 44 | } 45 | 46 | # Execute all commands from the root dir of streamr-docker-dev 47 | cd "$ROOT_DIR" || exit 1 48 | 49 | if [ -f .env ]; then 50 | # Read .env (from https://stackoverflow.com/questions/19331497/set-environment-variables-from-file-of-key-value-pairs/20909045#20909045) 51 | set -o allexport 52 | # shellcheck disable=SC1091 53 | source .env 54 | set +o allexport 55 | fi 56 | # Set default values for required env variables if not set in .env 57 | if [[ -z "${STREAMR_BASE_URL}" ]]; then 58 | export STREAMR_BASE_URL=http://10.200.10.1 59 | else 60 | echo "Using STREAMR_BASE_URL: ${STREAMR_BASE_URL}" 61 | fi 62 | 63 | if [[ -z "${STREAMR_WS_URL}" ]]; then 64 | export STREAMR_WS_URL=${STREAMR_BASE_URL/http/ws}/api/v2/ws # replace "http" with "ws" 65 | else 66 | echo "Using STREAMR_WS_URL: ${STREAMR_WS_URL}" 67 | fi 68 | 69 | help() { 70 | # shellcheck disable=SC2086 71 | "$ORIG_DIRNAME/help_scripts.sh" $SERVICES 72 | } 73 | 74 | services() { 75 | $DOCKER_COMPOSE config --services 76 | } 77 | 78 | start() { 79 | ip_lines=$(/sbin/ifconfig | grep -c 10.200.10.1) 80 | if [ "$ip_lines" -eq "0" ]; then 81 | COMMANDS_TO_RUN+=("echo Binding the internal IP address 10.200.10.1 to the loopback interface.") 82 | COMMANDS_TO_RUN+=("echo This requires sudo privileges, so please provide your password if requested") 83 | 84 | # Binding the loopback address is OS-specific 85 | case "$OSTYPE" in 86 | darwin*) 87 | COMMANDS_TO_RUN+=("sudo ifconfig lo0 alias 10.200.10.1/24") 88 | ;; 89 | linux*) 90 | COMMANDS_TO_RUN+=("sudo ip addr add 10.200.10.1 dev lo label lo:1") 91 | ;; 92 | msys*|cygwin*) # windows 93 | # maybe something like this: netsh interface ip add address "loopback" 10.200.10.1 255.255.255.255 94 | echo "streamr-docker-dev: unsupported operating system: $OSTYPE" 1>&2 95 | exit 1 96 | ;; 97 | *) 98 | echo "streamr-docker-dev: unknown operating system: $OSTYPE" 1>&2 99 | exit 1 100 | ;; 101 | esac 102 | fi 103 | [[ $DETACHED == 1 ]] && FLAGS+=" -d" 104 | [[ $SERVICES == "" ]] && msg="Starting all" || msg="Starting $SERVICES" 105 | 106 | # only start these if started explicitly 107 | for service in "${EXCEPT_SERVICES_DEFAULT[@]}" 108 | do 109 | if [[ ! "$SERVICES" =~ $service ]]; then 110 | EXCEPT_SERVICES+=("$service") 111 | fi 112 | done 113 | 114 | # use --scale $service=0 to prevent start of --except services 115 | [[ ! "${#EXCEPT_SERVICES[@]}" -eq 0 ]] && msg+=" except:" 116 | for service in "${EXCEPT_SERVICES[@]}" 117 | do 118 | msg+=" $service" 119 | FLAGS+=" --scale $service=0" 120 | done 121 | 122 | FLAGS+=" --remove-orphans" 123 | 124 | COMMANDS_TO_RUN+=("echo $msg") 125 | COMMANDS_TO_RUN+=("$DOCKER_COMPOSE up $FLAGS $SERVICES") 126 | 127 | if [ $WAIT == 1 ]; then 128 | COMMANDS_TO_RUN+=("wait") 129 | fi 130 | } 131 | 132 | stop() { 133 | [[ $SERVICES == "" ]] && msg="Stopping all" || msg="Stopping $SERVICES" 134 | COMMANDS_TO_RUN+=("echo $msg") 135 | COMMANDS_TO_RUN+=("$DOCKER_COMPOSE kill $SERVICES") 136 | COMMANDS_TO_RUN+=("$DOCKER_COMPOSE rm -f $SERVICES") 137 | } 138 | 139 | restart() { 140 | stop && COMMANDS_TO_RUN+=("printf \n") && start 141 | } 142 | 143 | wait() { 144 | echo "Waiting for pending health checks to pass (timeout: $WAIT_TIMEOUT sec)..." 145 | declare -i time_waited 146 | while [[ $time_waited -lt $WAIT_TIMEOUT ]]; do 147 | waiting_for_services=() 148 | 149 | # Get the id of each image we have in docker-compose 150 | for image_id in $($DOCKER_COMPOSE ps -q) 151 | do 152 | service_name=$(docker inspect -f "{{.Name}}" "$image_id") 153 | # Try to read health state of each image 154 | health_state=$(docker inspect -f "{{.State.Health.Status}}" "$image_id" 2> /dev/null) 155 | # shellcheck disable=SC2181 156 | if [ $? -eq 0 ]; then 157 | # Successfully got health state. Is the service healthy? 158 | if [ "$health_state" != "healthy" ]; then 159 | waiting_for_services+=("$service_name ($health_state)") 160 | fi 161 | else 162 | # Error while fetching health state. Maybe a health check is not configured. Did the image exit successfully? 163 | exit_status=$(docker inspect -f "{{.State.Status}}" "$image_id") 164 | exit_code=$(docker inspect -f "{{.State.ExitCode}}" "$image_id") 165 | if [ "$exit_status" != "exited" ]; then 166 | # Didn't exit yet, keep waiting... 167 | waiting_for_services+=("$service_name (no health check -> waiting for it to exit)") 168 | elif [ "$exit_code" != "0" ]; then 169 | # Exited but errored 170 | waiting_for_services+=("$service_name (ERROR: exit code $exit_code)") 171 | fi 172 | fi 173 | done 174 | 175 | if [ ${#waiting_for_services[@]} -gt 0 ]; then 176 | if (( time_waited >= 30 )); then 177 | echo "***********************************" 178 | echo "Still waiting for the following services:" 179 | for s in "${waiting_for_services[@]}"; do echo "$s"; done 180 | fi 181 | sleep 1s 182 | time_waited=$((time_waited+1)) 183 | else 184 | echo "All services up and running." 185 | break 186 | fi 187 | done 188 | 189 | if [[ $time_waited -ge $WAIT_TIMEOUT ]]; then 190 | echo "ERROR: Timed out waiting for health checks to pass. (Timeout: $WAIT_TIMEOUT sec)" 191 | exit 1 192 | fi 193 | } 194 | 195 | ps() { 196 | COMMANDS_TO_RUN+=("$DOCKER_COMPOSE ps $SERVICES") 197 | } 198 | 199 | log() { 200 | FLAGS+=" -t --tail=1000" 201 | if [ $FOLLOW == 1 ];then 202 | FLAGS+=" -f" 203 | fi 204 | COMMANDS_TO_RUN+=("$DOCKER_COMPOSE logs $FLAGS $SERVICES") 205 | } 206 | 207 | shell() { 208 | # Assumes standardized container names that begin with $CONTAINER_PREFIX 209 | COMMANDS_TO_RUN+=("docker exec -ti $CONTAINER_PREFIX$SERVICES /bin/sh") 210 | } 211 | 212 | pull() { 213 | # Pull latest images define on docker compose 214 | COMMANDS_TO_RUN+=("$DOCKER_COMPOSE pull $SERVICES") 215 | } 216 | 217 | update() { 218 | # git pull latest version 219 | COMMANDS_TO_RUN+=("git pull") 220 | } 221 | 222 | wipe() { 223 | stop 224 | COMMANDS_TO_RUN+=("echo Wiping persistent data of services") 225 | COMMANDS_TO_RUN+=("docker volume prune --all --force") 226 | } 227 | 228 | factory-reset() { 229 | wipe 230 | COMMANDS_TO_RUN+=("echo Pruning docker images. This may take a while...") 231 | COMMANDS_TO_RUN+=("docker system prune --all --force --volumes") 232 | } 233 | 234 | OPERATION=$1 235 | shift 236 | 237 | # Read arguments & options 238 | while [ $# -gt 0 ]; do # if there are arguments 239 | if [[ "$1" = -* ]]; then 240 | case $1 in 241 | --except ) 242 | EXCEPT_SERVICES+=("$2") 243 | shift # skip over the next arg, which was already consumed above 244 | ;; 245 | --wait ) 246 | WAIT=1 247 | ;; 248 | --timeout ) 249 | WAIT_TIMEOUT=$2 250 | shift # skip over the next arg, which was already consumed above 251 | ;; 252 | -f | --follow ) 253 | FOLLOW=1 254 | ;; 255 | --dry-run ) 256 | DRY_RUN=1 257 | ;; 258 | --attached ) 259 | DETACHED=0 260 | ;; 261 | * ) 262 | help 263 | echo "ERROR: Invalid option: $1" 264 | exit 1 265 | ;; 266 | esac 267 | else 268 | SERVICES+="$1 " 269 | fi 270 | shift 271 | done 272 | 273 | # shellcheck disable=SC2207 274 | EXCEPT_SERVICES_DEFAULT=($(expandServiceAliases "${EXCEPT_SERVICES_DEFAULT[*]}")) 275 | SERVICES=$(expandServiceAliases "$SERVICES") 276 | # shellcheck disable=SC2207 277 | EXCEPT_SERVICES=($(expandServiceAliases "${EXCEPT_SERVICES[*]}")) 278 | 279 | # Populate COMMANDS_TO_RUN by executing the relevant method 280 | case $OPERATION in 281 | "" | help ) 282 | help 283 | ;; 284 | start ) 285 | start 286 | ;; 287 | stop ) 288 | stop 289 | ;; 290 | restart ) 291 | restart 292 | ;; 293 | wait ) 294 | wait 295 | ;; 296 | ps ) 297 | ps 298 | ;; 299 | log ) 300 | log 301 | ;; 302 | shell ) 303 | shell 304 | ;; 305 | pull ) 306 | pull 307 | ;; 308 | update ) 309 | update 310 | ;; 311 | wipe ) 312 | wipe 313 | ;; 314 | factory-reset ) 315 | factory-reset 316 | ;; 317 | services ) 318 | services 319 | ;; 320 | * ) 321 | help 322 | echo "ERROR: Invalid command: $OPERATION" 323 | exit 1 324 | ;; 325 | esac 326 | 327 | # Run or dry-run COMMANDS_TO_RUN 328 | for command in "${COMMANDS_TO_RUN[@]}" 329 | do 330 | if [ $DRY_RUN == 1 ]; then 331 | echo "$command" 332 | else 333 | $command 334 | EXIT=$? 335 | if [ $EXIT -ne 0 ]; then 336 | echo "streamr-docker-dev: error while running command: \"$command\"" 1>&2 337 | exit $EXIT 338 | fi 339 | fi 340 | done 341 | -------------------------------------------------------------------------------- /streamr-docker-dev/help_scripts.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | common_help() { 4 | echo " 5 | streamr-docker-dev – Streamr Docker Developer Environment. 6 | 7 | Usage: streamr-docker-dev [ [options] [--] ...] 8 | 9 | Commands: 10 | help show this screen 11 | start start services 12 | stop stop services 13 | services list all services 14 | restart stop and start services 15 | wait wait for all health checks to pass 16 | ps list docker containers 17 | log show logs 18 | shell shell into a container 19 | pull pulls latest versions of images 20 | update updates streamr-docker-dev by doing git pull 21 | wipe wipes the data persisted by all services 22 | factory-reset removes all docker images and generated files 23 | (WARNING: this deletes ALL docker images on 24 | your system and their state - 25 | not just Streamr-related ones!) 26 | Examples: 27 | streamr-docker-dev start 28 | streamr-docker-dev stop tracker-1 29 | streamr-docker-dev start --except tracker-1 --wait 30 | streamr-docker-dev log -f tracker-1 31 | streamr-docker-dev shell tracker-1 32 | streamr-docker-dev pull 33 | 34 | Options: 35 | --dry-run echo commands instead of executing them 36 | 37 | Show command-specific options: 38 | streamr-docker-dev help 39 | " 40 | } 41 | 42 | start_help() { 43 | echo " 44 | Starts the given services, or all services if none are specified. 45 | 46 | Usage: streamr-docker-dev start [--] ... 47 | 48 | Examples: 49 | streamr-docker-dev start 50 | streamr-docker-dev start tracker-1 51 | streamr-docker-dev start --except tracker-1 --wait 52 | 53 | Options: 54 | --except [service] start services except the one given 55 | --wait after starting, wait for services to become healthy (same as 'wait' command) 56 | --attached start in attached mode 57 | " 58 | } 59 | 60 | stop_help() { 61 | echo " 62 | Stops the given services, or all services if none are specified. 63 | 64 | Usage: streamr-docker-dev stop [options] [--] ... 65 | 66 | Examples: 67 | streamr-docker-dev stop 68 | streamr-docker-dev stop tracker-1 69 | " 70 | } 71 | 72 | restart_help() { 73 | echo " 74 | Restarts the given services, or all services if none are specified 75 | 76 | Usage: streamr-docker-dev restart [options] [--] ... 77 | 78 | Examples: 79 | streamr-docker-dev restart 80 | streamr-docker-dev restart tracker-1 81 | " 82 | } 83 | 84 | wait_help() { 85 | echo " 86 | Waits until pending health checks pass. Ignores services without health checks. 87 | 88 | Usage: streamr-docker-dev wait [options] 89 | 90 | Examples: 91 | streamr-docker-dev wait 92 | streamr-docker-dev wait --timeout 300 93 | 94 | Options: 95 | --timeout [sec] Sets the maximum time to wait. Default: 300 sec 96 | " 97 | } 98 | 99 | ps_help() { 100 | echo " 101 | Shows currently running services. If no services given, shows all 102 | 103 | Usage: streamr-docker-dev ps [...] 104 | 105 | Examples: 106 | streamr-docker-dev ps 107 | streamr-docker-dev ps tracker-1 108 | " 109 | } 110 | 111 | log_help() { 112 | echo " 113 | Shows the logs of the given services 114 | 115 | Usage: streamr-docker-dev log [[options] [--] ...] 116 | 117 | Examples: 118 | streamr-docker-dev log 119 | streamr-docker-dev log tracker-1 120 | streamr-docker-dev log -f tracker-1 121 | 122 | Options: 123 | -f --follow follow log in realtime 124 | " 125 | } 126 | 127 | shell_help() { 128 | echo " 129 | Opens an interactive shell into the target container. 130 | 131 | Usage: streamr-docker-dev shell 132 | 133 | Examples: 134 | streamr-docker-dev shell tracker-1 135 | " 136 | } 137 | 138 | pull_help() { 139 | echo " 140 | Pulls images defined in docker-compose files. If no services are given, pulls all of them. 141 | 142 | Usage: streamr-docker-dev pull [ [--] ...] 143 | 144 | Examples: 145 | streamr-docker-dev pull 146 | streamr-docker-dev pull tracker-1 147 | " 148 | } 149 | 150 | update_help() { 151 | echo " 152 | Updates streamr-docker-dev to the latest version by doing a "git pull" in the streamr-docker-dev directory. 153 | 154 | Usage: streamr-docker-dev update 155 | " 156 | } 157 | 158 | wipe_help() { 159 | echo " 160 | Wipes the data persisted by all services. 161 | 162 | Usage: streamr-docker-dev wipe 163 | " 164 | } 165 | 166 | factory-reset_help() { 167 | echo " 168 | Resets the environment by removing all persisted data and all Docker images. 169 | Warning: this deletes ALL docker images on your system and their state - 170 | not just Streamr-related ones! 171 | 172 | Usage: streamr-docker-dev factory-reset 173 | " 174 | } 175 | 176 | case $1 in 177 | "" ) 178 | common_help 179 | ;; 180 | "start" ) 181 | start_help 182 | ;; 183 | "stop" ) 184 | stop_help 185 | ;; 186 | "restart" ) 187 | restart_help 188 | ;; 189 | "wait" ) 190 | wait_help 191 | ;; 192 | "ps" ) 193 | ps_help 194 | ;; 195 | "log" ) 196 | log_help 197 | ;; 198 | "shell" ) 199 | shell_help 200 | ;; 201 | "pull" ) 202 | pull_help 203 | ;; 204 | "update" ) 205 | update_help 206 | ;; 207 | "wipe" ) 208 | wipe_help 209 | ;; 210 | "factory-reset" ) 211 | factory-reset_help 212 | ;; 213 | * ) common_help 214 | echo "ERROR: No help available for invalid command: $1" 215 | exit 1 216 | ;; 217 | esac 218 | -------------------------------------------------------------------------------- /yamllint-config.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # https://yamllint.readthedocs.io/en/stable/configuration.html 4 | 5 | extends: default 6 | rules: 7 | new-line-at-end-of-file: 8 | level: error 9 | hyphens: 10 | max-spaces-after: 1 11 | trailing-spaces: 12 | level: error 13 | line-length: disable 14 | truthy: disable 15 | comments: disable 16 | comments-indentation: disable 17 | braces: 18 | max-spaces-inside: 1 19 | level: error 20 | brackets: 21 | max-spaces-inside: 1 22 | level: error 23 | colons: 24 | max-spaces-after: -1 25 | level: error 26 | commas: 27 | max-spaces-after: -1 28 | level: error 29 | empty-lines: 30 | max: 3 31 | level: error 32 | --------------------------------------------------------------------------------