├── .gitignore ├── LICENSE ├── README.md ├── build.js ├── build ├── 00.html ├── 01.html ├── 02.html ├── 03.html ├── 04.html ├── 05.html ├── 06.html ├── 07.html ├── 08.html └── 09.html ├── github-markdown.css ├── index.html ├── package.json ├── problems ├── 00.md ├── 01.md ├── 02.md ├── 03.md ├── 04.md ├── 05.md ├── 06.md ├── 07.md ├── 08.md └── 09.md └── solutions ├── 01 ├── index.js ├── npm-debug.log └── package.json ├── 02 ├── index.js └── package.json ├── 03 ├── index.js └── package.json ├── 04 ├── index.js └── package.json ├── 05 ├── index.js ├── package.json └── user.message ├── 06 ├── index.js └── package.json └── 07 ├── index.js └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) Yoshua Wuyts 2 | 3 | Permission is hereby granted, free of charge, to any person ob- 4 | taining a copy of this software and associated documentation 5 | files (the "Software"), to deal in the Software without restric- 6 | tion, including without limitation the rights to use, copy, modi- 7 | fy, merge, publish, distribute, sublicense, and/or sell copies of 8 | the Software, and to permit persons to whom the Software is fur- 9 | nished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 16 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONIN- 17 | FRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 19 | ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # workshop-distributed-patterns 2 | Learn how to create robust multi-server applications in Node 3 | 4 | ## Contents 5 | - [intro](00.html) 6 | - [rpc](01.html) 7 | - [message queue](02.html) 8 | - [pubsub](03.html) 9 | - [encode protobufs](04.html) 10 | - [decode protobufs](05.html) 11 | - [pubsub with channels](06.html) 12 | - [extended REQREP](07.html) 13 | - [PUSHPULL pipeline](08.html) 14 | - [PUSHPULL pipeline with kill]() 15 | 16 | ## Thanks 17 | Shout out to [Mathias](https://github.com/mafintosh) for trailblazing this 18 | workshop format. 19 | 20 | ## See Also 21 | - [nodeschool.io](https://nodeschool.io) 22 | - [mafintosh/p2p-workshop](https://github.com/mafintosh/p2p-workshop) 23 | - [production ready node architecture](https://www.youtube.com/watch?v=9Qg9q5ABsvE) 24 | - [12 factor app](http://12factor.net/) 25 | 26 | ## License 27 | [MIT](https://tldrlegal.com/license/mit-license) 28 | 29 | [log]: https://engineering.linkedin.com/distributed-systems/log-what-every-software-engineer-should-know-about-real-time-datas-unifying 30 | -------------------------------------------------------------------------------- /build.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | var fs = require('fs') 3 | var marked = require('marked') 4 | var path = require('path') 5 | var rimraf = require('rimraf') 6 | 7 | var css = fs.readFileSync(path.join(__dirname, 'github-markdown.css'), 'utf-8') 8 | var base = path.join(__dirname, 'problems') 9 | 10 | rimraf.sync(path.join(__dirname, 'build')) 11 | fs.readdirSync(base).forEach(function (name) { 12 | var input = path.join(base, name) 13 | var output = path.join(__dirname, 'build', name.replace('.md', '.html')) 14 | var html = marked(fs.readFileSync(input, 'utf-8')) 15 | console.log(html) 16 | var file = ` 17 | 18 | 19 | Problem ${name.replace('.md', '')} 20 | 21 | 22 | ${html} 23 | 24 | ` 25 | 26 | try { 27 | fs.mkdirSync(path.join(__dirname, 'build')) 28 | } catch (err) { 29 | // do nothing 30 | } 31 | 32 | fs.writeFileSync(output, file) 33 | }) 34 | -------------------------------------------------------------------------------- /build/00.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Problem 00 5 | 661 | 662 |

Distributed Patterns Workshop

663 |

Hey there stranger, welcome to the distributed patterns workshop. In this 664 | workshop we'll be going over different async models using node(1), nanomsg 665 | and protobuf. We use these technologies because they happen to work well 666 | together, but the patterns we discuss in this workshop can be implemented using 667 | wildly different languages and technologies.

668 |

Requirements

669 |

This workshop assumes you're running a recent-ish version of Node (>=3.0) and 670 | are comfortable with writing JS. Should you have any questions: there'll be 671 | mentor(s) to help out, so don't hesitate to ask. Good luck!

672 |
673 |

Click here to go to the first exercise

674 | 675 | 676 | -------------------------------------------------------------------------------- /build/01.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Problem 01 5 | 661 | 662 |

1 - REQREP

663 |

REQUEST / REPLY is the cornerstone of virtually every networked 664 | application. A client calls a service, the service performs a computation and 665 | sends data back.

666 |

In browser land this is often referred to as an "API call", which almost always 667 | uses JSON as the encoding layer (JSON over HTTP/TCP). A browser talks to a 668 | server, and a server sends data back.

669 |

In distributed systems land this is often referred to as Remote Procedure Call 670 | (RPC, where "procedure" means "function"), which means servers call other 671 | servers to perform tasks, and send the result back once they're done. This is 672 | often also done using TCP, but opinions on which encoding layers to use 673 | differ (e.g. JSON, msgpack, protobuf, capnproto).

674 |

A REQRES call can be visualized as:

675 | 
    ┌──────────┐
676 |     │  Client  │
677 |     ├──────────┤
678 |     │   REQ    │
679 |     └──┬────▲──┘
680 |        │    │
681 |  Hello │    │ World
682 |        │    │
683 |     ┌──▼────┴──┐
684 |     │   REP    │
685 |     ├──────────┤
686 |     │  Server  │
687 |     └──────────┘
688 |
689 |

In this exercise we'll create a REQREP pair using nanomsg.

690 |

Task

691 |

Create a req/rep socket pair using nanomsg. Send the string "is this" 692 | from req, and reply with "the real RPC?" from rep. When the rep socket 693 | receives a string, log it. Also log when the req socket receives a response. 694 | Close both sockets once the call is complete.

695 |

Tips

696 | 708 |

Opening a client socket

709 | 
const nano = require('nanomsg')
710 | const req = nano.socket('req')
711 | req.bind('tcp://127.0.0.1:7789')
712 | req.send('hello')
713 |
714 |

Opening a host socket

715 | 
const nano = require('nanomsg')
716 | const rep = nano.socket('rep')
717 | rep.connect('tcp://127.0.0.1:7789')
718 | rep.on('data', function (data) {
719 |   console.log(String(data))
720 | })
721 |
722 |

See Also

723 | 727 |
728 |

Click here to go to the next exercise

729 | 730 | 731 | -------------------------------------------------------------------------------- /build/02.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Problem 02 5 | 661 | 662 |

2- PUSHPULL

663 |

To create computational pipelines, the PUSHPULL system comes in handy. It's a 664 | system where a single distributor (also known as "ventilator", because it 665 | "fans out" data - hah) puts data onto a queue, and multiple workers take data 666 | off the queue when they're ready to process it.

667 |

PUSHPULL can be used for parallel task distribution and collection patterns. 668 | An example of parallel task distribution is where workers take data and persist 669 | to a database. It can be visualized as:

670 | 
                ┌──────────┐
671 |                 │Ventilator│
672 |                 ├──────────┤
673 |                 │   Push   │
674 |                 └──────────┘
675 |                     Tasks
676 |        ┌──────────────┼──────────────┐
677 |        │              │              │
678 |  ┌─────▼────┐   ┌─────▼────┐   ┌─────▼────┐
679 |  │   PULL   │   │   PULL   │   │   PULL   │
680 |  ├──────────┤   ├──────────┤   ├──────────┤
681 |  │  Worker  │   │  Worker  │   │  Worker  │
682 |  └──────────┘   └──────────┘   └──────────┘
683 |
684 |

An example of the collection pattern is a mapreduce style 685 | system where large amounts of data is processed by workers, and collected by a 686 | sink at the end to form a single response. It can be visualized as:

687 | 
                ┌──────────┐
688 |                 │Ventilator│
689 |                 ├──────────┤
690 |                 │   Push   │
691 |                 └──────────┘
692 |                     Tasks
693 |        ┌──────────────┼──────────────┐
694 |        │              │              │
695 |  ┌─────▼────┐   ┌─────▼────┐   ┌─────▼────┐
696 |  │   PULL   │   │   PULL   │   │   PULL   │
697 |  ├──────────┤   ├──────────┤   ├──────────┤
698 |  │  Worker  │   │  Worker  │   │  Worker  │
699 |  ├──────────┤   ├──────────┤   ├──────────┤
700 |  │   PUSH   │   │   PUSH   │   │   PUSH   │
701 |  └──────────┘   └──────────┘   └──────────┘
702 |        │              │               │
703 |        └──────────────┼───────────────┘
704 |                    Results
705 |                 ┌─────▼────┐
706 |                 │   PULL   │
707 |                 ├──────────┤
708 |                 │   Sink   │
709 |                 └──────────┘
710 |
711 |

Task

712 |

Create a push/pull socket pair using nanomsg. Create a ventilator that pushes 713 | the numbers 1..100 onto a queue. Create a single worker that pulls a number 714 | off the queue every 20ms, sums it with the previously fetched numbers, and 715 | logs the result every time. When the number 100 is hit, it should close both 716 | sockets and log the string "done!".

717 |

Tips

718 | 728 |

See Also

729 | 733 |
734 |

Click here to go to the next exercise

735 | 736 | 737 | -------------------------------------------------------------------------------- /build/03.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Problem 03 5 | 661 | 662 |

3 - PUBSUB

663 |

Sometimes a system produces data that has multiple consumers. In computer 664 | science terms this is a "one-to-many" relationship.

665 | 
                ┌──────────┐
666 |                 │    PUB   │
667 |                 └──────────┘
668 |                       │
669 |        ┌──────────────┼──────────────┐
670 |        │              │              │
671 |  ┌─────▼────┐   ┌─────▼────┐   ┌─────▼────┐
672 |  │   SUB    │   │   SUB    │   │   SUB    │
673 |  └──────────┘   └──────────┘   └──────────┘
674 |
675 |

A single publisher produces data, and all subscribers receive the data. 676 | This is different from the PUSHPULL systems that we described in the last 677 | section, where data is randomly distribued between multiple consumers.

678 |

An example PUBSUB system is that of a change feed. Whenever values are 679 | persisted to a database, a publisher reads out the changes and starts 680 | publishing them. Subscribers then listen to those events and can do things like 681 | send push notifications, aggregate metrics or trigger other parts of the 682 | system.

683 |

Sending all data to every subscriber is quite inefficient, so in order to only 684 | send the relevant bits PUBSUB systems usually expose something called 685 | "channels" (or "topics" if you like). Subscribers usually know what data 686 | they're interested in, so they can hook into the desired channel.

687 |

Generally there are two kinds of feeds. The first kind is the traditional 688 | pub-sub where the server sends data and the clients receive it. This type of 689 | system is simple to implement, but lacks any type of backpressure.

690 |

The second kind of system is a stream with cursors, where the server stores a 691 | feed (indexed linked list of the sorts) of data and clients keep cursors on 692 | where in the feed they are. If a client disconnects they can fetch data right 693 | where they left off when they reconnect. This style of system was popularized 694 | by Apache's Kafka, and forms the basis of the (at the time of writing) upcoming 695 | Redis STREAM data type.

696 |

Nanomsg's PUBSUB primitive can be leveraged to create both types of feed. In 697 | this section we'll be creating a pub-sub system that can publish to multiple 698 | topics.

699 |

Task

700 |

Create a pub/sub socket pair using nanomsg. Send the message hello world 701 | from the sub to the pub. Then copy to the sub code to a different file, and 702 | run multiple subscribers on the same publisher.

703 |

Tips

704 | 709 |
710 |

Click here to go to the next exercise

711 | 712 | 713 | -------------------------------------------------------------------------------- /build/04.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Problem 04 5 | 661 | 662 |

04 - encode protocol buffers

663 |

Protocol buffers are a way of statically encoding messages as binary. This 664 | means that they can be transferred over the network more efficiently. They can 665 | also be versioned by incrementing the count for the value in the schema. 666 | Protocol buffers are not bound to any particular transport.

667 |

An example schema for a company:

668 | 
message Company {
669 |   required string name = 1;
670 |   repeated Employee employees = 2;
671 |   optional string country = 3;
672 | 
673 |   message Employee {
674 |     required string name = 1;
675 |     required uint32 age = 2;
676 |   }
677 | }
678 |
679 |

Schemas are generally suffixed as .proto.

680 |

Task

681 |

Create a CLI application that takes two arguments of "name" and "age" (argv 1, 682 | 2) and encodes them using a protocol buffer and save the encoded message to a 683 | file (user.message).

684 |

Tips

685 | 691 |

Bonus

692 | 696 |

See Also

697 | 702 |
703 |

Click here to go to the next exercise

704 | 705 | 706 | -------------------------------------------------------------------------------- /build/05.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Problem 05 5 | 661 | 662 |

05 - decode protocol buffers

663 |

Now that we've encoded a message, let's decode it!

664 |

Task

665 |

Create a CLI application that takes the location for a message, reads it and 666 | decodes it using the schema created in the previous exercise. Log the decoded 667 | message to stdout.

668 |

Tips

669 | 674 |
675 |

Click here to go to the next exercise

676 | 677 | 678 | -------------------------------------------------------------------------------- /build/06.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Problem 06 5 | 661 | 662 |

06 - pubsub with channels

663 |

Sometimes messages need to be sent to multiple subscribers, but not all 664 | subscribers are interested in all messages. Filtering messages can be done 665 | using channels.

666 |

Task

667 |

Create one publisher and two subscriber. sub1 should listen on the channels 668 | foo and bar. sub2 should listen on the channels beep and boop. Every 669 | 1000ms 4 messages should be sent. Shen a sub receives a message, it should 670 | echo the message prefixed by the name of the sub.

671 |

Tips

672 | 676 |
677 |

Click here to go to the next exercise

678 | 679 | 680 | -------------------------------------------------------------------------------- /build/07.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Problem 07 5 | 661 | 662 |

07 - sychronized pubsub

663 |

Order in distributed systems is a hard problem. When connecting multiple 664 | systems there are no guarantees that the other system will be available when 665 | you try to connect to it. So in order to do that it must be verified. Sometimes 666 | people choose to use timers / timeouts before connecting, but the more reliable 667 | option is to listen for events.

668 |

The sychronized pubsub combines multiple sockets to achieve this. It only 669 | starts emitting data once it knows that there are listeners to send data to:

670 | 
 ┌───────────┐
671 |  │ Publisher │
672 |  ├─────┬─────┤
673 |  │ REP │ PUB │
674 |  └▲──┬─┴──┬──┘
675 |   1  │    │
676 |   │  2    3
677 |  ┌┴──▼─┬──▼──┐
678 |  │ REQ │ SUB │
679 |  ├─────┴─────┤
680 |  │Subscriber │
681 |  └───────────┘
682 |
683 |

Exercise

684 |

Create a publisher in one file and a subscriber in another. Emit a number in 685 | the fibonacci sequence every 500ms, but only if a subscriber is available.

686 |

Tips

687 | 691 |

Bonus

692 | 700 |
701 |

Click here to go to the next exercise

702 | 703 | 704 | -------------------------------------------------------------------------------- /build/08.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Problem 08 5 | 661 | 662 |

08 - PUSHPULL pipeline

663 |

In the second exercise we created a PUSHPULL system. In this exercise we're 664 | going to expand it into a mapreduce:

665 | 
                ┌──────────┐
666 |                 │Ventilator│
667 |                 ├──────────┤
668 |                 │   Push   │
669 |                 └──────────┘
670 |                     Tasks
671 |        ┌──────────────┼──────────────┐
672 |        │              │              │
673 |  ┌─────▼────┐   ┌─────▼────┐   ┌─────▼────┐
674 |  │   PULL   │   │   PULL   │   │   PULL   │
675 |  ├──────────┤   ├──────────┤   ├──────────┤
676 |  │  Worker  │   │  Worker  │   │  Worker  │
677 |  ├──────────┤   ├──────────┤   ├──────────┤
678 |  │   PUSH   │   │   PUSH   │   │   PUSH   │
679 |  └──────────┘   └──────────┘   └──────────┘
680 |        │              │               │
681 |        └──────────────┼───────────────┘
682 |                    Results
683 |                 ┌─────▼────┐
684 |                 │   PULL   │
685 |                 ├──────────┤
686 |                 │   Sink   │
687 |                 └──────────┘
688 |
689 |

Task

690 |

Create a PUSHPULL system where the ventilator pushes the first 100 digits of 691 | the fibonnacci sequence onto workers, each worker squares the result and then 692 | pushes it into a sink that adds all values and logs the result.

693 |

Tips

694 | 699 |
700 |

Click here to go to the next exercise

701 | 702 | 703 | -------------------------------------------------------------------------------- /build/09.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Problem 09 5 | 661 | 662 |

09 - PUSH PULL with kill

663 |

Expanding on the previous exercise, we want to kill the workers once they're 664 | done and only start pushing data once all workers have been started up:

665 | 
                ┌──────────┐
666 |                 │Ventilator│
667 |                 ├──────────┤
668 |                 │   PUSH   │
669 |                 └──────────┘
670 |                     Tasks
671 |     ┌──────────────┬──┴───────────┐
672 |     │      ─ ─ ─ ─ ┼ ─ ─ ┬ ─ ─ ─ ─│─ ─ ─ ─ ─ ─
673 |     │     │        │              │     │     │
674 |  ┌──▼──┬──▼──┐  ┌──▼──┬──▼──┐  ┌──▼──┬──▼──┐
675 |  │PULL │ SUB │  │PULL │ SUB │  │PULL │ SUB │  │
676 |  ├─────┴─────┤  ├─────┴─────┤  ├─────┴─────┤
677 |  │  Worker   │  │  Worker   │  │  Worker   │  │
678 |  ├───────────┤  ├───────────┤  ├───────────┤
679 |  │   PUSH    │  │   PUSH    │  │   PUSH    │  │
680 |  └───────────┘  └───────────┘  └───────────┘
681 |        │              │              │        │
682 |        └──────────────┼──────────────┘
683 |                    Results                    │
684 |                 ┌─────▼────┐
685 |                 │   PULL   │                  │
686 |                 ├──────────┤
687 |                 │   Sink   │─ ─ KILL signal ─ ┘
688 |                 └──────────┘
689 |
690 |

Task

691 |

Create a PUSHPULL system that starts up 3 nodes, and only starts pushing data 692 | once the nodes have connected. Push the first 100 digits of the fibonnacci 693 | sequence onto workers, each worker squares the result and then pushes it into a 694 | sink that adds all values and logs the result. Once the sink has received all 695 | values, it should kill all workers.

696 |

Bonus

697 | 704 |

Fin

705 |

The end, congratulations on running through the workshop!

706 | 707 | 708 | -------------------------------------------------------------------------------- /github-markdown.css: -------------------------------------------------------------------------------- 1 | @font-face { 2 | font-family: octicons-anchor; 3 | src: url(data:font/woff;charset=utf-8;base64,d09GRgABAAAAAAYcAA0AAAAACjQAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAABGRlRNAAABMAAAABwAAAAca8vGTk9TLzIAAAFMAAAARAAAAFZG1VHVY21hcAAAAZAAAAA+AAABQgAP9AdjdnQgAAAB0AAAAAQAAAAEACICiGdhc3AAAAHUAAAACAAAAAj//wADZ2x5ZgAAAdwAAADRAAABEKyikaNoZWFkAAACsAAAAC0AAAA2AtXoA2hoZWEAAALgAAAAHAAAACQHngNFaG10eAAAAvwAAAAQAAAAEAwAACJsb2NhAAADDAAAAAoAAAAKALIAVG1heHAAAAMYAAAAHwAAACABEAB2bmFtZQAAAzgAAALBAAAFu3I9x/Nwb3N0AAAF/AAAAB0AAAAvaoFvbwAAAAEAAAAAzBdyYwAAAADP2IQvAAAAAM/bz7t4nGNgZGFgnMDAysDB1Ml0hoGBoR9CM75mMGLkYGBgYmBlZsAKAtJcUxgcPsR8iGF2+O/AEMPsznAYKMwIkgMA5REMOXicY2BgYGaAYBkGRgYQsAHyGMF8FgYFIM0ChED+h5j//yEk/3KoSgZGNgYYk4GRCUgwMaACRoZhDwCs7QgGAAAAIgKIAAAAAf//AAJ4nHWMMQrCQBBF/0zWrCCIKUQsTDCL2EXMohYGSSmorScInsRGL2DOYJe0Ntp7BK+gJ1BxF1stZvjz/v8DRghQzEc4kIgKwiAppcA9LtzKLSkdNhKFY3HF4lK69ExKslx7Xa+vPRVS43G98vG1DnkDMIBUgFN0MDXflU8tbaZOUkXUH0+U27RoRpOIyCKjbMCVejwypzJJG4jIwb43rfl6wbwanocrJm9XFYfskuVC5K/TPyczNU7b84CXcbxks1Un6H6tLH9vf2LRnn8Ax7A5WQAAAHicY2BkYGAA4teL1+yI57f5ysDNwgAC529f0kOmWRiYVgEpDgYmEA8AUzEKsQAAAHicY2BkYGB2+O/AEMPCAAJAkpEBFbAAADgKAe0EAAAiAAAAAAQAAAAEAAAAAAAAKgAqACoAiAAAeJxjYGRgYGBhsGFgYgABEMkFhAwM/xn0QAIAD6YBhwB4nI1Ty07cMBS9QwKlQapQW3VXySvEqDCZGbGaHULiIQ1FKgjWMxknMfLEke2A+IJu+wntrt/QbVf9gG75jK577Lg8K1qQPCfnnnt8fX1NRC/pmjrk/zprC+8D7tBy9DHgBXoWfQ44Av8t4Bj4Z8CLtBL9CniJluPXASf0Lm4CXqFX8Q84dOLnMB17N4c7tBo1AS/Qi+hTwBH4rwHHwN8DXqQ30XXAS7QaLwSc0Gn8NuAVWou/gFmnjLrEaEh9GmDdDGgL3B4JsrRPDU2hTOiMSuJUIdKQQayiAth69r6akSSFqIJuA19TrzCIaY8sIoxyrNIrL//pw7A2iMygkX5vDj+G+kuoLdX4GlGK/8Lnlz6/h9MpmoO9rafrz7ILXEHHaAx95s9lsI7AHNMBWEZHULnfAXwG9/ZqdzLI08iuwRloXE8kfhXYAvE23+23DU3t626rbs8/8adv+9DWknsHp3E17oCf+Z48rvEQNZ78paYM38qfk3v/u3l3u3GXN2Dmvmvpf1Srwk3pB/VSsp512bA/GG5i2WJ7wu430yQ5K3nFGiOqgtmSB5pJVSizwaacmUZzZhXLlZTq8qGGFY2YcSkqbth6aW1tRmlaCFs2016m5qn36SbJrqosG4uMV4aP2PHBmB3tjtmgN2izkGQyLWprekbIntJFing32a5rKWCN/SdSoga45EJykyQ7asZvHQ8PTm6cslIpwyeyjbVltNikc2HTR7YKh9LBl9DADC0U/jLcBZDKrMhUBfQBvXRzLtFtjU9eNHKin0x5InTqb8lNpfKv1s1xHzTXRqgKzek/mb7nB8RZTCDhGEX3kK/8Q75AmUM/eLkfA+0Hi908Kx4eNsMgudg5GLdRD7a84npi+YxNr5i5KIbW5izXas7cHXIMAau1OueZhfj+cOcP3P8MNIWLyYOBuxL6DRylJ4cAAAB4nGNgYoAALjDJyIAOWMCiTIxMLDmZedkABtIBygAAAA==) format('woff'); 4 | } 5 | 6 | .markdown-body { 7 | -webkit-text-size-adjust: 100%; 8 | text-size-adjust: 100%; 9 | color: #333; 10 | font-family: "Helvetica Neue", Helvetica, "Segoe UI", Arial, freesans, sans-serif; 11 | font-size: 16px; 12 | line-height: 1.6; 13 | word-wrap: break-word; 14 | } 15 | 16 | .markdown-body a { 17 | background-color: transparent; 18 | } 19 | 20 | .markdown-body a:active, 21 | .markdown-body a:hover { 22 | outline: 0; 23 | } 24 | 25 | .markdown-body strong { 26 | font-weight: bold; 27 | } 28 | 29 | .markdown-body h1 { 30 | font-size: 2em; 31 | margin: 0.67em 0; 32 | } 33 | 34 | .markdown-body img { 35 | border: 0; 36 | } 37 | 38 | .markdown-body hr { 39 | box-sizing: content-box; 40 | height: 0; 41 | } 42 | 43 | .markdown-body pre { 44 | overflow: auto; 45 | } 46 | 47 | .markdown-body code, 48 | .markdown-body kbd, 49 | .markdown-body pre { 50 | font-family: monospace, monospace; 51 | font-size: 1em; 52 | } 53 | 54 | .markdown-body input { 55 | color: inherit; 56 | font: inherit; 57 | margin: 0; 58 | } 59 | 60 | .markdown-body html input[disabled] { 61 | cursor: default; 62 | } 63 | 64 | .markdown-body input { 65 | line-height: normal; 66 | } 67 | 68 | .markdown-body input[type="checkbox"] { 69 | box-sizing: border-box; 70 | padding: 0; 71 | } 72 | 73 | .markdown-body table { 74 | border-collapse: collapse; 75 | border-spacing: 0; 76 | } 77 | 78 | .markdown-body td, 79 | .markdown-body th { 80 | padding: 0; 81 | } 82 | 83 | .markdown-body * { 84 | box-sizing: border-box; 85 | } 86 | 87 | .markdown-body input { 88 | font: 13px/1.4 Helvetica, arial, nimbussansl, liberationsans, freesans, clean, sans-serif, "Segoe UI Emoji", "Segoe UI Symbol"; 89 | } 90 | 91 | .markdown-body a { 92 | color: #4078c0; 93 | text-decoration: none; 94 | } 95 | 96 | .markdown-body a:hover, 97 | .markdown-body a:active { 98 | text-decoration: underline; 99 | } 100 | 101 | .markdown-body hr { 102 | height: 0; 103 | margin: 15px 0; 104 | overflow: hidden; 105 | background: transparent; 106 | border: 0; 107 | border-bottom: 1px solid #ddd; 108 | } 109 | 110 | .markdown-body hr:before { 111 | display: table; 112 | content: ""; 113 | } 114 | 115 | .markdown-body hr:after { 116 | display: table; 117 | clear: both; 118 | content: ""; 119 | } 120 | 121 | .markdown-body h1, 122 | .markdown-body h2, 123 | .markdown-body h3, 124 | .markdown-body h4, 125 | .markdown-body h5, 126 | .markdown-body h6 { 127 | margin-top: 15px; 128 | margin-bottom: 15px; 129 | line-height: 1.1; 130 | } 131 | 132 | .markdown-body h1 { 133 | font-size: 30px; 134 | } 135 | 136 | .markdown-body h2 { 137 | font-size: 21px; 138 | } 139 | 140 | .markdown-body h3 { 141 | font-size: 16px; 142 | } 143 | 144 | .markdown-body h4 { 145 | font-size: 14px; 146 | } 147 | 148 | .markdown-body h5 { 149 | font-size: 12px; 150 | } 151 | 152 | .markdown-body h6 { 153 | font-size: 11px; 154 | } 155 | 156 | .markdown-body blockquote { 157 | margin: 0; 158 | } 159 | 160 | .markdown-body ul, 161 | .markdown-body ol { 162 | padding: 0; 163 | margin-top: 0; 164 | margin-bottom: 0; 165 | } 166 | 167 | .markdown-body ol ol, 168 | .markdown-body ul ol { 169 | list-style-type: lower-roman; 170 | } 171 | 172 | .markdown-body ul ul ol, 173 | .markdown-body ul ol ol, 174 | .markdown-body ol ul ol, 175 | .markdown-body ol ol ol { 176 | list-style-type: lower-alpha; 177 | } 178 | 179 | .markdown-body dd { 180 | margin-left: 0; 181 | } 182 | 183 | .markdown-body code { 184 | font-family: Consolas, "Liberation Mono", Menlo, Courier, monospace; 185 | font-size: 12px; 186 | } 187 | 188 | .markdown-body pre { 189 | margin-top: 0; 190 | margin-bottom: 0; 191 | font: 12px Consolas, "Liberation Mono", Menlo, Courier, monospace; 192 | } 193 | 194 | .markdown-body .select::-ms-expand { 195 | opacity: 0; 196 | } 197 | 198 | .markdown-body .octicon { 199 | font: normal normal normal 16px/1 octicons-anchor; 200 | display: inline-block; 201 | text-decoration: none; 202 | text-rendering: auto; 203 | -webkit-font-smoothing: antialiased; 204 | -moz-osx-font-smoothing: grayscale; 205 | -webkit-user-select: none; 206 | -moz-user-select: none; 207 | -ms-user-select: none; 208 | user-select: none; 209 | } 210 | 211 | .markdown-body .octicon-link:before { 212 | content: '\f05c'; 213 | } 214 | 215 | .markdown-body>*:first-child { 216 | margin-top: 0 !important; 217 | } 218 | 219 | .markdown-body>*:last-child { 220 | margin-bottom: 0 !important; 221 | } 222 | 223 | .markdown-body a:not([href]) { 224 | color: inherit; 225 | text-decoration: none; 226 | } 227 | 228 | .markdown-body .anchor { 229 | display: inline-block; 230 | padding-right: 2px; 231 | margin-left: -18px; 232 | } 233 | 234 | .markdown-body .anchor:focus { 235 | outline: none; 236 | } 237 | 238 | .markdown-body h1, 239 | .markdown-body h2, 240 | .markdown-body h3, 241 | .markdown-body h4, 242 | .markdown-body h5, 243 | .markdown-body h6 { 244 | margin-top: 1em; 245 | margin-bottom: 16px; 246 | font-weight: bold; 247 | line-height: 1.4; 248 | } 249 | 250 | .markdown-body h1 .octicon-link, 251 | .markdown-body h2 .octicon-link, 252 | .markdown-body h3 .octicon-link, 253 | .markdown-body h4 .octicon-link, 254 | .markdown-body h5 .octicon-link, 255 | .markdown-body h6 .octicon-link { 256 | color: #000; 257 | vertical-align: middle; 258 | visibility: hidden; 259 | } 260 | 261 | .markdown-body h1:hover .anchor, 262 | .markdown-body h2:hover .anchor, 263 | .markdown-body h3:hover .anchor, 264 | .markdown-body h4:hover .anchor, 265 | .markdown-body h5:hover .anchor, 266 | .markdown-body h6:hover .anchor { 267 | text-decoration: none; 268 | } 269 | 270 | .markdown-body h1:hover .anchor .octicon-link, 271 | .markdown-body h2:hover .anchor .octicon-link, 272 | .markdown-body h3:hover .anchor .octicon-link, 273 | .markdown-body h4:hover .anchor .octicon-link, 274 | .markdown-body h5:hover .anchor .octicon-link, 275 | .markdown-body h6:hover .anchor .octicon-link { 276 | visibility: visible; 277 | } 278 | 279 | .markdown-body h1 { 280 | padding-bottom: 0.3em; 281 | font-size: 2.25em; 282 | line-height: 1.2; 283 | border-bottom: 1px solid #eee; 284 | } 285 | 286 | .markdown-body h1 .anchor { 287 | line-height: 1; 288 | } 289 | 290 | .markdown-body h2 { 291 | padding-bottom: 0.3em; 292 | font-size: 1.75em; 293 | line-height: 1.225; 294 | border-bottom: 1px solid #eee; 295 | } 296 | 297 | .markdown-body h2 .anchor { 298 | line-height: 1; 299 | } 300 | 301 | .markdown-body h3 { 302 | font-size: 1.5em; 303 | line-height: 1.43; 304 | } 305 | 306 | .markdown-body h3 .anchor { 307 | line-height: 1.2; 308 | } 309 | 310 | .markdown-body h4 { 311 | font-size: 1.25em; 312 | } 313 | 314 | .markdown-body h4 .anchor { 315 | line-height: 1.2; 316 | } 317 | 318 | .markdown-body h5 { 319 | font-size: 1em; 320 | } 321 | 322 | .markdown-body h5 .anchor { 323 | line-height: 1.1; 324 | } 325 | 326 | .markdown-body h6 { 327 | font-size: 1em; 328 | color: #777; 329 | } 330 | 331 | .markdown-body h6 .anchor { 332 | line-height: 1.1; 333 | } 334 | 335 | .markdown-body p, 336 | .markdown-body blockquote, 337 | .markdown-body ul, 338 | .markdown-body ol, 339 | .markdown-body dl, 340 | .markdown-body table, 341 | .markdown-body pre { 342 | margin-top: 0; 343 | margin-bottom: 16px; 344 | } 345 | 346 | .markdown-body hr { 347 | height: 4px; 348 | padding: 0; 349 | margin: 16px 0; 350 | background-color: #e7e7e7; 351 | border: 0 none; 352 | } 353 | 354 | .markdown-body ul, 355 | .markdown-body ol { 356 | padding-left: 2em; 357 | } 358 | 359 | .markdown-body ul ul, 360 | .markdown-body ul ol, 361 | .markdown-body ol ol, 362 | .markdown-body ol ul { 363 | margin-top: 0; 364 | margin-bottom: 0; 365 | } 366 | 367 | .markdown-body li>p { 368 | margin-top: 16px; 369 | } 370 | 371 | .markdown-body dl { 372 | padding: 0; 373 | } 374 | 375 | .markdown-body dl dt { 376 | padding: 0; 377 | margin-top: 16px; 378 | font-size: 1em; 379 | font-style: italic; 380 | font-weight: bold; 381 | } 382 | 383 | .markdown-body dl dd { 384 | padding: 0 16px; 385 | margin-bottom: 16px; 386 | } 387 | 388 | .markdown-body blockquote { 389 | padding: 0 15px; 390 | color: #777; 391 | border-left: 4px solid #ddd; 392 | } 393 | 394 | .markdown-body blockquote>:first-child { 395 | margin-top: 0; 396 | } 397 | 398 | .markdown-body blockquote>:last-child { 399 | margin-bottom: 0; 400 | } 401 | 402 | .markdown-body table { 403 | display: block; 404 | width: 100%; 405 | overflow: auto; 406 | word-break: normal; 407 | word-break: keep-all; 408 | } 409 | 410 | .markdown-body table th { 411 | font-weight: bold; 412 | } 413 | 414 | .markdown-body table th, 415 | .markdown-body table td { 416 | padding: 6px 13px; 417 | border: 1px solid #ddd; 418 | } 419 | 420 | .markdown-body table tr { 421 | background-color: #fff; 422 | border-top: 1px solid #ccc; 423 | } 424 | 425 | .markdown-body table tr:nth-child(2n) { 426 | background-color: #f8f8f8; 427 | } 428 | 429 | .markdown-body img { 430 | max-width: 100%; 431 | box-sizing: content-box; 432 | background-color: #fff; 433 | } 434 | 435 | .markdown-body code { 436 | padding: 0; 437 | padding-top: 0.2em; 438 | padding-bottom: 0.2em; 439 | margin: 0; 440 | font-size: 85%; 441 | background-color: rgba(0,0,0,0.04); 442 | border-radius: 3px; 443 | } 444 | 445 | .markdown-body code:before, 446 | .markdown-body code:after { 447 | letter-spacing: -0.2em; 448 | content: "\00a0"; 449 | } 450 | 451 | .markdown-body pre>code { 452 | padding: 0; 453 | margin: 0; 454 | font-size: 100%; 455 | word-break: normal; 456 | white-space: pre; 457 | background: transparent; 458 | border: 0; 459 | } 460 | 461 | .markdown-body .highlight { 462 | margin-bottom: 16px; 463 | } 464 | 465 | .markdown-body .highlight pre, 466 | .markdown-body pre { 467 | padding: 16px; 468 | overflow: auto; 469 | font-size: 85%; 470 | line-height: 1.45; 471 | background-color: #f7f7f7; 472 | border-radius: 3px; 473 | } 474 | 475 | .markdown-body .highlight pre { 476 | margin-bottom: 0; 477 | word-break: normal; 478 | } 479 | 480 | .markdown-body pre { 481 | word-wrap: normal; 482 | } 483 | 484 | .markdown-body pre code { 485 | display: inline; 486 | max-width: initial; 487 | padding: 0; 488 | margin: 0; 489 | overflow: initial; 490 | line-height: inherit; 491 | word-wrap: normal; 492 | background-color: transparent; 493 | border: 0; 494 | } 495 | 496 | .markdown-body pre code:before, 497 | .markdown-body pre code:after { 498 | content: normal; 499 | } 500 | 501 | .markdown-body kbd { 502 | display: inline-block; 503 | padding: 3px 5px; 504 | font-size: 11px; 505 | line-height: 10px; 506 | color: #555; 507 | vertical-align: middle; 508 | background-color: #fcfcfc; 509 | border: solid 1px #ccc; 510 | border-bottom-color: #bbb; 511 | border-radius: 3px; 512 | box-shadow: inset 0 -1px 0 #bbb; 513 | } 514 | 515 | .markdown-body .pl-c { 516 | color: #969896; 517 | } 518 | 519 | .markdown-body .pl-c1, 520 | .markdown-body .pl-s .pl-v { 521 | color: #0086b3; 522 | } 523 | 524 | .markdown-body .pl-e, 525 | .markdown-body .pl-en { 526 | color: #795da3; 527 | } 528 | 529 | .markdown-body .pl-s .pl-s1, 530 | .markdown-body .pl-smi { 531 | color: #333; 532 | } 533 | 534 | .markdown-body .pl-ent { 535 | color: #63a35c; 536 | } 537 | 538 | .markdown-body .pl-k { 539 | color: #a71d5d; 540 | } 541 | 542 | .markdown-body .pl-pds, 543 | .markdown-body .pl-s, 544 | .markdown-body .pl-s .pl-pse .pl-s1, 545 | .markdown-body .pl-sr, 546 | .markdown-body .pl-sr .pl-cce, 547 | .markdown-body .pl-sr .pl-sra, 548 | .markdown-body .pl-sr .pl-sre { 549 | color: #183691; 550 | } 551 | 552 | .markdown-body .pl-v { 553 | color: #ed6a43; 554 | } 555 | 556 | .markdown-body .pl-id { 557 | color: #b52a1d; 558 | } 559 | 560 | .markdown-body .pl-ii { 561 | background-color: #b52a1d; 562 | color: #f8f8f8; 563 | } 564 | 565 | .markdown-body .pl-sr .pl-cce { 566 | color: #63a35c; 567 | font-weight: bold; 568 | } 569 | 570 | .markdown-body .pl-ml { 571 | color: #693a17; 572 | } 573 | 574 | .markdown-body .pl-mh, 575 | .markdown-body .pl-mh .pl-en, 576 | .markdown-body .pl-ms { 577 | color: #1d3e81; 578 | font-weight: bold; 579 | } 580 | 581 | .markdown-body .pl-mq { 582 | color: #008080; 583 | } 584 | 585 | .markdown-body .pl-mi { 586 | color: #333; 587 | font-style: italic; 588 | } 589 | 590 | .markdown-body .pl-mb { 591 | color: #333; 592 | font-weight: bold; 593 | } 594 | 595 | .markdown-body .pl-md { 596 | background-color: #ffecec; 597 | color: #bd2c00; 598 | } 599 | 600 | .markdown-body .pl-mi1 { 601 | background-color: #eaffea; 602 | color: #55a532; 603 | } 604 | 605 | .markdown-body .pl-mdr { 606 | color: #795da3; 607 | font-weight: bold; 608 | } 609 | 610 | .markdown-body .pl-mo { 611 | color: #1d3e81; 612 | } 613 | 614 | .markdown-body kbd { 615 | display: inline-block; 616 | padding: 3px 5px; 617 | font: 11px Consolas, "Liberation Mono", Menlo, Courier, monospace; 618 | line-height: 10px; 619 | color: #555; 620 | vertical-align: middle; 621 | background-color: #fcfcfc; 622 | border: solid 1px #ccc; 623 | border-bottom-color: #bbb; 624 | border-radius: 3px; 625 | box-shadow: inset 0 -1px 0 #bbb; 626 | } 627 | 628 | .markdown-body:before { 629 | display: table; 630 | content: ""; 631 | } 632 | 633 | .markdown-body:after { 634 | display: table; 635 | clear: both; 636 | content: ""; 637 | } 638 | 639 | .markdown-body .task-list-item { 640 | list-style-type: none; 641 | } 642 | 643 | .markdown-body .task-list-item+.task-list-item { 644 | margin-top: 3px; 645 | } 646 | 647 | .markdown-body .task-list-item input { 648 | margin: 0 0.35em 0.25em -1.6em; 649 | vertical-align: middle; 650 | } 651 | 652 | .markdown-body :checked+.radio-label { 653 | z-index: 1; 654 | position: relative; 655 | border-color: #4078c0; 656 | } -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | A Distributed Patterns Workshop 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "workshop-distributed-patterns", 3 | "version": "1.0.0", 4 | "description": "Learn how to create robust multi-server applications in Node", 5 | "main": "build.js", 6 | "private": "true", 7 | "scripts": { 8 | "build": "./build.js" 9 | }, 10 | "author": "Yoshua Wuyts ", 11 | "license": "MIT", 12 | "dependencies": { 13 | "marked": "^0.3.5", 14 | "rimraf": "^2.5.2" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /problems/00.md: -------------------------------------------------------------------------------- 1 | # Distributed Patterns Workshop 2 | Hey there stranger, welcome to the distributed patterns workshop. In this 3 | workshop we'll be going over different async models using `node(1)`, `nanomsg` 4 | and `protobuf`. We use these technologies because they happen to work well 5 | together, but the patterns we discuss in this workshop can be implemented using 6 | wildly different languages and technologies. 7 | 8 | ## Requirements 9 | This workshop assumes you're running a recent-ish version of Node (`>=3.0`) and 10 | are comfortable with writing JS. Should you have any questions: there'll be 11 | mentor(s) to help out, so don't hesitate to ask. Good luck! 12 | 13 | --- 14 | [Click here to go to the first exercise](01.html) 15 | -------------------------------------------------------------------------------- /problems/01.md: -------------------------------------------------------------------------------- 1 | # 1 - REQREP 2 | `REQUEST / REPLY` is the cornerstone of virtually every networked 3 | application. A client calls a service, the service performs a computation and 4 | sends data back. 5 | 6 | In browser land this is often referred to as an "API call", which almost always 7 | uses JSON as the encoding layer (`JSON` over `HTTP/TCP`). A browser talks to a 8 | server, and a server sends data back. 9 | 10 | In distributed systems land this is often referred to as Remote Procedure Call 11 | (`RPC`, where "procedure" means "function"), which means servers call other 12 | servers to perform tasks, and send the result back once they're done. This is 13 | often also done using `TCP`, but opinions on which encoding layers to use 14 | differ (e.g. `JSON`, `msgpack`, `protobuf`, `capnproto`). 15 | 16 | A `REQRES` call can be visualized as: 17 | ```txt 18 | ┌──────────┐ 19 | │ Client │ 20 | ├──────────┤ 21 | │ REQ │ 22 | └──┬────▲──┘ 23 | │ │ 24 | Hello │ │ World 25 | │ │ 26 | ┌──▼────┴──┐ 27 | │ REP │ 28 | ├──────────┤ 29 | │ Server │ 30 | └──────────┘ 31 | ``` 32 | 33 | In this exercise we'll create a `REQREP` pair using `nanomsg`. 34 | 35 | ## Task 36 | Create a `req/rep` socket pair using `nanomsg`. Send the string `"is this"` 37 | from `req`, and reply with `"the real RPC?"` from `rep`. When the `rep` socket 38 | receives a string, log it. Also log when the `req` socket receives a response. 39 | Close both sockets once the call is complete. 40 | 41 | ## Tips 42 | - just like with `HTTP`, `REQREP` socket pairs need to connect to the same uri 43 | - use `tcp://` instead of other connection methods to simulate real-world 44 | behavior locally - Erlang does this too, and to great success 45 | - the `RES` socket holds the `TCP` socket; the `connect` keyword signals that 46 | - data is passed as Node buffers between sockets. Wrap the received buffer in 47 | `String()` to cast it to a string. 48 | - all `nanosmg` sockets in Node are implemented as streams, therefore listeners 49 | such as `.on('data')` and `.on('error')` work as expected. 50 | - `nanomsg` is blocking, therefore it's safe to use `res.send()` with multiple 51 | requests coming in. 52 | 53 | ## Opening a client socket 54 | ```js 55 | const nano = require('nanomsg') 56 | const req = nano.socket('req') 57 | req.bind('tcp://127.0.0.1:7789') 58 | req.send('hello') 59 | ``` 60 | 61 | ## Opening a host socket 62 | ```js 63 | const nano = require('nanomsg') 64 | const rep = nano.socket('rep') 65 | rep.connect('tcp://127.0.0.1:7789') 66 | rep.on('data', function (data) { 67 | console.log(String(data)) 68 | }) 69 | ``` 70 | 71 | ## See Also 72 | - http://zguide.zeromq.org/page:all#header-11 73 | - [node-nanomsg/examples/reqrep](https://github.com/nickdesaulniers/node-nanomsg/blob/master/examples/reqrep.js) 74 | 75 | --- 76 | [Click here to go to the next exercise](02.html) 77 | -------------------------------------------------------------------------------- /problems/02.md: -------------------------------------------------------------------------------- 1 | # 2- PUSHPULL 2 | 3 | To create computational pipelines, the `PUSHPULL` system comes in handy. It's a 4 | system where a single distributor (also known as "ventilator", because it 5 | "fans out" data - hah) puts data onto a queue, and multiple workers take data 6 | off the queue when they're ready to process it. 7 | 8 | `PUSHPULL` can be used for parallel task distribution and collection patterns. 9 | An example of parallel task distribution is where workers take data and persist 10 | to a database. It can be visualized as: 11 | ```txt 12 | ┌──────────┐ 13 | │Ventilator│ 14 | ├──────────┤ 15 | │ Push │ 16 | └──────────┘ 17 | Tasks 18 | ┌──────────────┼──────────────┐ 19 | │ │ │ 20 | ┌─────▼────┐ ┌─────▼────┐ ┌─────▼────┐ 21 | │ PULL │ │ PULL │ │ PULL │ 22 | ├──────────┤ ├──────────┤ ├──────────┤ 23 | │ Worker │ │ Worker │ │ Worker │ 24 | └──────────┘ └──────────┘ └──────────┘ 25 | ``` 26 | 27 | An example of the collection pattern is a `mapreduce` style 28 | system where large amounts of data is processed by workers, and collected by a 29 | sink at the end to form a single response. It can be visualized as: 30 | ```txt 31 | ┌──────────┐ 32 | │Ventilator│ 33 | ├──────────┤ 34 | │ Push │ 35 | └──────────┘ 36 | Tasks 37 | ┌──────────────┼──────────────┐ 38 | │ │ │ 39 | ┌─────▼────┐ ┌─────▼────┐ ┌─────▼────┐ 40 | │ PULL │ │ PULL │ │ PULL │ 41 | ├──────────┤ ├──────────┤ ├──────────┤ 42 | │ Worker │ │ Worker │ │ Worker │ 43 | ├──────────┤ ├──────────┤ ├──────────┤ 44 | │ PUSH │ │ PUSH │ │ PUSH │ 45 | └──────────┘ └──────────┘ └──────────┘ 46 | │ │ │ 47 | └──────────────┼───────────────┘ 48 | Results 49 | ┌─────▼────┐ 50 | │ PULL │ 51 | ├──────────┤ 52 | │ Sink │ 53 | └──────────┘ 54 | ``` 55 | 56 | ## Task 57 | Create a `push/pull` socket pair using nanomsg. Create a ventilator that pushes 58 | the numbers `1..100` onto a queue. Create a single worker that pulls a number 59 | off the queue every `20ms`, sums it with the previously fetched numbers, and 60 | logs the result every time. When the number `100` is hit, it should close both 61 | sockets and log the string `"done!"`. 62 | 63 | ## Tips 64 | - all nanomsg sockets are `Duplex` streams 65 | - use `.pause()` on your pull socket to signal it should halt 66 | - use `.resume()` on your pull socket to signal it's ready to resume 67 | - use `setTimeout(cb, timeout)` to halt execution for n miliseconds 68 | - data is passed as Node buffers between sockets. Wrap the received buffer in 69 | `Number()` to cast it to a number. 70 | - don't worry, we'll create the complete `PUSHPULL` pipeline in a later 71 | exercise 72 | 73 | ## See Also 74 | - [node-nanomsg/examples/pipeline](https://github.com/nickdesaulniers/node-nanomsg/blob/master/examples/pipeline.js) 75 | - [nodejs/api/stream](https://nodejs.org/api/stream.html) 76 | 77 | --- 78 | [Click here to go to the next exercise](03.html) 79 | -------------------------------------------------------------------------------- /problems/03.md: -------------------------------------------------------------------------------- 1 | # 3 - PUBSUB 2 | Sometimes a system produces data that has multiple consumers. In computer 3 | science terms this is a "one-to-many" relationship. 4 | 5 | ```txt 6 | ┌──────────┐ 7 | │ PUB │ 8 | └──────────┘ 9 | │ 10 | ┌──────────────┼──────────────┐ 11 | │ │ │ 12 | ┌─────▼────┐ ┌─────▼────┐ ┌─────▼────┐ 13 | │ SUB │ │ SUB │ │ SUB │ 14 | └──────────┘ └──────────┘ └──────────┘ 15 | ``` 16 | 17 | A single publisher produces data, and __all__ subscribers receive the data. 18 | This is different from the PUSHPULL systems that we described in the last 19 | section, where data is randomly distribued between multiple consumers. 20 | 21 | An example PUBSUB system is that of a change feed. Whenever values are 22 | persisted to a database, a publisher reads out the changes and starts 23 | publishing them. Subscribers then listen to those events and can do things like 24 | send push notifications, aggregate metrics or trigger other parts of the 25 | system. 26 | 27 | Sending all data to every subscriber is quite inefficient, so in order to only 28 | send the relevant bits PUBSUB systems usually expose something called 29 | "channels" (or "topics" if you like). Subscribers usually know what data 30 | they're interested in, so they can hook into the desired channel. 31 | 32 | Generally there are two kinds of feeds. The first kind is the traditional 33 | pub-sub where the server sends data and the clients receive it. This type of 34 | system is simple to implement, but lacks any type of backpressure. 35 | 36 | The second kind of system is a stream with cursors, where the server stores a 37 | feed (indexed linked list of the sorts) of data and clients keep cursors on 38 | where in the feed they are. If a client disconnects they can fetch data right 39 | where they left off when they reconnect. This style of system was popularized 40 | by Apache's Kafka, and forms the basis of the (at the time of writing) upcoming 41 | Redis `STREAM` data type. 42 | 43 | Nanomsg's `PUBSUB` primitive can be leveraged to create both types of feed. In 44 | this section we'll be creating a pub-sub system that can publish to multiple 45 | topics. 46 | 47 | ## Task 48 | Create a `pub/sub` socket pair using nanomsg. Send the message `hello world` 49 | from the sub to the pub. Then copy to the `sub` code to a different file, and 50 | run multiple subscribers on the same publisher. 51 | 52 | ## Tips 53 | - This should be pretty similar to the previous exercise 54 | - Try and separate concerns so that moving one file to another can be done 55 | smoothly 56 | 57 | --- 58 | [Click here to go to the next exercise](04.html) 59 | -------------------------------------------------------------------------------- /problems/04.md: -------------------------------------------------------------------------------- 1 | # 04 - encode protocol buffers 2 | Protocol buffers are a way of statically encoding messages as binary. This 3 | means that they can be transferred over the network more efficiently. They can 4 | also be versioned by incrementing the count for the value in the schema. 5 | Protocol buffers are not bound to any particular transport. 6 | 7 | An example schema for a company: 8 | ```protobuf 9 | message Company { 10 | required string name = 1; 11 | repeated Employee employees = 2; 12 | optional string country = 3; 13 | 14 | message Employee { 15 | required string name = 1; 16 | required uint32 age = 2; 17 | } 18 | } 19 | ``` 20 | Schemas are generally suffixed as `.proto`. 21 | 22 | ## Task 23 | Create a CLI application that takes two arguments of "name" and "age" (argv 1, 24 | 2) and encodes them using a protocol buffer and save the encoded message to a 25 | file (`user.message`). 26 | 27 | ## Tips 28 | - Get the cli arguments using `process.argv` 29 | - Writing to files is easiest using `fs.createWriteStream` 30 | - Use the `protocol-buffers` library from npm 31 | - Use `require('assert')` to validate the right arguments are passed 32 | 33 | ## Bonus 34 | - assert the right arguments are passed 35 | - log a "usage" cli output if an error is encountered 36 | 37 | ## See Also 38 | - https://github.com/mafintosh/protocol-buffers 39 | - https://developers.google.com/protocol-buffers/docs/overview 40 | - [protobuf language guide](https://developers.google.com/protocol-buffers/docs/proto3) 41 | 42 | --- 43 | [Click here to go to the next exercise](05.html) 44 | -------------------------------------------------------------------------------- /problems/05.md: -------------------------------------------------------------------------------- 1 | # 05 - decode protocol buffers 2 | 3 | Now that we've encoded a message, let's decode it! 4 | 5 | ## Task 6 | Create a CLI application that takes the location for a message, reads it and 7 | decodes it using the schema created in the previous exercise. Log the decoded 8 | message to stdout. 9 | 10 | ## Tips 11 | - if you hadn't already, you might want to save the schema created in the 12 | previous example as a `.proto` file 13 | - catch errors using `try ... catch` 14 | 15 | --- 16 | [Click here to go to the next exercise](06.html) 17 | -------------------------------------------------------------------------------- /problems/06.md: -------------------------------------------------------------------------------- 1 | # 06 - pubsub with channels 2 | 3 | Sometimes messages need to be sent to multiple subscribers, but not all 4 | subscribers are interested in all messages. Filtering messages can be done 5 | using `channels`. 6 | 7 | ## Task 8 | Create one publisher and two subscriber. `sub1` should listen on the channels 9 | `foo` and `bar`. `sub2` should listen on the channels `beep` and `boop`. Every 10 | `1000ms` 4 messages should be sent. Shen a sub receives a message, it should 11 | echo the message prefixed by the name of the sub. 12 | 13 | ## Tips 14 | - channels can be set by doing `sub.chan([])` 15 | - filterable messages can be sent by prefixing the message 16 | 17 | --- 18 | [Click here to go to the next exercise](07.html) 19 | -------------------------------------------------------------------------------- /problems/07.md: -------------------------------------------------------------------------------- 1 | # 07 - sychronized pubsub 2 | 3 | Order in distributed systems is a hard problem. When connecting multiple 4 | systems there are no guarantees that the other system will be available when 5 | you try to connect to it. So in order to do that it must be verified. Sometimes 6 | people choose to use timers / timeouts before connecting, but the more reliable 7 | option is to listen for events. 8 | 9 | The sychronized pubsub combines multiple sockets to achieve this. It only 10 | starts emitting data once it knows that there are listeners to send data to: 11 | 12 | ```txt 13 | ┌───────────┐ 14 | │ Publisher │ 15 | ├─────┬─────┤ 16 | │ REP │ PUB │ 17 | └▲──┬─┴──┬──┘ 18 | 1 │ │ 19 | │ 2 3 20 | ┌┴──▼─┬──▼──┐ 21 | │ REQ │ SUB │ 22 | ├─────┴─────┤ 23 | │Subscriber │ 24 | └───────────┘ 25 | ``` 26 | 27 | ## Exercise 28 | Create a publisher in one file and a subscriber in another. Emit a number in 29 | the fibonacci sequence every 500ms, but only if a subscriber is available. 30 | 31 | ## Tips 32 | - remember that nanomsg sockets are blocking, if messages are not coming 33 | through it might be because of that 34 | 35 | ## Bonus 36 | - implement multiple subscribers 37 | - send a "disconnect" signal from the subscriber when it's closed 38 | - send a heartbeat check from the publisher to validate if the subscriber is 39 | still up 40 | - keep sent values in a cache, and only drop them off if the heartbeat to all 41 | children was ok 42 | 43 | --- 44 | [Click here to go to the next exercise](08.html) 45 | -------------------------------------------------------------------------------- /problems/08.md: -------------------------------------------------------------------------------- 1 | # 08 - PUSHPULL pipeline 2 | 3 | In the second exercise we created a `PUSHPULL` system. In this exercise we're 4 | going to expand it into a mapreduce: 5 | ```txt 6 | ┌──────────┐ 7 | │Ventilator│ 8 | ├──────────┤ 9 | │ Push │ 10 | └──────────┘ 11 | Tasks 12 | ┌──────────────┼──────────────┐ 13 | │ │ │ 14 | ┌─────▼────┐ ┌─────▼────┐ ┌─────▼────┐ 15 | │ PULL │ │ PULL │ │ PULL │ 16 | ├──────────┤ ├──────────┤ ├──────────┤ 17 | │ Worker │ │ Worker │ │ Worker │ 18 | ├──────────┤ ├──────────┤ ├──────────┤ 19 | │ PUSH │ │ PUSH │ │ PUSH │ 20 | └──────────┘ └──────────┘ └──────────┘ 21 | │ │ │ 22 | └──────────────┼───────────────┘ 23 | Results 24 | ┌─────▼────┐ 25 | │ PULL │ 26 | ├──────────┤ 27 | │ Sink │ 28 | └──────────┘ 29 | ``` 30 | 31 | ## Task 32 | Create a `PUSHPULL` system where the ventilator pushes the first 100 digits of 33 | the fibonnacci sequence onto workers, each worker squares the result and then 34 | pushes it into a sink that adds all values and logs the result. 35 | 36 | ## Tips 37 | - remember that nanomsg is blocking 38 | - try and log out the intermediate value 39 | - if values are pushed too fast, consider setting a `setInterval` 40 | 41 | --- 42 | [Click here to go to the next exercise](09.html) 43 | -------------------------------------------------------------------------------- /problems/09.md: -------------------------------------------------------------------------------- 1 | # 09 - PUSH PULL with kill 2 | 3 | Expanding on the previous exercise, we want to kill the workers once they're 4 | done and only start pushing data once all workers have been started up: 5 | ```txt 6 | ┌──────────┐ 7 | │Ventilator│ 8 | ├──────────┤ 9 | │ PUSH │ 10 | └──────────┘ 11 | Tasks 12 | ┌──────────────┬──┴───────────┐ 13 | │ ─ ─ ─ ─ ┼ ─ ─ ┬ ─ ─ ─ ─│─ ─ ─ ─ ─ ─ 14 | │ │ │ │ │ │ 15 | ┌──▼──┬──▼──┐ ┌──▼──┬──▼──┐ ┌──▼──┬──▼──┐ 16 | │PULL │ SUB │ │PULL │ SUB │ │PULL │ SUB │ │ 17 | ├─────┴─────┤ ├─────┴─────┤ ├─────┴─────┤ 18 | │ Worker │ │ Worker │ │ Worker │ │ 19 | ├───────────┤ ├───────────┤ ├───────────┤ 20 | │ PUSH │ │ PUSH │ │ PUSH │ │ 21 | └───────────┘ └───────────┘ └───────────┘ 22 | │ │ │ │ 23 | └──────────────┼──────────────┘ 24 | Results │ 25 | ┌─────▼────┐ 26 | │ PULL │ │ 27 | ├──────────┤ 28 | │ Sink │─ ─ KILL signal ─ ┘ 29 | └──────────┘ 30 | ``` 31 | 32 | ## Task 33 | Create a `PUSHPULL` system that starts up 3 nodes, and only starts pushing data 34 | once the nodes have connected. Push the first 100 digits of the fibonnacci 35 | sequence onto workers, each worker squares the result and then pushes it into a 36 | sink that adds all values and logs the result. Once the sink has received all 37 | values, it should kill all workers. 38 | 39 | ## Bonus 40 | - send a "disconnect" signal from the subscriber when it's closed 41 | - send a heartbeat check from the publisher to validate if the subscriber is 42 | still up 43 | - keep sent values in a cache, and only drop them off if the heartbeat to all 44 | children was ok 45 | 46 | ## Fin 47 | The end, congratulations on running through the workshop! 48 | -------------------------------------------------------------------------------- /solutions/01/index.js: -------------------------------------------------------------------------------- 1 | const nano = require('nanomsg') 2 | 3 | const addr = 'tcp://127.0.0.1:7789' 4 | const req = nano.socket('req') 5 | const rep = nano.socket('rep') 6 | 7 | req.bind(addr) 8 | rep.connect(addr) 9 | 10 | rep.on('data', function (data) { 11 | console.log(String(data)) 12 | rep.send('the real RPC?') 13 | }) 14 | 15 | req.on('data', function (data) { 16 | console.log(String(data)) 17 | rep.close() 18 | req.close() 19 | }) 20 | 21 | req.send('is this') 22 | -------------------------------------------------------------------------------- /solutions/01/npm-debug.log: -------------------------------------------------------------------------------- 1 | 0 info it worked if it ends with ok 2 | 1 verbose cli [ '/Users/yw/.nvm/versions/node/v5.9.1/bin/node', 3 | 1 verbose cli '/Users/yw/.nvm/versions/node/v5.9.1/bin/npm', 4 | 1 verbose cli 'start' ] 5 | 2 info using npm@2.15.1 6 | 3 info using node@v5.9.1 7 | 4 verbose run-script [ 'prestart', 'start', 'poststart' ] 8 | 5 info prestart 01@ 9 | 6 info start 01@ 10 | 7 verbose unsafe-perm in lifecycle true 11 | 8 info 01@ Failed to exec start script 12 | 9 verbose stack Error: 01@ start: `node .` 13 | 9 verbose stack Exit status 1 14 | 9 verbose stack at EventEmitter. (/Users/yw/.nvm/versions/node/v5.9.1/lib/node_modules/npm/lib/utils/lifecycle.js:217:16) 15 | 9 verbose stack at emitTwo (events.js:100:13) 16 | 9 verbose stack at EventEmitter.emit (events.js:185:7) 17 | 9 verbose stack at ChildProcess. (/Users/yw/.nvm/versions/node/v5.9.1/lib/node_modules/npm/lib/utils/spawn.js:24:14) 18 | 9 verbose stack at emitTwo (events.js:100:13) 19 | 9 verbose stack at ChildProcess.emit (events.js:185:7) 20 | 9 verbose stack at maybeClose (internal/child_process.js:850:16) 21 | 9 verbose stack at Process.ChildProcess._handle.onexit (internal/child_process.js:215:5) 22 | 10 verbose pkgid 01@ 23 | 11 verbose cwd /Users/yw/src/yw/distributed-patterns-workshop/solutions/01 24 | 12 error Darwin 14.5.0 25 | 13 error argv "/Users/yw/.nvm/versions/node/v5.9.1/bin/node" "/Users/yw/.nvm/versions/node/v5.9.1/bin/npm" "start" 26 | 14 error node v5.9.1 27 | 15 error npm v2.15.1 28 | 16 error code ELIFECYCLE 29 | 17 error 01@ start: `node .` 30 | 17 error Exit status 1 31 | 18 error Failed at the 01@ start script 'node .'. 32 | 18 error This is most likely a problem with the 01 package, 33 | 18 error not with npm itself. 34 | 18 error Tell the author that this fails on your system: 35 | 18 error node . 36 | 18 error You can get information on how to open an issue for this project with: 37 | 18 error npm bugs 01 38 | 18 error Or if that isn't available, you can get their info via: 39 | 18 error 40 | 18 error npm owner ls 01 41 | 18 error There is likely additional logging output above. 42 | 19 verbose exit [ 1, true ] 43 | -------------------------------------------------------------------------------- /solutions/01/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "01", 3 | "private": "true", 4 | "main": "index.js", 5 | "scripts": { 6 | "start": "node ." 7 | }, 8 | "dependencies": { 9 | "nanomsg": "^3.1.1" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /solutions/02/index.js: -------------------------------------------------------------------------------- 1 | const nano = require('nanomsg') 2 | 3 | const addr = 'tcp://127.0.0.1:7789' 4 | const push = nano.socket('push') 5 | const pull = nano.socket('pull') 6 | 7 | push.bind(addr) 8 | pull.connect(addr) 9 | 10 | var counter = 0 11 | pull.on('data', function (data) { 12 | const num = Number(data) 13 | pull.pause() 14 | 15 | setTimeout(() => { 16 | pull.resume() 17 | console.log(counter += num) 18 | 19 | if (num === 100) { 20 | push.close() 21 | pull.close() 22 | console.log('done!') 23 | } 24 | }, 20) 25 | }) 26 | 27 | for (var i = 1; i <= 100; i++) { 28 | push.send(i) 29 | } 30 | -------------------------------------------------------------------------------- /solutions/02/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "01", 3 | "private": "true", 4 | "main": "index.js", 5 | "scripts": { 6 | "start": "node ." 7 | }, 8 | "dependencies": { 9 | "nanomsg": "^3.1.1" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /solutions/03/index.js: -------------------------------------------------------------------------------- 1 | const nano = require('nanomsg') 2 | 3 | const addr = 'tcp://127.0.0.1:7789' 4 | const pub = nano.socket('pub') 5 | const sub = nano.socket('sub') 6 | 7 | pub.bind(addr) 8 | sub.connect(addr) 9 | 10 | sub.on('data', function (buf) { 11 | console.log(String(buf)) 12 | pub.close() 13 | sub.close() 14 | }) 15 | 16 | pub.send('hello world') 17 | -------------------------------------------------------------------------------- /solutions/03/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "01", 3 | "private": "true", 4 | "main": "index.js", 5 | "scripts": { 6 | "start": "node ." 7 | }, 8 | "dependencies": { 9 | "nanomsg": "^3.1.1" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /solutions/04/index.js: -------------------------------------------------------------------------------- 1 | const pbuf = require('protocol-buffers') 2 | const assert = require('assert') 3 | const fs = require('fs') 4 | 5 | const schema = pbuf(` 6 | message Foo { 7 | string name = 1; 8 | float age = 2; 9 | } 10 | `) 11 | 12 | const argv = process.argv.slice(2) 13 | assert.equal(argv.length, 2, 'usage: encode ') 14 | 15 | const name = argv[0] 16 | const age = argv[1] 17 | 18 | const buf = schema.Foo.encode({ 19 | name: name, 20 | age: age 21 | }) 22 | 23 | const ws = fs.createWriteStream('./user.message') 24 | ws.end(buf) 25 | -------------------------------------------------------------------------------- /solutions/04/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "01", 3 | "private": "true", 4 | "main": "index.js", 5 | "scripts": { 6 | "start": "node ." 7 | }, 8 | "dependencies": { 9 | "nanomsg": "^3.1.1", 10 | "protocol-buffers": "^3.1.6" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /solutions/05/index.js: -------------------------------------------------------------------------------- 1 | const pbuf = require('protocol-buffers') 2 | const assert = require('assert') 3 | const fs = require('fs') 4 | 5 | const schema = pbuf(` 6 | message Foo { 7 | string name = 1; 8 | float age = 2; 9 | } 10 | `) 11 | 12 | const argv = process.argv.slice(2) 13 | assert.equal(argv.length, 1, 'usage: decode ') 14 | 15 | const file = argv[0] 16 | 17 | const buf = fs.readFileSync(file) 18 | const msg = schema.Foo.decode(buf) 19 | console.log(msg) 20 | -------------------------------------------------------------------------------- /solutions/05/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "01", 3 | "private": "true", 4 | "main": "index.js", 5 | "scripts": { 6 | "start": "node ." 7 | }, 8 | "dependencies": { 9 | "nanomsg": "^3.1.1", 10 | "protocol-buffers": "^3.1.6" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /solutions/05/user.message: -------------------------------------------------------------------------------- 1 | 2 | yosh@A -------------------------------------------------------------------------------- /solutions/06/index.js: -------------------------------------------------------------------------------- 1 | var nano = require('nanomsg') 2 | 3 | var addr = 'tcp://127.0.0.1:8976' 4 | 5 | var pub = nano.socket('pub') 6 | pub.bind(addr) 7 | 8 | var sub1 = nano.socket('sub') 9 | sub1.connect(addr) 10 | sub1.chan(['foo', 'bar']) 11 | sub1.on('data', (buf) => console.log(String(buf))) 12 | 13 | var sub2 = nano.socket('sub') 14 | sub2.connect(addr) 15 | sub2.chan(['beep', 'boop']) 16 | sub2.on('data', (buf) => console.log(String(buf))) 17 | 18 | setInterval(function () { 19 | pub.send('foo world') 20 | pub.send('bar world') 21 | pub.send('beep world') 22 | pub.send('boop world') 23 | }, 1000) 24 | -------------------------------------------------------------------------------- /solutions/06/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "01", 3 | "private": "true", 4 | "main": "index.js", 5 | "scripts": { 6 | "start": "node ." 7 | }, 8 | "dependencies": { 9 | "nanomsg": "^3.1.1", 10 | "protocol-buffers": "^3.1.6" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /solutions/07/index.js: -------------------------------------------------------------------------------- 1 | var nano = require('nanomsg') 2 | 3 | var addr = 'tcp://127.0.0.1:8976' 4 | 5 | var pub = nano.socket('pub') 6 | pub.bind(addr) 7 | 8 | var sub1 = nano.socket('sub') 9 | sub1.connect(addr) 10 | sub1.chan(['foo', 'bar']) 11 | sub1.on('data', (buf) => console.log(String(buf))) 12 | 13 | var sub2 = nano.socket('sub') 14 | sub2.connect(addr) 15 | sub2.chan(['beep', 'boop']) 16 | sub2.on('data', (buf) => console.log(String(buf))) 17 | 18 | setTimeout(function () { 19 | pub.send('foo world') 20 | pub.send('bar world') 21 | pub.send('beep world') 22 | pub.send('boop world') 23 | }, 1000) 24 | -------------------------------------------------------------------------------- /solutions/07/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "01", 3 | "private": "true", 4 | "main": "index.js", 5 | "scripts": { 6 | "start": "node ." 7 | }, 8 | "dependencies": { 9 | "nanomsg": "^3.1.1", 10 | "protocol-buffers": "^3.1.6" 11 | } 12 | } 13 | --------------------------------------------------------------------------------