├── .dir-locals.el ├── .envrc ├── .github └── workflows │ ├── build-deploy.yml │ └── test.yml ├── .gitignore ├── .gitmodules ├── LICENSE ├── Makefile ├── README.org ├── config.toml ├── flake.lock ├── flake.nix ├── org ├── _20201007_221440screenshot.png ├── _20201007_222154screenshot.png ├── _20201007_224144screenshot.png ├── _20201007_224746screenshot.png ├── _20201008_232738screenshot.png ├── _20201008_235124screenshot.png ├── _20201008_235425screenshot.png ├── _20201008_235848screenshot.png ├── _20201009_000309screenshot.png ├── _20201010_164050screenshot.png ├── _20201010_165840screenshot.png ├── _20201010_170205screenshot.png ├── _20201011_180936screenshot.png ├── _20201012_203109screenshot.png ├── _20201013_221335screenshot.png ├── _20201013_222125screenshot.png ├── _20201013_222507screenshot.png ├── _20201014_220602screenshot.png ├── _20201014_224503screenshot.png ├── _index.org ├── actor-model.org ├── akka.org ├── akka_actors.org ├── akka_cluster.org ├── akka_dispatcher.org ├── akka_router.org ├── akka_sharding.org ├── akka_streams.org ├── akka_testing.org ├── amdah_s_law.org ├── anti-corruption-layer.png ├── base_transaction.org ├── bug-management.org ├── cloud_testing.org ├── command_query_responsibility_segregation.org ├── command_sourcing.org ├── consistency_and_availability.org ├── context-map.png ├── digital_garden.org ├── domain_driven_design.org ├── emacs.org ├── event_sourcing_es_es.org ├── from-data-to-knowledge.png ├── git.org ├── gossip_protocol.org ├── gpg.org ├── gunther_s_universal_scalability_law.org ├── hands_on_scala_programming.org ├── heartbeat.org ├── interview_questions.org ├── jackson.org ├── json.org ├── kotlin.org ├── linux.org ├── message_driven_architecture.org ├── microservices.org ├── monolith.org ├── nix.org ├── postgresql.org ├── reactive_streams.org ├── reactive_systems.org ├── referential_transparency.org ├── scala_experiments.org ├── sharding_or_partitioning.org ├── sli_slo.org ├── stateless.org ├── states-of-data.org ├── thinking_tools.org ├── using_psql_as_job_queue.org ├── way-of-work.org └── web-stack-enties.org ├── shell.nix ├── static ├── .gitignore └── favicon.png └── tools ├── init.el └── versions └── default.el /.dir-locals.el: -------------------------------------------------------------------------------- 1 | ((nil . ((eval . (setq-local org-roam-directory (concat (file-name-as-directory (projectile-project-root)) "org"))) 2 | (eval . (setq-local org-hugo-base-dir (projectile-project-root))) 3 | (eval . (setq-local org-attach-id-dir (concat (file-name-as-directory (projectile-project-root)) "org"))) 4 | (org-hugo-section . "notes") 5 | (org-roam-file-exclude-regexp . "_index.org"))) 6 | ("org" . ((org-mode . ((eval . (org-hugo-auto-export-mode))))))) 7 | -------------------------------------------------------------------------------- /.envrc: -------------------------------------------------------------------------------- 1 | use flake 2 | -------------------------------------------------------------------------------- /.github/workflows/build-deploy.yml: -------------------------------------------------------------------------------- 1 | name: Build & Deploy 2 | on: 3 | push: 4 | branches: 5 | - master 6 | 7 | env: 8 | HUGO_VERSION: 0.88.1 9 | EMACS_VERSION: 28.1 10 | 11 | jobs: 12 | gh-pages-deploy: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - name: Checkout repository 16 | uses: actions/checkout@v2 17 | with: 18 | submodules: true 19 | 20 | - name: Install Hugo 21 | uses: peaceiris/actions-hugo@v2 22 | with: 23 | hugo-version: ${{ env.HUGO_VERSION }} 24 | extended: true 25 | 26 | - name: Install emacs 27 | uses: purcell/setup-emacs@master 28 | with: 29 | version: ${{ env.EMACS_VERSION }} 30 | 31 | # Files under static/ox-hugo are automatic and should be published but not version controlled (they are a copy). 32 | - name: Force ox-hugo publish 33 | run: rm static/.gitignore 34 | 35 | - name: Build 36 | run: make deploy 37 | 38 | - name: Deploy 39 | uses: peaceiris/actions-gh-pages@v3 40 | with: 41 | personal_token: ${{ secrets.DEPLOY_TOKEN }} 42 | publish_dir: ./public 43 | publish_branch: gh-pages 44 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | on: 3 | pull_request: 4 | branches: 5 | - master 6 | 7 | env: 8 | HUGO_VERSION: 0.88.1 9 | EMACS_VERSION: 28.1 10 | 11 | jobs: 12 | build_page: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - name: Checkout repository 16 | uses: actions/checkout@v2 17 | with: 18 | submodules: true 19 | 20 | - name: Install Hugo 21 | uses: peaceiris/actions-hugo@v2 22 | with: 23 | hugo-version: ${{ env.HUGO_VERSION }} 24 | extended: true 25 | 26 | - name: Install emacs 27 | uses: purcell/setup-emacs@master 28 | with: 29 | version: ${{ env.EMACS_VERSION }} 30 | 31 | - name: Build 32 | run: make build 33 | 34 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Hugo Go 2 | public/ 3 | resources/_gen/ 4 | content/ 5 | *.db 6 | .hugo_build.lock 7 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "themes/explorer"] 2 | path = themes/explorer 3 | url = git@github.com:bphenriques/explorer-hugo-theme.git 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Bruno Henriques 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | BASE_DIR := $(shell pwd) 2 | EMACS_BUILD_SRC := $(shell pwd)/tools 3 | EMACS_BUILD_DIR := /tmp/knowledge-base-home-build 4 | BASE_URL := https://bphenriques.github.io/knowledge-base 5 | 6 | all: clean build-content serve 7 | 8 | .PHONY: clean 9 | clean: 10 | rm -rf content public 11 | rm -rf static/ox-hugo 12 | 13 | .PHONY: serve 14 | serve: 15 | hugo server --minify --disableFastRender --baseURL localhost:1313 16 | 17 | .PHONY: build-content 18 | build-content: 19 | mkdir -p $(EMACS_BUILD_DIR) 20 | cp -r $(EMACS_BUILD_SRC)/* $(EMACS_BUILD_DIR) 21 | # Build temporary minimal EMACS installation separate from the one in the machine. 22 | env HOME=$(EMACS_BUILD_DIR) KNOWLEDGE_BASE_DIR=$(BASE_DIR) emacs -Q --batch --load $(EMACS_BUILD_DIR)/init.el --execute "(build/export-all)" --kill 23 | # Fixes bad URLs when the baseURL is not the root URL (this case). 24 | # There is the option to change the template but IMO it is intrusive: https://github.com/kaushalmodi/ox-hugo/issues/460 25 | find $(BASE_DIR)/content -type f -exec sed -i '' -e 's|figure src="/ox-hugo/|figure src="ox-hugo/|g' {} \; 26 | 27 | .PHONY: build-site 28 | build-site: 29 | hugo --minify --cleanDestinationDir --baseURL $(BASE_URL) 30 | 31 | .PHONY: build 32 | build: build-content build-site 33 | 34 | .PHONY: deploy 35 | deploy: build-content build-site 36 | 37 | update-sub-modules: 38 | git submodule update --init --recursive 39 | git submodule foreach git pull origin main 40 | -------------------------------------------------------------------------------- /README.org: -------------------------------------------------------------------------------- 1 | * Archived: Migrating out of emacs. Feel free to re-use the [[https://github.com/bphenriques/explorer-hugo-theme][Explorer theme]]. 2 | 3 | [[https://img.shields.io/badge/hugo-0.85.0-blue.svg]] 4 | [[https://github.com/bphenriques/knowledge-base/workflows/Build%20&%20Deploy/badge.svg?event=push]] 5 | [[https://img.shields.io/badge/License-MIT-blue.svg]] 6 | 7 | * Digital Garden 8 | 9 | My digital garden is available at [[https://bphenriques.github.io/knowledge-base][bphenriques.github.io/knowledge-base]] and uses the [[https://github.com/bphenriques/explorer-hugo-theme][Explorer Book Theme]]. 10 | 11 | * How it works 12 | 13 | The content is written using ~org-mode~ in the ~org~ directory where each ~*.org~ file is a new note. 14 | 15 | Then, using [[https://ox-hugo.scripter.co/][ox-hugo]], each ~*.org~ file is converted to ~*.md~ which will feed ~gohugo~ to generate my digital garden. 16 | 17 | * Run locally 18 | 19 | 1. Export existing ~*.org~ content and run the website locally: 20 | #+BEGIN_SRC bash 21 | $ make clean build-content serve 22 | #+END_SRC 23 | 2. Open [[http://localhost:1313][localhost:1313]]. 24 | 25 | * Contributing 26 | 27 | Feel free to send feedback! I look forward to include your insights on my notes :) 28 | 29 | * Reference 30 | 31 | - [[https://gohugo.io/][Hugo Static Page Generator]] 32 | - [[https://github.com/bphenriques/explorer-hugo-theme][Explorer Book Theme]] 33 | - [[https://github.com/alex-shpak/hugo-book][Hugo Book Theme]] 34 | - [[https://ox-hugo.scripter.co/][Ox-Hugo]] 35 | 36 | * Inspired by 37 | 38 | There are several personal Wikis made by other people that inspired me to make one for me. Follows some notable examples: 39 | - [[https://github.com/jethrokuan/braindump]] 40 | - [[https://wiki.nikitavoloboev.xyz]] 41 | - https://github.com/maggiedelano/digital-garden/ 42 | - https://beepb00p.xyz/exobrain/ 43 | - http://okmij.org/ftp/ 44 | - https://notes.andymatuschak.org/About_these_notes 45 | - http://okmij.org/ftp/ 46 | - https://notes.aravindballa.com/ 47 | 48 | * Theme 49 | 50 | The theme is [[https://github.com/bphenriques/explorer-hugo-theme][Explorer]], a theme made by me heavily based on [[https://github.com/alex-shpak/hugo-book][Hugo Book Theme]] with a twist of [[https://github.com/mrmartineau/gatsby-theme-code-notes][Gatsby Theme Code Nodes]] and [[https://www.maggiedelano.com/garden/][Maggie Delano's digital garden]]. 51 | 52 | I am no front-end expert so I look forward for your help improving the theme if you like it! 53 | -------------------------------------------------------------------------------- /config.toml: -------------------------------------------------------------------------------- 1 | theme = "explorer" 2 | title = "Bruno Henriques's Digital Garden 🌱" 3 | 4 | [markup.goldmark.renderer] 5 | unsafe = true 6 | 7 | [params] 8 | BrandTitle = 'Bruno Henriques' 9 | Theme = 'dark' 10 | -------------------------------------------------------------------------------- /flake.lock: -------------------------------------------------------------------------------- 1 | { 2 | "nodes": { 3 | "flake-utils": { 4 | "locked": { 5 | "lastModified": 1653893745, 6 | "narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=", 7 | "owner": "numtide", 8 | "repo": "flake-utils", 9 | "rev": "1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1", 10 | "type": "github" 11 | }, 12 | "original": { 13 | "owner": "numtide", 14 | "repo": "flake-utils", 15 | "type": "github" 16 | } 17 | }, 18 | "nixpkgs": { 19 | "locked": { 20 | "lastModified": 1653918726, 21 | "narHash": "sha256-C+BpRKWjuwR4a9R0w5V5AFaQiyDtRPSRec7VJCX1NI4=", 22 | "owner": "nixos", 23 | "repo": "nixpkgs", 24 | "rev": "39945562539b572eb43915992ef82f46a7176364", 25 | "type": "github" 26 | }, 27 | "original": { 28 | "owner": "nixos", 29 | "ref": "nixos-21.11", 30 | "repo": "nixpkgs", 31 | "type": "github" 32 | } 33 | }, 34 | "root": { 35 | "inputs": { 36 | "flake-utils": "flake-utils", 37 | "nixpkgs": "nixpkgs" 38 | } 39 | } 40 | }, 41 | "root": "root", 42 | "version": 7 43 | } 44 | -------------------------------------------------------------------------------- /flake.nix: -------------------------------------------------------------------------------- 1 | { 2 | description = "bphenriques's knowledge base"; 3 | inputs = { 4 | nixpkgs.url = "github:nixos/nixpkgs/nixos-21.11"; 5 | flake-utils.url = "github:numtide/flake-utils"; 6 | }; 7 | 8 | outputs = { self, nixpkgs, flake-utils }: 9 | flake-utils.lib.eachDefaultSystem 10 | (system: 11 | let pkgs = nixpkgs.legacyPackages.${system}; in 12 | { 13 | devShell = import ./shell.nix { inherit pkgs; }; 14 | } 15 | ); 16 | } 17 | -------------------------------------------------------------------------------- /org/_20201007_221440screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bphenriques/knowledge-base/0490478c2ce82c257883eed1b6e6468fc66184f5/org/_20201007_221440screenshot.png -------------------------------------------------------------------------------- /org/_20201007_222154screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bphenriques/knowledge-base/0490478c2ce82c257883eed1b6e6468fc66184f5/org/_20201007_222154screenshot.png -------------------------------------------------------------------------------- /org/_20201007_224144screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bphenriques/knowledge-base/0490478c2ce82c257883eed1b6e6468fc66184f5/org/_20201007_224144screenshot.png -------------------------------------------------------------------------------- /org/_20201007_224746screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bphenriques/knowledge-base/0490478c2ce82c257883eed1b6e6468fc66184f5/org/_20201007_224746screenshot.png -------------------------------------------------------------------------------- /org/_20201008_232738screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bphenriques/knowledge-base/0490478c2ce82c257883eed1b6e6468fc66184f5/org/_20201008_232738screenshot.png -------------------------------------------------------------------------------- /org/_20201008_235124screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bphenriques/knowledge-base/0490478c2ce82c257883eed1b6e6468fc66184f5/org/_20201008_235124screenshot.png -------------------------------------------------------------------------------- /org/_20201008_235425screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bphenriques/knowledge-base/0490478c2ce82c257883eed1b6e6468fc66184f5/org/_20201008_235425screenshot.png -------------------------------------------------------------------------------- /org/_20201008_235848screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bphenriques/knowledge-base/0490478c2ce82c257883eed1b6e6468fc66184f5/org/_20201008_235848screenshot.png -------------------------------------------------------------------------------- /org/_20201009_000309screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bphenriques/knowledge-base/0490478c2ce82c257883eed1b6e6468fc66184f5/org/_20201009_000309screenshot.png -------------------------------------------------------------------------------- /org/_20201010_164050screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bphenriques/knowledge-base/0490478c2ce82c257883eed1b6e6468fc66184f5/org/_20201010_164050screenshot.png -------------------------------------------------------------------------------- /org/_20201010_165840screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bphenriques/knowledge-base/0490478c2ce82c257883eed1b6e6468fc66184f5/org/_20201010_165840screenshot.png -------------------------------------------------------------------------------- /org/_20201010_170205screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bphenriques/knowledge-base/0490478c2ce82c257883eed1b6e6468fc66184f5/org/_20201010_170205screenshot.png -------------------------------------------------------------------------------- /org/_20201011_180936screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bphenriques/knowledge-base/0490478c2ce82c257883eed1b6e6468fc66184f5/org/_20201011_180936screenshot.png -------------------------------------------------------------------------------- /org/_20201012_203109screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bphenriques/knowledge-base/0490478c2ce82c257883eed1b6e6468fc66184f5/org/_20201012_203109screenshot.png -------------------------------------------------------------------------------- /org/_20201013_221335screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bphenriques/knowledge-base/0490478c2ce82c257883eed1b6e6468fc66184f5/org/_20201013_221335screenshot.png -------------------------------------------------------------------------------- /org/_20201013_222125screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bphenriques/knowledge-base/0490478c2ce82c257883eed1b6e6468fc66184f5/org/_20201013_222125screenshot.png -------------------------------------------------------------------------------- /org/_20201013_222507screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bphenriques/knowledge-base/0490478c2ce82c257883eed1b6e6468fc66184f5/org/_20201013_222507screenshot.png -------------------------------------------------------------------------------- /org/_20201014_220602screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bphenriques/knowledge-base/0490478c2ce82c257883eed1b6e6468fc66184f5/org/_20201014_220602screenshot.png -------------------------------------------------------------------------------- /org/_20201014_224503screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bphenriques/knowledge-base/0490478c2ce82c257883eed1b6e6468fc66184f5/org/_20201014_224503screenshot.png -------------------------------------------------------------------------------- /org/_index.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b678b0 3 | :ROAM_EXCLUDE: t 4 | :END: 5 | #+TITLE: Home 6 | #+HUGO_SECTION: 7 | #+hugo_custom_front_matter: :ContentMenu false 8 | 9 | * Hi! 🌱 10 | 11 | Welcome to my [[id:2fa03d4f-948e-4a6e-a38b-178456b578c4][Digital Garden]]! A frictionless writing space where I collect, arrange, and connect my thoughts as I explore 12 | new or existing topics. 13 | 14 | Learning can be hard. Having incremental notes and linking them is a great way to turn insights into knowledge, and with 15 | experience, make good use of it! 16 | 17 | #+ATTR_HTML: :alt Connecting insights dots leads to building knowledge :class center width-85 18 | [[file:from-data-to-knowledge.png]] 19 | 20 | * Navigation 21 | 22 | You can explore all the notes in this digital garden using the following graph: 23 | 24 | #+begin_export html 25 | {{< graph >}} 26 | #+end_export 27 | 28 | * Contributing 29 | 30 | It is an extension of my thinking process, therefore, some of these notes may be raw or incomplete as I explore them. Let me know if you find mistakes or just want to say hi! 31 | 32 | This wiki is open-source and available on [[https://github.com/bphenriques/knowledge-base][Github]]. 33 | -------------------------------------------------------------------------------- /org/actor-model.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578b0 3 | :END: 4 | #+TITLE: Actor Model 5 | #+filetags: concurrency 6 | 7 | Concept from 1973 by Carl Hewitt: 8 | #+begin_quote 9 | Actor is the fundamental unit of computation embodying processing, storage, and communication. 10 | #+end_quote 11 | 12 | 13 | * Fundamentals 14 | 15 | - All computation occurs inside of the actor. 16 | - Each actor has an address. 17 | - Actors may create new actors, send messages to them and changing their own behavior to handle new messages (e.g., change the state). 18 | 19 | An actor does not exist isolated, it works in tandem with others and are arranged in hierarquy: 20 | - Actors can split up and delegate tasks to child actors. 21 | - Child actors are supervised and delegate their failures back to their parent. 22 | 23 | * Anatomy 24 | 25 | Each actor has: 26 | - An address. 27 | - An mailbox. 28 | - An dispatcher. 29 | 30 | Messages are sent to an Actor Address which are enqued in the mailbox and only then are dispatched for processing. Note that, only one message is dispatched at a time leading to the ilusion of single thread (there may be multiple actor within the same system, each one handling messages in different threads). 31 | 32 | It is important that that actors communicate exclusively through messages and do not shared state to ensure /Strong Consistency/! Moreover messages must be immutable. 33 | 34 | [[file:_20201011_180936screenshot.png]] 35 | 36 | * Reactive Systems 37 | 38 | In the context of Reactive Systems, Actor model is a reactive tool, a paradigm that: 39 | - Actors only communicate through asyncronous messages. 40 | - Message driven - All communication between actors is done with async non-blocking messages. 41 | - Abstractions provide elasticity and resiliency. 42 | 43 | Akka uses the actor model - https://doc.akka.io/docs/akka/current/typed/guide/actors-intro.html?language=scala 44 | 45 | The message driven system provides location transparency, i.e., the technique remainins the same regardless of where the actors are. This allows better resiliency and elastic (hmm.. questions on this bit). This is different from "Transparent remoting" as this hides potential networking issues while making it seem like local calls. Location transparency makes the opposite which is makes local calls seem like remote calls, therefore whoever uses is aware that there are potential failures. 46 | 47 | [[id:2fa03d4f-948e-4a6e-a38b-178456b578b2][Akka Actors]] is an implementation of the Actor Model. 48 | -------------------------------------------------------------------------------- /org/akka.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578b1 3 | :END: 4 | #+TITLE: Akka 5 | #+filetags: akka 6 | 7 | * Akka 8 | 9 | Toolkit and runtime for building highly concurrent, distributed and fault tolerant message-driven application in the JVM. It can be used to build [[id:2fa03d4f-948e-4a6e-a38b-178456b578e1][Reactive Systems]]. 10 | 11 | Proposes unified programming model for: 12 | - Simpler concurrency: single threaded illusion as each actor processes a message at a time (no need to locks or synchronization strategies). 13 | - Simpler distribution: is distributed by default (see more on [[id:2fa03d4f-948e-4a6e-a38b-178456b578b3][Akka Cluster]]). 14 | - Simpler fault tolerance: Decouples communication from failure handling. 15 | -------------------------------------------------------------------------------- /org/akka_actors.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578b2 3 | :END: 4 | #+TITLE: Akka Actors 5 | #+filetags: akka 6 | 7 | * Handling Messages Asyncronously 8 | 9 | Blocking threads inside the actors creates contention therefore the handling of the messages must happen in a async fashion, including any DB writes and DB reads when starting the actor. For example, using interfaces such as ~Future[T]~. However this may lead to concurrency within the actor itself removing the ilusion of a single thread. This means that messages must be /stash/ until other operations complete. 10 | 11 | In essence the actor has 3 states: 12 | - Loading - Load the state from the DB. 13 | - Running - Regular behavior. 14 | - Waiting. 15 | 16 | Question: If DB fails, then it is recommended to throw the exception leading to a restart of the Actor that in turn will re-read the state from the DB. So: 17 | - It is explained that the stash is not lost, how? 18 | - What happens if there is a persistent issue in the DB? Will there be a loop? 19 | 20 | * Akka Actor Lifecycle 21 | 22 | Can be stopped by himself or by others. 23 | 24 | Lifecycle (without faults): 25 | #+BEGIN_SRC 26 | hook:preStart() -> started -[stop]-> stopped -> hook:postStop() -> terminated 27 | #+END_SRC 28 | 29 | The actor is created assyncronously and available right away through the ~ActorRef~. 30 | 31 | Stopping an actor will: 32 | - Finishes the processing the current message. 33 | - Suspends message processing. 34 | - *Stop its children* - See [[id:2fa03d4f-948e-4a6e-a38b-178456b578b0][Actor Model]]. 35 | - Waits for their termination confirmations and then stops himself. 36 | 37 | How to stop: ~PoisonPill~ (~context.stop(self())~) and ~actorRef ! Kill~ messages (throw ~ActorKilledException~). They are *not* appropriate to perform a cleanup before shutting down as the Actor does not "see" those messages (it is handled internally). It is best to use a dedicated message such as ~StopMeGraciouslyMessage~. 38 | 39 | ** Monitor 40 | 41 | /Dead Watch/ allows monitoring another actor's termination (regular ~Terminated~ message in the ~receive~ block). 42 | 43 | * Failure Handling 44 | 45 | Akka deals with failures at the level of the individual actor (bulkheading has it only affects that actor). 46 | 47 | Does not throw the message back to the sender (b/c the sender does not know how to handle it). Instead the error is sent to a responsibile entity (e.g., "Manager") that determines the required steps to recover. 48 | 49 | When an actor fails, Akka provides two configurable strategies: 50 | - /OneForOneStrategy/: Only the faulty child is affected. 51 | - /AllForOneStrategy/: All children are affected by one faulty child. 52 | 53 | Both are configured with a ~type Decider = PartialFunction[Throwable, Directive]~. If not defined a directive, then the parent is consired faulty. Where ~Directive~: 54 | - Resume: Resume message processing. Use if the state remains valid. 55 | - Restart: Start a new actor in its place and resume, however all childs are stopped (by default unless ~preRestart~ hook is changed). It supports max number of retries and within a time limit. 56 | - Stop. 57 | - Escalate: Delegate the decision to the supervisor's parent. 58 | 59 | By default it is /OneForOneStrategy/ with some directives that are too specific to group here and we can check the documentation. In short, by default, the actor will be restarted. In any case, message processing is suspended. 60 | - All descendants of the actor are suspended. 61 | - The actor's parent handles the failure. 62 | 63 | Proper tuning leads to a self-healing system. Some exceptions are worth stopping the actor while others are worth recovering. 64 | 65 | ** Full Lifecycle 66 | 67 | [[file:_20201012_203109screenshot.png]] 68 | 69 | * Ask vs Tell 70 | 71 | Ask: ~actorRef ? Message~ 72 | Tell: ~actorRef ! Message~ 73 | 74 | Use Ask when: 75 | - Bridging non-actor code to actor-code (e.g., bridging with HTTP controllers ?). 76 | - We are expecting a response within a timeout. In this case we use ~actorRef ? Message pipeTo self~ which in turn will will handle the response, e.g., ~val receive: Receive = { case MessageResponse => stuff }~. 77 | 78 | Use tell when: 79 | - We do not care about the response. 80 | 81 | Neverthless, when using the ask operator, always ue pipeTo within the actor system to avoid breaking the single thread illusion. 82 | 83 | * Testing 84 | 85 | See [[id:2fa03d4f-948e-4a6e-a38b-178456b578b8][Akka Testing]]. 86 | -------------------------------------------------------------------------------- /org/akka_cluster.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578b3 3 | :END: 4 | #+TITLE: Akka Cluster 5 | #+filetags: akka 6 | 7 | * Akka Cluster 8 | 9 | *Scenario:* Make actors communicate across the network. 10 | 11 | Allows actors to communicate across the network, greatly simplifying the process. Each node represents an actor system and they all share the same name. 12 | 13 | * Akka Cluster Aware Routers 14 | 15 | *Scenario*: High workload. 16 | 17 | Scalling vertically has limits. Introducing Akka Cluster Aware Routers, that allows scalling the system horizontally. I.e., large tasks are broken on smaller tasks that are routed to an especific node of our cluster. 18 | 19 | * Akka Cluster Sharding 20 | 21 | *Scenario*: Database becomes the bottleneck. 22 | 23 | Many applications leverage the database for strong-consistency. However it may become the source of contention (see [[id:2fa03d4f-948e-4a6e-a38b-178456b578b9][Amdah's Law]]) as the load increases. We may migrate to a cache service (e.g., Memcached or Redis) but it will eventually become the bottleneck. 24 | 25 | In order to solve this, Akka Cluster Sharding allows distributing Actors across the cluster responsible for managing the state of a especific database entitiy (given a hashing function). With the aid of a single-thread illusion, we can cache the entities in-memory without the risk of desyncronizing with the database, leading to strong consistency. This is great as most applications are read-heavy as opposed to write-heavy. 26 | 27 | 28 | * Akka Distributed Data 29 | 30 | *Scenario*: Critical information that is required continously and we need maintain it. Especially small data sets with infrequent updates that require high availabiltiy. 31 | 32 | Akka Distributed Data is a local, replicated and in-memory data storage. The data is asyncronously replicated to other nodes. The consistency model varies and is [[https://doc.akka.io/docs/akka/current/typed/distributed-data.html#read-consistency][configurable]]. Through this, we can perform updates from any node without coordination and any concurrent updates will be automatically resolved by a monotic merge function explicitly provided. 33 | 34 | For this end we use a especific data-structure called Conflict Free Replicated Data Types (CRDTs). 35 | 36 | For more, please check [[https://doc.akka.io/docs/akka/current/typed/distributed-data.html][here]]. 37 | 38 | 39 | ** Conflict Free Replicated Data Types (CRDTs) 40 | 41 | CRDTs are stored in-memory and can be copied to disk to speed up recovery if a replica fails. 42 | 43 | - A marker that shows something was deleted. 44 | - Can result in data types that only get larger and never smaller. 45 | - Aka CRDT Garbage 46 | 47 | Limitations CRDT: Do not work with every data type that require a merge function. Some data types are too complex to merge and require the use of /tombstone/. 48 | 49 | ** Limitations 50 | 51 | - It may not be possible depending on the data model due to the merge function. 52 | - Eventual Consistency. Strong consistency is possible at the expense of availability. 53 | - The number of top-level entries should me limited (< 1 million) given that it must be transferred to to the nodes. 54 | - The entity musn't be large given that its full-state may replicated to other nodes. 55 | 56 | For more, please check [[https://doc.akka.io/docs/akka/current/typed/distributed-data.html#limitations][here]]. 57 | 58 | ** TODO Concrete use-cases for Akka Distributed Data 59 | 60 | The lack of Akka Distributed Data may lead to frequent network requests to fetch as especific entity. However, its usage also requires querying several nodes to look for a quorum. Both options have drawbacks, I am curious on knowing the decision thought behind it. 61 | 62 | * Akka Address 63 | 64 | 65 | May be local or remote in the form: 66 | ~akka://@:/~ 67 | 68 | Several protocols are available and depend on the use-case: 69 | - ~aeron-udp~: High throughput and low latency. 70 | - ~tcp~: Good thorughout and latency but lower. 71 | - ~tls-tcp~: When encryption is required. 72 | 73 | * Joining a Cluster 74 | 75 | Requires "Seed Nodes", i.e., contact nodes. Any node is eligible. Best practice is to use "Akka Cluster Bootstrap" to avoid setting static seed-nodes in each configuration file. 76 | 77 | Must be enabled! And it does not bring any advantage until we set the application to leverage this: 78 | 79 | #+BEGIN_SRC scala 80 | val loyaltyActorSupervisor = ClusterSharding(system).start( 81 | "shared-region-name", 82 | MyActorActor.props(someProp), 83 | ClusterShardingSettings(system), 84 | MyActorSupervisor.idExtractor, 85 | MyActorSupervisor.shardIdExtractor 86 | ) 87 | #+END_SRC 88 | 89 | * Akka Cluster Management 90 | 91 | Set of tools served through a HTTP Api to manage the cluster. Must start after the actor system. 92 | 93 | Must be enabled! 94 | 95 | ** Akka Discovery 96 | 97 | Service to locate and discover services. 98 | 99 | ** Akka Cluster Bootstrap 100 | 101 | Automated seed node discovery using Akka Discovery. 102 | 103 | ** Health Check Endpoints 104 | 105 | Useful when integrating with orchestrating platforms (e.g., K8S). 106 | 107 | * Communication 108 | 109 | It is done by using [[id:2fa03d4f-948e-4a6e-a38b-178456b578c9][Gossip Protocol]]. 110 | 111 | * Network Partitions 112 | 113 | This issue cannot be recovered by simply rebooting the affected node. In order to fix this: 114 | 1. Decide which partitions needs to be cleaned up - How? 115 | 2. Shutdown the members 116 | 3. Inform the cluster that those members are down - ~PUT -F operation=down /cluster/members/~. 117 | 4. Create new members to replace the old. 118 | 119 | Step 2. is important otherwise it continues to operate unware that it has been removed from the cluster which can lead to multiple copies of the same shard. 120 | 121 | * Split Brain 122 | 123 | 124 | Occurs when single cluster splits into two or more distinctive clusters. It normally does not occur unless poor management (not stopping processes that are /Down/) or configuration (there are strategies to solve this automatically). Can be caused by improper /Downing/ a member leading to the node creating another cluster as the process was not terminated. 125 | 126 | It may also occur with a network partition. If this extend, the /Unreachable Nodes/ will be marked as downed but will not be terminated. 127 | 128 | Simpler solutions may be solved automatically through orchestration platforms that automatically stop the process. More complicated split brains may be solved using /Lightbend Split Brain Resolver/. 129 | 130 | ** When using sharding or singleton for data consistency 131 | 132 | Each cluster can have a copy of the actor leading to a inconsistency and data corruption specially if both shards have access to the database. 133 | 134 | * Lighbend Split Brain Resolver 135 | 136 | Set of customizable strategies for terminating members in order to avoid Split Brain scenarios. Terminating members allow orchestration platforms to take over and heal the problem. 137 | 138 | ** Static Quorum 139 | 140 | 141 | Fixed sized quorom of node. All nodes will evaluate their situation and /Down/ unreachable. If quorum is set then a smaller cluster will prevail, otherwise the nodes will shutdown themselves. The quorum value must at least ~n/2 + 1~. 142 | 143 | ** Keep Majority 144 | 145 | Similar to previous but dynamically tracks the size of the cluster. 146 | 147 | ** Keep Oldest 148 | 149 | 150 | Monitors the oldest node in the cluster. Members that are not communicating with that node will be marked as down and the nodes will terminate themselves. If the oldest node has crashed so will the cluster but is configurable in a way, that in that case only the oldest will be /Downed/. 151 | 152 | ** Keep Referee 153 | 154 | Similar to the other one but designate a specific node as /referee/ (based on its address). As far as I can see, it is not configurable to avoid crashing the cluster if the /referee/ is down. 155 | 156 | ** Down Allows 157 | 158 | All nodes terminate themselves relying on good orchestration tools to reduce downtime - Me not like this one. 159 | 160 | ** Lease Majority 161 | 162 | Reserved for Kubernetes deployments. 163 | 164 | 165 | It uses a distributed /lock/ (lock) to make it's decision. Each partition will attempt to obtain it the loser terminates and the winnner remains. 166 | 167 | There is a bit of nice hack (IMO but can't understand exactly how this is achieved) which is that the side that is theoretically smaller will delay the attempt to obtain the lock so that the majority wins. 168 | 169 | ** Some Edge Cases 170 | 171 | 172 | - Indirect connected Edges (for some reason is connected to only one member). 173 | - Unstable nodes (keeps on disconnecting from some nodes). 174 | 175 | These edge-caes are automatically handled. 176 | 177 | * Orphaned Node 178 | 179 | Is down but not terminated. 180 | 181 | * TODO Cluster Singleton 182 | -------------------------------------------------------------------------------- /org/akka_dispatcher.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578b4 3 | :END: 4 | #+TITLE: Akka Dispatcher 5 | #+filetags: akka dispatcher 6 | 7 | The engine of Akka as it decides when a actor should process messsages and when it must yield the thread for others. It means that dispatchers are in control of the thread time and of the threads themselves. 8 | 9 | - ~Dispatcher~ (default): Event-driven dispatcher, sharing threads from thread pool. 10 | - ~PinnedDispatcher~: Dedicated thread per actor. 11 | - ~CallingThreadDispatcher~: Just for testing. 12 | 13 | The best tips are: 14 | - Adjust ~throughput~ to deliver more messages to the actors before yielding. 15 | - Adjust dispatcher settings according to the use-case. 16 | -------------------------------------------------------------------------------- /org/akka_router.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578b5 3 | :END: 4 | #+TITLE: Akka Router 5 | #+filetags: akka router 6 | 7 | * Intro 8 | 9 | In Akka, Router routes messages to destination [[id:2fa03d4f-948e-4a6e-a38b-178456b578b2][Akka Actors]] called routees that can process messages in parallel to improve throughput. The way routing is done may be configured to fit the use-case. 10 | 11 | * Routing Strategies 12 | 13 | Determines how router routes to its routees. 14 | 15 | - ~RandomRoutingLogic~: Pure random and fast. May be unbalanced. 16 | - ~RoundRobinRoutingLogic~: Take turns. Is more fair and distributed. But depending on the messages, some actors may have more work than others. 17 | - ~SmallestMailboxRoutingLogic~: Tracks the size of the mailbox. 18 | - ~ConsistentHashingRoutingLogic~: Messages go to a specific routee. 19 | - ~BroadcastRoutingLogic~: Sends to all routees. 20 | - ~ScatterGatherFirstCompletedRoutingLogic~: Similar to broadcast but looks for the first response. This optimizes speed as the picks the first one. 21 | - ~TailChoppingRoutingLogic~: Similar to previous but delays a bit before sending to the next routee. 22 | 23 | * Types of Router 24 | 25 | How the routees are managed. In this case, the message delivery is optimized: 26 | - Messages are not enqueued into the router's mailbox but are delivered directly to the routees. 27 | - but delive 28 | 29 | * ~Pool Router~ 30 | 31 | Creates and supervises a number of routees according to a configuration. E.g., create 5 routees. It can be configured to dynamically adjust the number of routees. 32 | 33 | * ~Group Router~ 34 | 35 | Router is configured to route to existing actors (routees). Each actor is supervised by the parents and not by the router as opposed to previous type. In this case we need to especify the routees and will find all that match. 36 | 37 | * Special Messages 38 | 39 | - ~PoisonPill~: Is never send to the routee. In this case, this only kills the router. The outcome of the routees will depend on the type of router. On ~Pool Router~ (routees are supervised by the router), they are also go with it. On ~Group Router~ the router will kill but the routees will not because are not supervised by it. 40 | - ~Kill~: Ditto. 41 | - ~Broadcast~: Delivers to all routees regardless of the routing logic. We can use it to stop all routees by wrapping a ~PoisonPill~ inside a ~Broadcast~ message. 42 | -------------------------------------------------------------------------------- /org/akka_sharding.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578b6 3 | :END: 4 | #+TITLE: Akka Sharding 5 | #+filetags: akka 6 | 7 | * Akka Cluster Sharding 8 | 9 | Distribute actors across a cluster: 10 | - /Entities/: The main unit (e.g., ~UserId~) 11 | - /Shards/: Holds entities (e.g., each shard holds 10 ~UserIds~). 12 | - /Shard Region/: Holds Shards. 13 | - /Shard Coordinator/: Manages shards. 14 | 15 | ** Entity 16 | 17 | The main unit is: Entity identified by the ~EntityId~ which in essence represents the aggregate root's identifier of a concept of our domain (e.g., ~UserId~) and is unique within the cluster. This leads to /Strong Consistency/ given that Akka provides the single thread illusion. 18 | 19 | The ~EntityId~ is extracted through the ~ExtractEntityId~ partial function. This is often modeled using a ~Envelope~ (which is not mandatory _when_ the message contains the identifier): 20 | 21 | #+BEGIN_SRC scala 22 | case class Envelope(entityId: String, message: Any) 23 | 24 | val idExtractor: ExtractEntityId = { 25 | case Envelope(id, msg) => (id, msg) 26 | } 27 | #+END_SRC 28 | 29 | The name of the actor essentially becomes the ~EntityId~. 30 | 31 | ** Shards 32 | 33 | In turn, these entities are grouped into shards. The distribution depends on the its identifiers (~ShardId~) are generated which is usually based on ~EntityId~. An improper distribution leads to a unbalanced cluster, which leads to hotspots (e.g., names, dates, auto-incrementing ids). As rule of thumb, use ~10 shards per node as too many may be costly to find them and too short reduces the capability to distribute them. 34 | 35 | In general, the identifier shard identifier is modelled as follows: 36 | #+begin_src scala 37 | val shardIdExtractor: ExtractShardId = { 38 | case Envelope(id, _) => (Math.abs(id.hashCode % totalShards)).toString 39 | } 40 | #+end_src 41 | 42 | ** Shard Region 43 | 44 | For a type of entity, there is usually one Shard Region per JVM, in other words, we instantiate a shard region (for a given actor type) on every node that we want to host shards. Alternatively, we may merely create a proxy to a shard region. Last but not the least, a shard region is a ~ActorRef~ that we can send messages as usual. 45 | 46 | ** Shard Coordinator 47 | 48 | Manages shards. It is responsible to route the messages addressed to a specific entity. It provides the location of the shard which can then be used to acccess the entity. 49 | 50 | Runs as an Akka Cluster singleton and it does not do much of work therefore rarely becomes a bottleneck. 51 | 52 | * From Stateless systems to Stateful systems 53 | 54 | Strong consistency in a stateless system is done by leveraging a DB as the source of truth, to maintain /Strong Consistency/. However, as the system grows the DB may become a bottleneck. We can leverage Sharding: 55 | - Load is distributed across multiple machines in the cluster. 56 | - State can be cached for fast access with no need to read from DB. 57 | - /Strong Consistency/ is guaranteed by sharded actors backed by the single thread illusion. 58 | 59 | The actors may now become a source of contention, however it is distributed across multiple machines. This means that the system is /Elastic/ and we can scale as needed. 60 | 61 | When there is a failure, it is located in a single actor - /Bulkheading/. 62 | 63 | ** Stateful Actors 64 | 65 | Read may be done directly from the state. But writes may have to go through the DB followed by a update of our internal state. This is important as when the Actor fails, we need to rebuild our internal state from DB. 66 | 67 | In order to avoid blocking the system while reading from the DB we can use the ~Stash~ to store incoming messages until we rebuild the internal state. See following example: 68 | #+begin_src scala 69 | class MyActor(repository: Repository) extends Actor with Stash { 70 | 71 | var state: Option[State] = None 72 | 73 | repository.read(id) // Non-blocking DB. Asyncronous read from DB. 74 | .map(state => StateLoaded(state)) // Notify so that we can change the current context 75 | .pipeTo(self) 76 | 77 | def receive: Receive = loading 78 | 79 | def loading: Receive = { 80 | case StateLoaded(s) => 81 | state = Some(s) 82 | unstashAll // read all accumulated messages as soon as we process this message 83 | context.become(running) 84 | case State.failure(ex) => throw ex // Best practice: trigger the restart of the actor by default. The stashed messages are not lost. 85 | case _ => stash() 86 | } 87 | 88 | def running: Receive = { 89 | //regular handler 90 | } 91 | } 92 | #+end_src 93 | 94 | Now when we introduce operations that need to update the DB we need to fine-tune this. 95 | #+begin_src scala 96 | case class UpdateState(foo: Int) 97 | case class StateUpdated(state: Option[State]) 98 | 99 | class MyActor(repository: Repository) extends Actor with Stash { 100 | 101 | var state: Option[State] = None 102 | 103 | repository.read(id) // Non-blocking DB. Asyncronous read from DB. 104 | .map(state => StateLoaded(state)) // Notify so that we can change the current context 105 | .pipeTo(self) 106 | 107 | def receive: Receive = loading 108 | 109 | def loading: Receive = { 110 | case StateLoaded(s) => 111 | state = Some(s) 112 | unstashAll // read all accumulated messages as soon as we process this message 113 | context.become(running) 114 | case State.failure(ex) => throw ex // Best practice: trigger the restart of the actor by default. The stashed messages are not lost. 115 | case _ => stash() 116 | } 117 | 118 | def running: Receive = { 119 | case UpdateState(foo) => 120 | context.become(waiting) 121 | repository.update(state.copy(foo = foo)) 122 | .map(StateUpdate.apply) 123 | .pipeTo(self)(sender()) // send message to self as soon as the operation is done with the original sender() to reply back to 124 | } 125 | 126 | def waiting: Receive = { 127 | case evt @ StateUpdate(state) => 128 | unstashAll() // enqueue messages not processed while the DB was being written. 129 | context.become(running) // can process messages as usual 130 | sender() ! evt // reply back to the original sender 131 | case failure @ Status.Failure(ex) => 132 | log.error(s"[orderId] FAILURE: ${ex.getMessage}") 133 | sender() ! failure // make sure we reply back 134 | throw ex // trigger actor restart 135 | case _ => 136 | stash() 137 | 138 | } 139 | #+end_src 140 | 141 | * Passivation 142 | 143 | This fenomenon can be observer through small dips in the throughput. This happens as the Actors attempts to manage the number of actors in-memory as keep all of them is unreasonable. E.g., idle actors. 144 | 145 | Each actor tracks the time it processed a message. If it hadn't processed a message within a configured time period, it will /Passivate/, leading to the removal of the actor in-memory. 146 | 147 | The period must be tune-up, too long may lead to OOM and too short may lead to constant reads from the DB. Best practise is to determine and then tune up by watching the memory usage. 148 | 149 | It can also be done manually by sending a /Passivate/ message to the parent. 150 | 151 | * Rebalancing 152 | 153 | Occurs whenever the size of the cluster changes. The Shard coordinator will initiate the rebalancing process by distributing the shards across the all available nodes in order to keep an even distribution of entities. 154 | 155 | This can only occur in a healthy cluster. Therefore any unreachable nodes must be removed (and terminated *before*) either manually through /Akka Management/ or using the /Lightbend Split Brain Resolver/. 156 | 157 | Steps: 158 | 1. Coordinator informs Regions that a rebalance has started. 159 | 2. Messages to an entity on a moving shard are buffered. 160 | 3. Oce shared was rebalanced, the queued messages are sent 161 | 162 | During rebalancing, the messages delivered follow the /at-most-once/ semantics. 163 | 164 | There are several Shard Allocation Strategies, the default one is ~LeastShardAllocationStrategy~. 165 | 166 | However, the _*shards are not automatically restarted*_. In order for it to happen one needs to use "Remember Entities" with some costs. 167 | 168 | ** Remember Entities 169 | 170 | By enabling ~remember-entities~, when a node restarts/rebalances, it will restore entities. This works by informing every member every time each entity starts or stops (using Akka distributed data) and stored in a durable storage in the disk (it can be recovered even after full cluster restart). However, this be disabled on environments without persistent storage (e.g., Kubernetes), in those cases use ~eventsourced~ data mode (see [[https://doc.akka.io/docs/akka/current/typed/cluster-sharding.html][documentation]]). 171 | 172 | Warning! 173 | - Enabling this disables *automatic* passivation. 174 | - It is not cheap as every node will have to be informed of all running entities, which leads to an overhead starting/stopping them. 175 | 176 | Best practice is to limit when we have a limited number of active entities. Most of times is not really needed as entities will be removed automatically through /Passivation/ brought back when needed. However some use-cases: 177 | - When the entity has a scheduled process that may not have completed. 178 | - When the time to restart an entity on demand could cause the system to backup (long startup times). 179 | - When the resource savings of passivating the Entities are insignificant. 180 | 181 | With this feature, the node's ~ExtractShardId~ function must handle ~ShardRegion.StartEntity(entityId)~. 182 | 183 | Note: During startup, some nodes may become overwhelmed. In order to avoid concentrating the shards on a single member of the cluster we set the minimum number of members under cluster settings. This allows unnecessary rebalances during the startup. While the cluster has not enough members, existing members will remain in the /Join/ state. 184 | -------------------------------------------------------------------------------- /org/akka_streams.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578b7 3 | :END: 4 | #+TITLE: Akka Streams 5 | #+filetags: akka stream 6 | 7 | Actor Streams leverage the actor system to consume streams of data. In fact, each element of a stream is a Message within the Actor System 8 | 9 | Here, data flows through a chain of processing stages: 10 | - ~Sources~: The "source" (e.g., CSV file). 11 | - ~Sinks~: The "destination" (e.g., a file). 12 | - ~Flows~: Transformations made to the data within the Stream (e.g., total number of lines). 13 | - ~Runnable Graphs~: A stream where all inputs and outputs are connected. 14 | 15 | All these stages are composable and, in order to start the flow they have to be materialized. As soon they are connected, each staga materializes a value. See available operators [[https://doc.akka.io/docs/akka/current/stream/operators/][here]]. 16 | 17 | Moreover, there are two types of Streams: 18 | - ~Linear Streams~: Linear flow. 19 | - ~Graphs~: Where there may be branches points in the Stream (aka ~Junctions~). Useful for more complex use-cases. 20 | 21 | In any case, by default the stages in a ~Linear Stream~ run *syncronously* inside a single together ("Fused" together) but can also be configured to run ~asyncronously~ in separate actors. 22 | 23 | Last but not he least: backpressure is managed by a pull/push mechanism. I.e.: 24 | 1. Subscriber signals demand which is sent upstream via subscription. 25 | 2. Publishers receives demand and pushes data (if available) downstream. 26 | 27 | * Source 28 | 29 | Stage with single output: ~Source[+Out, +Mat]~: 30 | - ~Out~: The type of each element that is produced. 31 | - ~Mat~: Type of the materialized value. Usually ~NotUsed~. 32 | 33 | Source only push data as long as there is demand. The source will have to deal with incoming data until demand resumes (how how largely demands on the use-case). 34 | 35 | See available operators [[https://doc.akka.io/docs/akka/current/stream/operators/index.html#source-operators][here]]. 36 | 37 | * Sink 38 | 39 | Stage with a single input: ~Sink[-In, +Mat]~: 40 | - In: The type of each element that is consumed. 41 | - Mat The type of each element that is produced. E.g., Future[Int]. 42 | 43 | It creates backpressure by controlling /Demand/. Note that if the stream is infinite, these sinks may never complete. 44 | 45 | See available operators [[https://doc.akka.io/docs/akka/current/stream/operators/index.html#sink-operators][here]]. 46 | 47 | * Flows 48 | 49 | Single input and single output: ~Flow[-In, +Out, +Mat]~. 50 | 51 | Acts both as producer and consumer therefore it propagates demand to the producer as well propagating (and transforming) messages produced to downstream stages. 52 | 53 | The most notable operators (way too many): 54 | - [[https://doc.akka.io/docs/akka/current/stream/operators/index.html#simple-operators][Simple Operators]] 55 | - [[https://doc.akka.io/docs/akka/current/stream/operators/index.html#timer-driven-operators][Timer Driven]] 56 | - [[https://doc.akka.io/docs/akka/current/stream/operators/index.html#asynchronous-operators][Asyncronous]] 57 | - [[https://doc.akka.io/docs/akka/current/stream/operators/index.html#backpressure-aware-operators][Backpressure Aware]] 58 | - [[https://doc.akka.io/docs/akka/current/stream/operators/index.html#nesting-and-flattening-operators][Nesting and flatenning]] 59 | - [[https://doc.akka.io/docs/akka/current/stream/operators/index.html#time-aware-operators][Time Aware]] 60 | - [[https://doc.akka.io/docs/akka/current/stream/operators/index.html#fan-in-operators][Fan-in]] 61 | - [[https://doc.akka.io/docs/akka/current/stream/operators/index.html#fan-out-operators][Fan-Out]] 62 | 63 | Note that some of these operations are directly accessible from ~Source~ and does not require additional typing. 64 | 65 | Additional notes: 66 | - ~Buffer~ smooths flow inconsistencies. 67 | - ~extrapolate~ to deal with slow producers. 68 | - ~batch~ to deal with slow consumers. 69 | - ~conflate~ which creates a summary of the elements when the producer is faster - What is the usefulness? 70 | 71 | * Runnable Graphs 72 | 73 | Connects source, flows and sinks so that data can start flowing. 74 | 75 | This is done using ~via~ followed ~to~ and finally ~run~ on a ~Source~ as follows: 76 | 77 | #+begin_src scala 78 | import $ivy.`com.typesafe.akka::akka-actor:2.6.3` 79 | import $ivy.`com.typesafe.akka::akka-stream:2.6.3` 80 | 81 | import akka.actor.ActorSystem 82 | import akka.stream.scaladsl.{Flow, Sink, Source} 83 | 84 | implicit val system = ActorSystem("QuickStart") 85 | Source(1 to 10) 86 | .via(Flow[Int].map(_ * 2)) 87 | .to(Sink.foreach(println)) 88 | .run 89 | #+end_src 90 | 91 | Notes: 92 | - ~via~: connects ~Flow~ to a ~Source~ returning a new ~Source~. Also allows composing two ~Flows~. 93 | - ~to~: connects a ~Sink~ to a ~Source~ returning a ~RunnableGraph~. Also connects to a ~Flow~ to build new ~Sink~. In essence, it materializes the value from the stage is called on. 94 | - The ~run~ is a terminal operator. There are others. 95 | 96 | While the flow is *running*, values are materialized. These values are then acessed using ~to~, ~source.toMat(Sink)(Transform/CombineFunction)~, ~~source.viaMat(flow)(Transform/CombineFunction)~. 97 | 98 | Finally, there are some shortcuts: 99 | - ~Source.runWith(Sink)~. 100 | - ~Source.runForeach(Function)~. 101 | - ~Source.runFold(0)(_ + _)~. 102 | - ~Source.runReduce(_ + _).~ 103 | 104 | * Fault Tolerancy - TODO: Review Examples 105 | 106 | Default strategy is to stop processing the stream and can be overriden within the ~ActorMaterializer~ by passing a decider that given an exception it either decides: 107 | - /Stop/: terminate with an error. 108 | - /Resume/: Drop the failing element. 109 | - /Restart/: The element is dropped and the stream continues after restarting the stage. Any state acumulated by that stage will be cleared. 110 | 111 | Via attributes, each stage can be fine-tuned: 112 | - ~Dispatcher~ 113 | - ~Buffer Sizes~ 114 | - ~Log Level~ 115 | - ~Supervision~ 116 | 117 | With a Supervision Strategy: 118 | #+begin_src scala 119 | import $ivy.`com.typesafe.akka::akka-actor:2.6.3` 120 | import $ivy.`com.typesafe.akka::akka-stream:2.6.3` 121 | 122 | import akka.NotUsed 123 | import akka.actor.ActorSystem 124 | import akka.stream.{ActorAttributes, Supervision} 125 | import akka.stream.scaladsl.{Flow, Sink, Source} 126 | import java.lang.ArithmeticException 127 | 128 | val decider: Supervision.Decider = { 129 | case _: ArithmeticException => Supervision.Resume 130 | case _ => Supervision.Stop 131 | } 132 | 133 | val possibleDivisionByZero = 134 | Flow[Int].map(i => 100 / i) 135 | .withAttributes( 136 | ActorAttributes.supervisionStrategy(decider) 137 | ) 138 | 139 | implicit val system = ActorSystem("QuickStart") 140 | Source(-1 to 1) 141 | .via(possibleDivisionByZero) 142 | .runWith(Sink.foreach(println)) 143 | #+end_src 144 | 145 | However some errors are recoverable, in this case we provide a ~PartialFunction[Throwable, T]~. It will terminate the stream graciously passing the resulting value as the final value. 146 | 147 | #+begin_src scala 148 | import $ivy.`com.typesafe.akka::akka-actor:2.6.3` 149 | import $ivy.`com.typesafe.akka::akka-stream:2.6.3` 150 | 151 | import akka.NotUsed 152 | import akka.actor.ActorSystem 153 | import akka.stream.{ActorAttributes, Supervision} 154 | import akka.stream.scaladsl.{Flow, Sink, Source} 155 | import java.lang.ArithmeticException 156 | 157 | val possibleDivisionByZero = 158 | Flow[Int].map(i => 100/i) 159 | .withAttributes( 160 | ActorAttributes.supervisionStrategy(decider) 161 | ) 162 | .recover { 163 | case _: ArithmeticException => 0 164 | } 165 | 166 | implicit val system = ActorSystem("QuickStart") 167 | Source(-1 to 1) 168 | .via(possibleDivisionByZero) 169 | .runWith(Sink.foreach(println)) 170 | #+end_src 171 | 172 | * Graphs 173 | 174 | Introduces ~Junctions~ which take multiple inputs and multiple outputs. Basic ones are: 175 | - ~Fan-in~: N inputs + 1 output. See [[https://doc.akka.io/docs/akka/current/stream/operators/index.html#fan-in-operator][operators]]. 176 | - ~Fan-out~: 1 input + N outputs. See [[https://doc.akka.io/docs/akka/current/stream/operators/index.html#fan-out-operators][operators]]. 177 | 178 | For example, using ~Fan-in~ we can randombly select one of the inputs, give preference or merely zip them. Then, using ~Fan-out~, we can broadcast the values or unzip to create two individual streams. 179 | 180 | [[file:_20201014_220602screenshot.png]] 181 | 182 | We can use ~GraphDSL~ to easily connect them visually ([[https://doc.akka.io/docs/akka/current/stream/stream-graphs.html][documentation]]). E.g.: 183 | #+begin_src scala 184 | import $ivy.`com.typesafe.akka::akka-actor:2.6.3` 185 | import $ivy.`com.typesafe.akka::akka-stream:2.6.3` 186 | 187 | import akka.NotUsed 188 | import akka.actor.ActorSystem 189 | import akka.stream.{ActorAttributes, ClosedShape, Supervision} 190 | import akka.stream.scaladsl.{Broadcast, Flow, Merge, RunnableGraph, Sink, Source, GraphDSL} 191 | 192 | implicit val system = ActorSystem("QuickStart") 193 | RunnableGraph.fromGraph(GraphDSL.create() { implicit builder: GraphDSL.Builder[NotUsed] => 194 | import GraphDSL.Implicits._ 195 | val in = Source(1 to 10) 196 | val out = Sink.foreach(println) 197 | 198 | val bcast = builder.add(Broadcast[Int](2)) 199 | val merge = builder.add(Merge[Int](2)) 200 | 201 | val f1, f2, f3, f4 = Flow[Int].map(_ + 10) 202 | 203 | in ~> f1 ~> bcast ~> f2 ~> merge ~> f3 ~> out 204 | bcast ~> f4 ~> merge 205 | ClosedShape // Indicates no open inputs or outputs, this means that the graph is runnable. The opposite means that it is a partial graph. 206 | }).run() 207 | #+end_src 208 | 209 | It is also possible to build partial graphs, or ~Shapes~. There are already some built-in: 210 | - Linear Shapes (~Shape~) 211 | - Junction Shapes with the same input/output types (~UniformFanShape~). 212 | - Junction Shapes with different inputs/outputs types (~FanShape~). 213 | 214 | Simpler graphs can be done using, for example, the simpler ~Sink.combine~ API. 215 | 216 | * Fusion 217 | 218 | By default, Akka "fuses" all stages onto a single syncronous one to run on a single actor (auto-fusing can be disabled) but this limits the benefits we are looking for. 219 | 220 | [[file:_20201014_224503screenshot.png]] 221 | 222 | In order to add a asyncronous boundary, we just need to add ~async~ which disables fusing for that stage, which means that we are adding an additional overhead (Actors, mailboxes and buffers). Its benefits largely depends on the use-case. A good principle is 223 | #+begin_quote 224 | 1. Insert an async boundary to bisect the stream into two subsections of roughly equal processing time. 225 | 2. We insert an async boundary to bisect the stream into two subsections of roughly equal processing time. 226 | #+end_quote 227 | 228 | In other words, check at the current pipeline where the stages can be split so that they can be performed in paralell and joined almost at the same time. This implies looking at Telemetry and verify which stages can be processed in paralell given the graph we have. 229 | -------------------------------------------------------------------------------- /org/akka_testing.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578b8 3 | :END: 4 | #+TITLE: Akka Testing 5 | #+filetags: akka 6 | 7 | How to test actors in Akka. 8 | 9 | * Test Actor Ref 10 | 11 | Synchronous and ideal for white-box testing as we can change the internal state. 12 | 13 | * Test Probes 14 | 15 | Black-box testing and then verify if specific messages were sent (and other assertions). 16 | 17 | (black box is always better :) ) 18 | -------------------------------------------------------------------------------- /org/amdah_s_law.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578b9 3 | :END: 4 | #+TITLE: Amdah's Law 5 | #+filetags: concurrency 6 | 7 | In short, contention limits paralelization. 8 | 9 | Defines the maximum improvement gaines by parallel procesing. Improvements from paralelization are limited to the code that can be paralelized. Contention limits such paralism reducing the advantages of the improvements. Does not matter as long as the contention exist. 10 | -------------------------------------------------------------------------------- /org/anti-corruption-layer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bphenriques/knowledge-base/0490478c2ce82c257883eed1b6e6468fc66184f5/org/anti-corruption-layer.png -------------------------------------------------------------------------------- /org/base_transaction.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578c0 3 | :END: 4 | #+TITLE: BASE Transaction 5 | #+filetags: system-design 6 | 7 | Sometimes ACID transactions are not possible in some cases, e.g., microservices. As alternative, we use BASE transactions: 8 | - Basically Available 9 | - Soft State 10 | - Eventual Consistency 11 | 12 | The difference from ACID transactions it that they cannot be rolledback easily. To rollback, a compensating action is needed to revert to the original state. Saga manages these kind of transactions and are often used to manage different aggregate roots (see [[id:2fa03d4f-948e-4a6e-a38b-178456b578c5][Domain Driven Design]]). 13 | -------------------------------------------------------------------------------- /org/bug-management.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578c1 3 | :END: 4 | #+TITLE: Bug Management 5 | #+filetags: monitoring 6 | 7 | Some personal notes on simplifying the process so that one can focus on getting back to the other tasks at hand. 8 | 9 | * On Reporting Bugs 10 | 11 | - Focus on the impact for the client. 12 | - You do not need to debug right away. 13 | - You do not need to establish the timeline - The report can solely include the context. 14 | - The person assigned to the issue will pick on the context provided in the ticket and explore. 15 | -------------------------------------------------------------------------------- /org/cloud_testing.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 7a28e879-b7f3-4cfd-b78c-381d66253825 3 | :END: 4 | #+title: Cloud Testing 5 | #+filetags: AWS Testing 6 | 7 | In order to test AWS services locally, we can use https://docs.localstack.cloud/ which runs locally and offline. 8 | -------------------------------------------------------------------------------- /org/command_query_responsibility_segregation.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 3fa03d4f-948e-4a6e-a38b-178456b578c1 3 | :END: 4 | #+TITLE: Command Query Responsibility Segregation (CQRS) 5 | #+filetags: system-design 6 | 7 | Command Query Responsibility Segregation (CQRS) is a architectural pattern that aims to split applications between two models: ~Command Model~ (writes) and ~Query Model~ (reads). Some use cases: 8 | - Auditing (e.g., banking, accounting) 9 | - High Scalability 10 | - High Resiliency 11 | 12 | This separation allows both concerns to evolve separately depending on the requirements. For example, some Aggregate Roots are a better fit for write models but do not fit other read models. 13 | 14 | This pattern is often combined with [[id:2fa03d4f-948e-4a6e-a38b-178456b578c7][Event Sourcing (ES)]]. 15 | 16 | Eventually consistent by design which always present but now is explicit about it. 17 | 18 | * How: 19 | 20 | Application is split between two models: 21 | - *Command Model*: Handles requests to change the state of the application and decide its side-effects, e.g., events or new commands. 22 | - *Query Model* (or View Model or Projection): Focus on data and not on behavior and are modelled to satisfy a very specific need, therefore it is usual having multiple of them. 23 | 24 | #+BEGIN_SRC 25 | API -Queries-> Read Model <- Data Store 26 | API -Commands-> Write Model -> Data Store 27 | #+END_SRC 28 | 29 | Separate process consumes the written events and persists on a denormalized event store, which is called /Projection/ used by the read model like so: 30 | 31 | #+BEGIN_SRC 32 | API -Queries-> Read Model <- Data Store 33 | API -Commands-> Write Model -> Data Store 34 | 35 | Data Store -Events-> Denormalized Data Store (with Projections) 36 | #+END_SRC 37 | 38 | The Data Store is usually denormalized to make sure that queries are faster. 39 | 40 | * Summary 41 | 42 | - Write model are optimized for writes 43 | - Read models are optimized for reads 44 | - Read and write models are decoupled which implies that they may use different data stores. 45 | - New /Projections/ are easy in CQRD/ES. 46 | - New projections are retroactive because we have the fully history. 47 | 48 | * Models as Microservices 49 | 50 | - Write Model can become a microservice 51 | - Read Model can become a microservice 52 | 53 | This assumes that it uses different databases (as expected from proper microservices). Otherwise the database may become the bottleneck. 54 | 55 | Better yet: Each projection in its own Microservice. $$$$$$$$$$$$ and maintentance. 56 | 57 | 58 | * Consistency 59 | 60 | Simple (without ES) has the same consistency as non-CQRS systems. 61 | CQRD/ES can have different consistency models for the read or the write models. 62 | 63 | ** Write Model 64 | 65 | Strong is often important here because we want that those write be based on the current state. This consistency is usually implemented through locks or sharding in a more reactive way. 66 | 67 | ** Read Model 68 | 69 | Pure reads are never consistent as they are often working with stale data. These reads do not need strong consistency. 70 | 71 | * Availability 72 | 73 | ** Write Model 74 | 75 | Due to the higher consistency, availability is lower. 76 | 77 | ** Read Model 78 | 79 | Due to the eventual consistency, we can leverage technicques to increase availability. 80 | 81 | 82 | * Cost 83 | 84 | CQRD/ES is often criticized for being more complex but it can be simpler. 85 | Without this, models are more bloated, complex and rigid. 86 | CQRD allow smaller models that are easier to modify and understand. 87 | Eventual consistency in CQRS can be isolated to where it is necessary. 88 | 89 | 90 | - More databases to maintain 91 | - More classes/objects to maintain 92 | - Support older versions can be challenging 93 | - Additinonal storage 94 | - Data duplication may result in desyncs that often solved by rebuilding project - Question: *when* ? Do we have monitoring over this? Can this be automatic? How often does this occur? 95 | - UI must be designed to be eventually consistent (which was always there in the past, it is now explicit) 96 | -------------------------------------------------------------------------------- /org/command_sourcing.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578c2 3 | :END: 4 | #+TITLE: Command Sourcing 5 | #+filetags: system-design 6 | 7 | * Command Sourcing 8 | 9 | Similar to [[id:2fa03d4f-948e-4a6e-a38b-178456b578c7][Event Sourcing (ES)]] but persists commands as opposed to events so: 10 | 1. Issue command 11 | 2. Persist command 12 | 3. Run asyncronous the command 13 | 14 | - They should be idempotent as they run multiple times (e.g., failures). 15 | - Must be validated so that they do not become stuck in the queue forever. 16 | - Bad: The sender might not be notified if the command fails due to the decouple nature. 17 | -------------------------------------------------------------------------------- /org/consistency_and_availability.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578c3 3 | :END: 4 | #+TITLE: Consistency And Availability 5 | #+filetags: system-design 6 | 7 | * Scalability 8 | 9 | It can meets increases in demand while remaining responsive. 10 | 11 | This is different from performance. Performance optimizes response time (latency) while scalability optimizes ability to handle load. Requests per second actually measures both but we do not know which aspect was improved. 12 | 13 | *Note* Scalability is not the number of requests qwe can handle a in a given period of time (req/sec) but he number of requests itself (load). 14 | 15 | If x axis is number of requests (Load) and y axis is response time. Improving performance leads to decrease in the y axis. Improving scalability means a shift on the x axis meaning that we can handle more requests with the same response time. 16 | 17 | Still confused as if I improve performance I should free up resources to handle more requests.. 18 | 19 | Reactive Microservices focus on improving scalability. 20 | 21 | * Consistency: 22 | 23 | All members of the system have the same view or state. This does not factors time. 24 | 25 | ** Eventual Consistency 26 | 27 | Guarantees that, in the absence of new updates, all accesses to a specific piece of data will eventually return the most recent data. 28 | 29 | Different forms: 30 | - Eventual Consistency 31 | - Causal Consistency 32 | - Sequential Conssitency 33 | - others 34 | 35 | E.g., Source control are eventually consistent. All the code reading is potentially out-of-date and a merge operations is relied upon to bring the local state back to speed. 36 | 37 | *** TODO: Check each one 38 | 39 | ** Strong Consistency 40 | 41 | An update to a piece of data needs agreement from all nodes before it becomes visible. 42 | 43 | Physically it is impossible therefore we simulate: Locks that introduce overhead in the form of contention. Now it becomes a single resource which eliminates the distributed nature. Tying the distributed problem to a non-distributed resource. 44 | 45 | Traditionally, monoliths are based around strong consistency. 46 | 47 | *** Effects of contention 48 | 49 | Definition: Any two things that contend for a single limited resource and only one will win and the other will be forced to wait. 50 | 51 | As the number of resources disputing for the resource, more time time it will take to finally free up the resources. 52 | 53 | See: 54 | -[[id:2fa03d4f-948e-4a6e-a38b-178456b578b9][ Amdah's Law]] 55 | - [[id:2fa03d4f-948e-4a6e-a38b-178456b578d1][Gunther's Universal Scalability Law]] 56 | 57 | **** Coherence Delay 58 | 59 | Definition: Time it takes for synchronization to complete on a distributed systems - My definition following below notes: 60 | 61 | Syncronization is done using crosstalk or gossip - Each system sends messages to each other node informing of any state changes. The time it takes for the cynscronization to complete is called *Coeherency Delay*. 62 | 63 | Increasing the number of nodes increases the delay. 64 | 65 | **** Laws of scalability 66 | 67 | Both these laws demonstrate that linear scalability is almost always unachivable. Such is only possible if the system lieve in total isolation. 68 | 69 | **** Reactive Systems 70 | 71 | Reduce contention by: 72 | - Isolating locks 73 | - Eliminating transactions 74 | - Avoiding blocking operations 75 | 76 | Mitigate coherency delays by: 77 | - Embracing Eventual Consistency 78 | - Building in Autonmy 79 | 80 | This allows for higher scalability as we reduce or eliminate these factors. 81 | 82 | 83 | * CAP Theorem 84 | 85 | States that a distributed system cannnot provide more than than two of the following: 86 | - Consistency 87 | - Availability 88 | - Partition Tolerance 89 | 90 | One has to pick one of the following combinations: 91 | - (CP) Consistent and Partition Tolerance 92 | - (AP) Available and Partition Tolerance. 93 | 94 | In practice, they may claim CP/AP except for some edge-cases. It is a balance. 95 | 96 | ** Partition Tolerance 97 | 98 | The system continues to operate despite an arbitrary number of messages being dropped (or delayed) by the network. 99 | 100 | They can occur due to: 101 | - Problems in the network. 102 | - When a node goes down. 103 | 104 | May be short or long lived. 105 | 106 | 107 | Two options: 108 | - (AP) Sacrifice Consistency: Allow writes to both sides of the partition. This require merging the data in order to restore consistency. 109 | - (CP) Sacrifice Availability: Disabling or terminating on side of the partitions. During this, some or all of your system will be unavailable. 110 | 111 | * Sharding as a way to have strong consistency 112 | 113 | Limit the scope of the contention and reduce crosstalk. Is applied within the application. It is not the same type of sharding used in some databases, the technique is similar though. 114 | 115 | Allows strong consistency. 116 | 117 | Partitions entities (or Actors) in the domain according to their id. 118 | 119 | Groups of entities are called a shard and each entity only exists in one shard. 120 | 121 | Each shard exists in only one location. This fact eliminates the distributed systems problem. 122 | 123 | The entity acts as a consistency boundary. 124 | 125 | In order for this to work, we need to have a coordinator that ensures that traffic for a particular entity is routed to the correct location. The coordinator uses the ID to calculate the appropriate shard. 126 | 127 | Aggregate Roots are good candidate for sharding. 128 | 129 | It is important to have a balanced shards and that requires a good sharding key - UUIDs or hashcodes. Poor key selections will result in hotspots. 130 | 131 | Rule of thumb: 10x as many shards as nodes. 132 | 133 | Akka provides this as a means to distribute actors across a cCLuster in a shared setup. Lagom persistent entities levarage akka cluster sharding to distribute the entities across the cluster. 134 | 135 | What about resharding? when a system goes down... 136 | 137 | Sharding allows a great caching solution as: 138 | - We can store the cache results after writing to the database 139 | - Databases is effectively write-only which can speed up things 140 | - We only consult the cache during reads. 141 | - Begs the question: How many items and what is the TTL? Well.. it for certain reduces the read on the DB but that is not forever unless we have infinite memory. 142 | 143 | ** Effects 144 | 145 | - Does not eliminate contention. It solely isolates to a single entity. 146 | - The router/coordinator represents a source of contention as well. 147 | - A shareded system minimizes contention by: 148 | - Limiting the amounf of work the router/coordinator performs - By storing where the shard is after asking the coordinator - How to invalidate that cache due to failures? 149 | - Isolates contention to individual entities 150 | 151 | Scalability is doen by distributing the shards over mode machines. 152 | Strong consistency is achiaved by isolating operations to a specific entity. 153 | Careful choice of shard keys is important to maintain a good scalability. 154 | 155 | ** Failure 156 | 157 | Sharding sacrifices availability. Once a shard goes down, there will be a period of time where it is unavailable and wil migrate to another node eventually. 158 | 159 | * CRDTs provide a availability solution based on async replication 160 | 161 | Conflict-free Replicated Data 162 | 163 | On the application level. 164 | 165 | Highly available and eventually consistent. 166 | 167 | Specially designed data type. 168 | 169 | Updates are applied on one replica and then copied async. 170 | 171 | Udpdates are merged to determine the final state. 172 | 173 | Two types: 174 | - CvRDT - Convergent Replicated Data Type copy state between replicas. Requires a merge operation that understands how to combine two states. These operations must be: commutative, associative and idempotent. 175 | - CmRDT - Commutative Replicated Data Types. These copy operations isntead of state. 176 | -------------------------------------------------------------------------------- /org/context-map.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bphenriques/knowledge-base/0490478c2ce82c257883eed1b6e6468fc66184f5/org/context-map.png -------------------------------------------------------------------------------- /org/digital_garden.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578c4 3 | :END: 4 | #+TITLE: Digital Garden 5 | #+filetags: note zettelkasten 6 | 7 | Digital gardening is a different take to blogging as it shifts the focus of publishing content to a public to the writing. 8 | 9 | Like gardening, digital gardens start with raw notes and, as we tend to it, connections will start to emerge. But it is personal and informal. There is no pressure on having complete thoughts. You can always circle back to the notes you made in the past and refine them. 10 | 11 | // TODO: Note on how entropy grows and correlate with gardening. See https://joelhooks.com/digital-garden 12 | 13 | * Visualizing Digital Gardens 14 | 15 | Visualizing your notes and how they connect with one another is a great way to have a bird-eyes view on how you perceive 16 | a topic and it can even help you maturing what you know about it. For example, this This digital garden can be visualized 17 | and explored through the following graph: 18 | 19 | #+begin_export html 20 | {{< graph >}} 21 | #+end_export 22 | 23 | * History 24 | 25 | See Maggie Appleton's excellent [[https://maggieappleton.com/garden-history][article]]. 26 | -------------------------------------------------------------------------------- /org/domain_driven_design.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578c5 3 | :END: 4 | #+TITLE: Domain Driven Design 5 | #+filetags: system-design lightbend 6 | 7 | Approach on Software Development focused on the design of a shared Model understood by domain experts and by who implements it. Being a model, it means that it can be implemented in different ways, from diagrams to software. 8 | 9 | * Concepts 10 | 11 | - *Context*: The setting a word appers determines its meaning. 12 | - *Domain*: Sphere of knowledge, influence or activity (aka /area/). 13 | - *Model*: System of abstractions that describes a part of the domain. 14 | - *Ubiquitous Language*: Common language between domain experts and developers. 15 | - *Aggregate*: Entity or group of entities bounded a root entity (Aggregate Root) that are always consistent (within a ACID transaction). Serves as a building block to implement ~Command~ within [[id:3fa03d4f-948e-4a6e-a38b-178456b578c1][Command Query Responsibility Segregation (CQRS)]]. 16 | 17 | * Strategies 18 | 19 | On complex domains, it is hard to have a single unified model, therefore the model is usually split into multiple ones. Follows some strategies used to maintain integrity. 20 | 21 | * Bounded Context 22 | 23 | Concepts may change from one context to another. It is important to be explicit about it: 24 | 1. Explicitely define the context where the model applies. 25 | 2. Explicitely set the boundaries, e.g., team organization, code-bases, database schemas. 26 | 3. Keep the model strictly consistent within these bounds. 27 | 28 | Maintain these boundaries strong allows smoother workflows. 29 | 30 | Typically [[id:2fa03d4f-948e-4a6e-a38b-178456b578d7][Microservices]] are build around these contexts. Follows some tips to define them: 31 | - Define use-cases. 32 | - Look how different groups of people interact with a given entity. 33 | - Look for variations of the ubiquitous language as they may suggest a new context. 34 | - Look for variations where informations starts to become or relevant or irrelevant. 35 | 36 | * Anti-Corruption Layer 37 | 38 | Solves issues where coupling starts. This layer leaves right next to the bounded context to avoid leaking info from/to that bounded context. 39 | 40 | Typical example: Let's put everything in the same bag. 41 | 42 | What happens is that we have a layer responsible for translating similar concepts from one bounded context to another. 43 | 44 | How to implement: Abstract interface as it represents the contract in its purest way without compromising - Sometimes abstractions are too much indirection but do understand. :shrug: 45 | 46 | #+BEGIN_SRC plantuml :file anti-corruption-layer.png :exports results :eval never-export 47 | actor User 48 | control "Anti Corruption Layer" as ACL 49 | entity Component 50 | 51 | User -> ACL: Process X 52 | ACL -> ACL: Translate concept X to Y 53 | ACL -> Component: Process Y 54 | Component --> ACL: Z 55 | ACL --> User: Z 56 | #+END_SRC 57 | 58 | #+RESULTS: 59 | [[file:anti-corruption-layer.png]] 60 | 61 | This is also useful for legacy systems. In this case a Anti Corruption Layer would be preferable on either end. 62 | 63 | * Context Map 64 | 65 | Are a way of of visualizating Bounded contexts and the relationships between them. 66 | 67 | #+BEGIN_SRC plantuml :file context-map.png :exports results :eval never-export 68 | [Bounded Context A] -> [Bounded Context B] 69 | [Bounded Context A] -> [Bounded Context C] 70 | [Bounded Context B] -> [Bounded Context C] 71 | [Bounded Context C] -> [Bounded Context D] 72 | #+END_SRC 73 | 74 | #+RESULTS: 75 | [[file:context-map.png]] 76 | 77 | 78 | - Arrows represent dependencies. 79 | - Lines may be labelled to indicate the nature of the relationship. 80 | 81 | * TODO Discovery Process using Event Storming 82 | 83 | * Types of Domain Activities 84 | 85 | - *Command*: A request yet to happen and can be rejected. Usually delivered to a specific destination and changes the state of the domain. 86 | - *Events*: Action that happened in the past. Hence they can not be rejected. Often broadcast to many destinations. Record a change to the state of the domain, often the result of a command (what are the alternatives?). 87 | - *Query*: Requestfor information about the domain. Usually delievered to a specific destination. Do not change the state of the domain. 88 | 89 | All of these represent the types of messages in a Reactive System and form the API for a Bounded Context. 90 | 91 | * Domain Objects 92 | 93 | - *Value objects*: Defined by its attribute. Immuatable. Messages in Reactive Systems are implemented as Value Objects. 94 | - *Entity*: Defined by an unique identity. The fields may change but not its identitity. Are the source of truth - Actors in Akka or Entitities in Lagom 95 | [[id:2fa03d4f-948e-4a6e-a38b-178456b578d8][Monolith]] - *Aggregate*: Collection of domain objects bound to a root entity: 96 | - Example: Person (Aggregate Root), Name (Aggregate), Address (Aggregate). 97 | - Transactions should not span multiple aggregate roots. 98 | - The Aggregate Root may change between bounded contexts. Aggregate Root == Root Entity. 99 | - Good cadnidates for distribution in Reactive Systems. 100 | - Question: How to determine? 101 | - Is the entity involved in more operations in the same bounded context? 102 | - Does it make sense deleting other entities when this one is deleted? 103 | - Will a single transaction span multiple entities? 104 | 105 | * Domain Abstractiosn 106 | ** Services 107 | 108 | Busines Logic encapsulated. Should be stateless otherwise they become an entity or a value object. 109 | 110 | Should be fairly thin. 111 | 112 | ** Factories 113 | 114 | Constructing domain object may not be trivial as they may have to access external resources (DBs, files, REst APIs, etc). 115 | 116 | ** Repositories 117 | 118 | Similar to factories but used to get or modify existing objects. They work often over databases but can work with files or Rest Apis (I actually prefer "Gateways" for Rest APIs). 119 | 120 | Note: Factories and Repositories can be the same in practice. 121 | 122 | * Hexagonal Architecutre 123 | 124 | Is not directly related with domain driven design but is very compatible. 125 | 126 | Domain is at the core and is at teh center becoming the architectural focus. Then there are ports to communicate with the domain exposed as API for the domain. INfrastructure contains adapters that map to the ports. 127 | 128 | Like an Onion 129 | 130 | - Domain 131 | - API - The ports 132 | Infrastructure - Adapts incoming and outgoing traffic in to the ports. 133 | 134 | Outer layers depends on inner layers. And inner layers have no knowledge of other layers. 135 | 136 | :thinking: This does not seem different from the typical layered design with DB -> Services -> API 137 | 138 | Ensures proper spearation of infrastructure from domain. 139 | 140 | These layers may be modelled through packages or projects. Details are not important. The important thing is to make the domain portable. 141 | 142 | Would like a concrete example on how it really differents from the N tiered design. 143 | -------------------------------------------------------------------------------- /org/emacs.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578c6 3 | :END: 4 | #+TITLE: Emacs 5 | #+filetags: editors 6 | 7 | * Org-Mode 8 | 9 | Org-protocol is cool and opens possibilities (like there weren't enoguh :overwhelmed:): 10 | - https://orgmode.org/worg/org-contrib/org-protocol.html#orgheadline8 11 | 12 | * Configuring 13 | 14 | Literate config is a thing but unfortunately as far as I researched it has to be contained in a single file. I rather have separate files per use-case. 15 | 16 | Cool reference links: 17 | - https://tecosaur.github.io/emacs-config/config.html 18 | - http://doc.norang.ca/org-mode.html:w 19 | - https://github.com/jethrokuan/dots/blob/0064ea2aab667f115a14ce48292731db46302c53/.doom.d/config.el#L495 20 | - https://github.com/nmartin84/.doom.d#orgb81fe7f 21 | - https://github.com/howardabrams/dot-files/blob/master/emacs-org.org 22 | -------------------------------------------------------------------------------- /org/event_sourcing_es_es.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578c7 3 | :END: 4 | #+TITLE: Event Sourcing (ES) 5 | #+filetags: system-design 6 | 7 | Representing the application's state through the history of events that have happened in the past. Use cases: 8 | - *Audit Logs*: Build tailored reports from the event stream. 9 | - *Analytics*: Extract behavior from the event stream. 10 | - *Temporal Reports*: Build the timeline that led to a certain state. 11 | 12 | This is the opposite of regular applications where the final state is stored. In Event Sourced applications, the final state is called ~Materialized State~. Whenever we need to obtain the current state we replay the logs until we reach the current state *without replaying the side-effects*. It is possible to keep both representation but that is complex and may become out-of-sync. Moreover, append-only operations are more efficient in databases. 13 | 14 | #+begin_src 15 | Event1 -> Event2 -> Event3 ----> Materialized State 16 | #+end_src 17 | 18 | As events become the source of truth, it is of utmost importance that they are immutables. 19 | 20 | * Optimization through snapshots 21 | 22 | What happens if the list of events is too large? 23 | *Solution*: Ocasionally persist a snapshot and we replay the events from that point on (issue: We may receive an event out-of-order, invalidating the snapshot). 24 | 25 | * Versioning 26 | 27 | Issue when we have to change the event's schema. This leads to ~ModelV1~, ~ModelV2~, etc. This requires _supporting_ all versions. And requires flexibile formats: JSON, ProtoBuf or AkkA Event Adapters in the lightbend ecosystem that is between the system and the DB translating the ~V1~, ~V2~, ~VN~ to the corresponding and unique Domain entity. 28 | 29 | * Problems 30 | 31 | When we need to perform queries that involve several aggregate roots. Because entities need to be rebuilt from events everytime they are visited. 32 | 33 | #+begin_quote 34 | The model used to persist are not compatible with the model required during queries. 35 | #+end_quote 36 | 37 | See [[id:3fa03d4f-948e-4a6e-a38b-178456b578c1][Command Query Responsibility Segregation (CQRS)]]. 38 | -------------------------------------------------------------------------------- /org/from-data-to-knowledge.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bphenriques/knowledge-base/0490478c2ce82c257883eed1b6e6468fc66184f5/org/from-data-to-knowledge.png -------------------------------------------------------------------------------- /org/git.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578c8 3 | :END: 4 | #+TITLE: Git 5 | #+filetags: snippets 6 | 7 | * Get Previous Tag 8 | 9 | #+begin_src 10 | $ git for-each-ref --sort=creatordate --format '%(refname)' refs/tags | sed 's#^refs/tags/##' | tail -n 2 | head -n 1 11 | #+end_src 12 | -------------------------------------------------------------------------------- /org/gossip_protocol.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578c9 3 | :END: 4 | #+TITLE: Gossip Protocol 5 | #+filetags: protocols 6 | 7 | Some of these text may be interwined with specifities of the Gossip Protocol within [[id:2fa03d4f-948e-4a6e-a38b-178456b578b3][Akka Cluster]]. 8 | 9 | Context: TODO 10 | 11 | At a regular interval, each member sends their view of the cluster state to a random node, including: 12 | - The status of each member 13 | - If each member has seen this version of the cluster state. 14 | 15 | Eventually consistent as after some time (aka ~convergence~), all nodes will share the same state. Each node decides if it has reached convergence. 16 | 17 | After it, each node will have the same ordered set of nodes: 18 | - Node 1 19 | - Node 2 20 | - Node 3 21 | 22 | Then the first eligible node, will become the leader and will perform leader's duties such as notify member state changes 23 | 24 | Note: that each leader may change after each round if the cluster membership has changed. 25 | 26 | * Member Lifecycle - Happy Path 27 | 28 | #+BEGIN_SRC 29 | Joining -> Up -> Leaving -> Exiting -> Removed (tombstoned) 30 | #+END_SRC 31 | 32 | In this case: 33 | - Once all nodes know that all nodes know that a new node is "Joining", it marks it as "Up". 34 | - A leaving node sets itself as ~Leaving~, then once all nodes know it, the leader marks it as ~Exiting~. 35 | - Once all leaders know that a node is ~Exiting~, the leader sets the node as ~Removed~. 36 | 37 | * Member Lifecycle - Unhappy Path 38 | 39 | #+BEGIN_SRC 40 | Joining -> Up -> Leaving -> Exiting -> Removed (tombstoned) 41 | | | | | /|\ 42 | |--disconnected--------------| | 43 | \|/ | 44 | Unreachable (F) ----> Down (F) ------------ 45 | #+END_SRC 46 | 47 | When a new member is disconnected, it is marked as Unreachable until it recovers. If it is permanent, it is flagged as Down and it moves to state Removed and can never come back. 48 | 49 | Each node is monitored for failures by at most 5 nodes using [[id:2fa03d4f-948e-4a6e-a38b-178456b578d3][Heartbeat]]. Once a member is deemed unreachable, then than information will be gossiped to the cluster. The member will remain like that until all nodes flag the node as ~Reachabe~ again. 50 | 51 | Reasons: 52 | - Crashes 53 | - Heavy load 54 | - Network Partition or failure 55 | 56 | ** Impact of Unreachable Nodes 57 | 58 | *Convergence will not be possible* therefore no leaders will be elected therefore new nodes cannot fully join the cluster. This will happen until the node is marked as ~Down~ (and then ~Removed~). 59 | 60 | In this moment, potential new members will have a state ~WeaklyUpMember~, which can transition to ~Up~ once convergence is complete. Note that, this member can be used by applications but *should not* if consistency is important (Cluster Shards or Cluster Singletons). 61 | 62 | * TODO 63 | 64 | - Check if Heartbeat and Gossip Protocol are /always/ interwined. 65 | -------------------------------------------------------------------------------- /org/gpg.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578d0 3 | :END: 4 | #+TITLE: GPG 5 | #+filetags: security signing 6 | 7 | Automatically export public key to a server: 8 | #+begin_src 9 | $ gpg --keyserver pgp.mit.edu --send-keys 10 | #+end_src 11 | -------------------------------------------------------------------------------- /org/gunther_s_universal_scalability_law.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578d1 3 | :END: 4 | #+TITLE: Gunther's Universal Scalability Law 5 | #+filetags: concurrency 6 | 7 | Increasing concurrency can cause negative resutrns due to contention and coherency delay. 8 | 9 | Picks from [[id:2fa03d4f-948e-4a6e-a38b-178456b578b9][Amdah's Law]]. In addition to contention, it accounts for coeherency delay. 10 | 11 | As the system scales up, the cost to coordinate between nodes exceeds any benefits. 12 | -------------------------------------------------------------------------------- /org/hands_on_scala_programming.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578d2 3 | :END: 4 | #+TITLE: Hands-on Scala Programming 5 | #+filetags: scala 6 | 7 | Follows my notes on the Haoyi Li.'s book: "Hands-on Scala Programming" (https://www.handsonscala.com/). 8 | 9 | As an experiment, I coding directly in the org-mode file using Babel to execute the Scala blocks: 10 | 11 | * TODO Some introduction 12 | - [ ] Point to dotfiles 13 | - [ ] Point to the Babel package 14 | 15 | * Notes 16 | 17 | - Lack of auto-complete when writing here. 18 | - Compilation errors are hard to track. Workaround is to open a separate buffer with the ammonite REPL console. 19 | - For sure will have to move to a dedicated project once I use Mill. Or maybe not. 20 | 21 | * Exercises 3.5 22 | 23 | ** 3.63 24 | #+begin_quote 25 | “Write a short program that prints each number from 1 to 100 on a new line. 26 | 27 | For each multiple of 3, print "Fizz" instead of the number. 28 | 29 | For each multiple of 5, print "Buzz" instead of the number. 30 | 31 | For numbers which are multiples of both 3 and 5, print "FizzBuzz" instead of the number.” 32 | #+end_quote 33 | 34 | #+begin_quote 35 | "Define a def flexibleFizzBuzz method that takes a String => Unit callback function as its argument, and allows the caller to decide what they want to do with the output. The caller can choose to ignore the output, println the output directly, or store the output in a previously-allocated array they already have handy.” 36 | #+end_quote 37 | 38 | #+BEGIN_SRC scala 39 | def fizzbuzz(n: Int): String = 40 | if (n % 3 == 0 && n % 5 == 0) "FizzBuzz" 41 | else if (n % 3 == 0) "Fizz" 42 | else if (n % 5 == 0) "Buzz" 43 | else n.toString() 44 | 45 | @main 46 | def main() { 47 | (1 to 100) 48 | .map(fizzbuzz) 49 | .foreach(println) 50 | } 51 | #+END_SRC 52 | 53 | ** 3.64 54 | 55 | #+begin_quote 56 | “Write a recursive method printMessages that can receive an array of Msg class instances, each with an optional parent ID, and use it to print out a threaded fashion. That means that child messages are print out indented underneath their parents, and the nesting can be arbitrarily deep.” 57 | #+end_quote 58 | 59 | - I assume that the items are ordered by their identifier which is incremental with each /new/ message. 60 | 61 | #+BEGIN_SRC scala 62 | class Msg(val id: Int, val parent: Option[Int], val txt: String) 63 | 64 | def printMessages(messages: Array[Msg]) { 65 | val padding = 2 66 | def printMessages(index: Int, msgLevel: Map[Int, Int]) { 67 | if (index < messages.length) { 68 | val message = messages(index) 69 | val level = message.parent match { 70 | case Some(parent) => msgLevel.getOrElse(parent, 0) + 1 71 | case None => 0 72 | } 73 | 74 | println(" " * level * padding + message.txt) 75 | printMessages(index + 1, msgLevel + (message.id -> level)) 76 | } 77 | } 78 | 79 | printMessages(index = 0, msgLevel = Map()) 80 | } 81 | 82 | printMessages(Array( 83 | new Msg(0, None, "Hello"), 84 | new Msg(1, Some(0), "World"), 85 | new Msg(2, None, "I am Cow"), 86 | new Msg(3, Some(2), "Hear me moo"), 87 | new Msg(4, Some(2), "Here I stand"), 88 | new Msg(5, Some(2), "I am Cow"), 89 | new Msg(6, Some(5), "Here me moo, moo") 90 | )) 91 | #+END_SRC 92 | 93 | #+RESULTS: 94 | : Hello 95 | : World 96 | : I am Cow 97 | : Hear me moo 98 | : Here I stand 99 | : I am Cow 100 | : Here me moo, moo 101 | : defined class Msg 102 | : defined function printMessages 103 | -------------------------------------------------------------------------------- /org/heartbeat.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578d3 3 | :END: 4 | #+TITLE: Heartbeat 5 | #+filetags: protocols 6 | 7 | In order to detect failures, systems communicate with one another to verify communication. If the communication is deemed broken, then 8 | the system may be considered as ~Unreachable~ depending on he heartbeat history and how the Failure Detection is configured. I.e., a single heartbeat does not mean that the member is ~Unreachable~. 9 | -------------------------------------------------------------------------------- /org/interview_questions.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: c67c6089-ecab-4cc1-94a9-d671760b8dde 3 | :END: 4 | #+title: Interview Questions 5 | #+filetags: 6 | 7 | * Behaviour Questions 8 | 9 | Follows a snippet made by Michael Kusters (link): 10 | #+begin_src 11 | Interviewer: "Can you give us an example of a situation where you had to resolve a conflict?" 12 | Candidate: "What does your management do to create a friendly environment where conflicts are easy to address and resolve?" 13 | 14 | Interviewer: "Could you tell us about a situation where you had to deal with a difficult coworker?" 15 | Candidate: "Do I have to worry about difficult coworkers here?" 16 | 17 | Interviewer: "How do you perform under pressure?" 18 | Candidate: "Since you're HR: what are you doing to reduce pressure?" 19 | 20 | Interviewer: "Tell us about your greatest weakness." 21 | Candiate: "I really love working with people where teamwork is understood as building upon each others' strengths. How do you build teams?" 22 | 23 | Interviewer: "What will you contribute to this company?" 24 | Candidate: "I love doing a great job and bringing my talents in. How do you define, doing a great job - and which of my talents do you currently need most?" 25 | 26 | Interviewer: "Why do you want to work here?" 27 | Candidate: "I'm here to find out. What do you think makes this place better than the competitors?" 28 | 29 | Interviewer: "Why do you want to leave your current company?" 30 | Candidate: "What's the main reason why people leave your company?" 31 | #+end_src 32 | 33 | I find it it interesting as it may provide further insights about the companies culture. We should be prepared to continue the conversation if we really want to challenge the interviewer's questions. 34 | -------------------------------------------------------------------------------- /org/jackson.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578d4 3 | :END: 4 | #+TITLE: Jackson 5 | #+filetags: jvm json 6 | 7 | Jackson is a library to serialize and deserialize JSON in the JVM world. 8 | 9 | * Sane Settings 10 | 11 | After working a while with this I want to register these sane defaults: 12 | #+BEGIN_SRC kotlin 13 | configure(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES, true) 14 | configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) 15 | #+END_SRC 16 | 17 | ** ~FAIL_ON_NULL_FOR_PRIMITIVES~ 18 | 19 | Setting ~FAIL_ON_NULL_FOR_PRIMITIVES~ forces clients to explicitely provide all values including primitives. Consider the following POJO: 20 | #+BEGIN_SRC kotlin 21 | data class Foo(bar: Boolean) 22 | #+END_SRC 23 | 24 | Without the setting, a payload such as ~{ }~ would render ~~Foo(bar=false)~~ despite the lack of default value. Given this, to guarantee consistency between the source-code and the external contract, I advise enabling this. 25 | 26 | ** ~FAIL_ON_UNKNOWN_PROPERTIES~ 27 | 28 | Setting ~FAIL_ON_UNKNOWN_PROPERTIES~ is useful when working on two systems in paralel. Settings this to ~false~ enables clients to send fields to the server that are not yet supported but will be. The alternative would be: 29 | 1. Server includes those fields as *optional* fields (to avoid breaking current clients). 30 | 2. Server roll-out. 31 | 3. Clients update their HTTP clients to include the new fields. 32 | 4. Once all-known clients support the new fields, make the same fields mandatory. 33 | 34 | By setting this to ~true~, this whole orchestration is not required. 35 | 36 | * TODO Sub-Types 37 | 38 | Consider the following example that attempts to model a DSL that supports ~+~ and ~-~ operations. 39 | 40 | #+BEGIN_SRC kotlin 41 | sealed class Expression 42 | data class Sum(val a: Int, val b: Int): Expression() 43 | data class Sub(val a: Int, val b: Int): Expression() 44 | 45 | data class Request( 46 | @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.EXTERNAL_PROPERTY, property = "type", visible = true) 47 | @JsonSubTypes( 48 | JsonSubTypes.Type(value = Sum::class, name = "+") 49 | JsonSubTypes.Type(value = Sub::class, name = "-") 50 | ) 51 | val operation: Expression 52 | ) 53 | #+END_SRC 54 | 55 | In short, this is saying if the JSON contains a field name ~type~ with value ~+~ it would deserialize to ~Sum~ and to ~Sub~ if ~type~ had value ~-~. I.e., a sum operation between 1 and 2 would require the following JSON payload: 56 | 57 | #+BEGIN_SRC json 58 | { 59 | "type": "+", 60 | "operation": { 61 | "a": 1, 62 | "b": 2 63 | } 64 | } 65 | #+END_SRC 66 | -------------------------------------------------------------------------------- /org/json.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: db01a80a-e0f1-4068-b3cf-20008af86be1 3 | :END: 4 | #+title: Json Processing Cookbook 5 | #+filetags: json data jq 6 | 7 | * Flatten Json 8 | 9 | #+begin_src bash 10 | jq -r '. | tostream 11 | | select(length==2) 12 | | (.[0] | join(".")) as $k 13 | | .[1] as $v 14 | | "\"\($k)\" = \"\($v)\";"' 15 | #+end_src 16 | -------------------------------------------------------------------------------- /org/kotlin.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578d5 3 | :END: 4 | #+TITLE: Kotlin 5 | #+filetags: snippets kotlin 6 | 7 | * Asserting asynchronous task 8 | 9 | Sometimes ~CountdownLatch~ is not feasible as it is not possible to inject it. The alternative pool until the system reaches the desired state. This approach is not ideal as it may block the system longer than strictly required. 10 | 11 | As mid-term solution, in any-case opt for ~CountdownLatch~ if possible. 12 | 13 | #+BEGIN_SRC kotlin 14 | fun assertWaitFor(timeoutMs: Long = 5000, intervalMs: Long = 250, fn: () -> Unit) { 15 | val start = Instant.now() 16 | var lastError: Throwable? = null 17 | while (Duration.between(start, Instant.now()).toMillis() <= timeoutMs) { 18 | try { 19 | fn() 20 | return 21 | } catch (ex: AssertionError) { 22 | lastError = ex 23 | } 24 | Thread.sleep(intervalMs) 25 | } 26 | 27 | fail("Failed to reach expected condition within $timeoutMs ms.", lastError!!) 28 | } 29 | #+END_SRC 30 | -------------------------------------------------------------------------------- /org/linux.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 4331316e-00b1-44fd-9c56-466de9122b70 3 | :END: 4 | #+title: linux 5 | #+filetags: 6 | 7 | How to format: 8 | #+begin_src 9 | ``` 10 | $ sudo mkfs.ext4 -L data /dev/sdb1 11 | ``` 12 | #+end_src 13 | 14 | If having problems writing to disk, review permissions: 15 | #+begin_src 16 | sudo chmod -R a+rwX /mnt/data 17 | #+end_src 18 | -------------------------------------------------------------------------------- /org/message_driven_architecture.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578d6 3 | :END: 4 | #+TITLE: Message Driven Architecture 5 | #+filetags: system-design 6 | 7 | Asyncronous and non-blocking. The sender does not actively wait for a response. 8 | 9 | Advantages: 10 | - Resources are freed immediatly. 11 | - Reduced contention 12 | - Messages can be queued for deleivery in case the receiver's is offline. Provides a higher level of reliability. 13 | 14 | Disavantages: 15 | - Make transactions more difficult. 16 | - How to manage long running transactions that span multiple microservices. 17 | - Holding transactions open for long periods result in slow, brittle systems. 18 | 19 | The role of syncronous messags: 20 | - Can you acknowledge the message but process it asyncronously? 21 | - The need for syncronous messages should be driven by domain requirements rather than technical convenience. 22 | 23 | * Sagas 24 | 25 | Represent long running transaction. Multiple requests are managed by a Saga. 26 | 27 | Individual requests are run in sequence or in paralel. 28 | 29 | How: 30 | - Each request is paired with a compensating action. 31 | - If any requests fails, compnesation actions are executed for all completed steps. 32 | - Then the saga is completed with a failure. 33 | - If compensation actions fails, then it is retried. This requires idempotency. 34 | 35 | 36 | Timeouts in distributed systems: 37 | - Eitehr the request failed. 38 | - Either it was successful but the reply failed. 39 | - The request is still queued and may success or fail eventually. 40 | 41 | Compensating actions != Rollbacks. 42 | 43 | Rollback: implies the transaction has not completed which removes the evidence of the transaction. 44 | Compensation: Applied on top of a previously completed action. Evidence of the orignal action remains. 45 | 46 | Disavantages: 47 | - Coupled to the failures unless we can move regardless of that. 48 | - Saga are often implemented using ackka actors and represented via Finite State Machine. 49 | 50 | * Two General Problem 51 | 52 | Illustrate the impossibility of reaching a concensus over an unreliable communication channel. 53 | 54 | * Delivery Guarantees 55 | 56 | - Exactly Once: Is not possible in the event of a network partition, or lost message. We never guarantee that the message was in-fact sent. Failure requires resending the message which creates /potential/ duplicates. Reuqires storage on both ends: unreliable network (as always); timeouts. 57 | - At most once - If a failure occur, no retries are done which means no duplications but there may be losses. Requires no storage of messages. 58 | - At least once - Require a acknoledge and teh sender needs to store state to track whether the messsage was acknowledge. It ahs to be stored in a durable data store. 59 | 60 | Exactly once can be simulated using at least once and idempotency. 61 | 62 | Akka: at most once by default. Akka persistence has option to have at least once. 63 | 64 | * Messaging Patterns 65 | 66 | ** Publish Subscribe 67 | 68 | Decoupled. The only coupling is on the message format and possibily the location (e.g., url, exchange on the message broker). Complexity is hard to see as we do not know where the message comes from. 69 | 70 | ** Point to point 71 | 72 | Dependencies more clear but coupling is higher. COmplexity is directly observable. 73 | 74 | ** Examples 75 | 76 | Kafaka, RabbitMQ. Kafka allows point to point and pub/sub and we can even acknowledge once we finish processing. 77 | 78 | Akka: Typically point to point messaging;Persistence Query: Pub/sub 79 | Lagom: Point to point communication between services. Messages broker API allow for pub/sub. 80 | -------------------------------------------------------------------------------- /org/microservices.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578d7 3 | :END: 4 | #+TITLE: Microservices 5 | #+filetags: system-design 6 | 7 | Subset of Service Oriented Architecture (SOA) where each service is deployed separately: 8 | - Microservices can be physically separated and independently deployed. 9 | - Each have its own data store. 10 | - Independent and self governing. 11 | - Communication is syncronous or asyncronous (e.g., through messaging systems). 12 | - Loose coupling between components (more or less by experience but that is design flaw likely :thinking:). 13 | - Shorter development and release cycles. 14 | - Each scale independently (either through physical or virtual machines). 15 | 16 | * Advantages 17 | 18 | + Deployed/Scaled as needed. 19 | + Increase availability due to reduced single point of failures. 20 | + Better isolation leading to less couling giving flexibility. 21 | + Supports multiple platforms and languages. 22 | + More indepentent 23 | + Shorter cycles 24 | 25 | Mention that cross team coordination become less necessary. That is true but often it still is and requires coordination. 26 | 27 | * Disavantages 28 | - Require multiple complex deployment nad monitoring approaches. 29 | - Cross service refactors are more challenging. 30 | - Require supporting old versions. 31 | - Organization Change 32 | 33 | * Designing one 34 | 35 | Business domains are typically large and complicated. Several ideas, actions and rules interact with one another in complex ways. A good approach on understanding such complex systems and how to model them is through a process named [[id:2fa03d4f-948e-4a6e-a38b-178456b578c5][Domain Driven Design]]. 36 | 37 | A correct separation of concerns (i.e., clear and single responsibilities) leads to smoother workflows. Awkarwdness when modelling may indicate lack of understanding of the domain. 38 | 39 | * Principles of Isolation 40 | 41 | ** State 42 | 43 | All access must go thorugh the API and there is no backdoors. Allows internal changes without affecting others. 44 | 45 | ** Space 46 | 47 | Should not care where the other services are deployed. Allows microservices to be scaled up or down to meet demands. However, if latency is an issue, they should be in the same region. 48 | 49 | ** Time 50 | 51 | Should not wait for each other which allows more efficient use of resources. Resources can be freed immediatly, rather than waiting for a request to finish. 52 | 53 | Between microservices we expect eventual consistency. It improves scalability as total consitency requires central coordination which hindes scalability. 54 | 55 | NOTE: This is discussible. However it may be a implementation details. 56 | 57 | ** Failure 58 | 59 | Isolate failures and should not cause failure on other systems. I.e., it still is responsive to attend other use-cases. 60 | 61 | * Isolation Techniques 62 | 63 | ** Bulkheading 64 | 65 | Failures are isolated to failure zones. Failures in one service will not propagate to other services. Overall the system remains operation but possibly in a degraded state. 66 | 67 | In practice it means that that systems that depend on the service that is considered a failure zone, will mark that information or service as unavailable. IMO this is tolerable if the service is non-critical. 68 | 69 | ** Circuit Breaker 70 | 71 | When a service is under stress we do not want to keep on retrying as it may make things worse. 72 | 73 | Way to avoid overloading a service. They qaurantine a failing service so it can /fail fast/. Allows the failing service to recover in its time before they fail. 74 | 75 | Types: 76 | - Closed - Normal Operation 77 | - Open - Fail fast 78 | - Half Open - Internally after a timeout will let one request to go through and if it fails, it goes back to Open. 79 | 80 | Transitions: 81 | - Closed (normal) -> Trip: Open (Fail Fast) 82 | - Open (Fail fast) -> Attempt Reset: Half Open 83 | - Half Open -> Trip: Open (fail Fast) 84 | - Half Open -> Reset: Closed (Normal) 85 | 86 | ** Message Driven Architecture 87 | 88 | - Async and non blocking messages allows decoupling both time and failure. 89 | - System do not depend on the response from on another. 90 | - If a request to a service fails, the failure will not propagated. 91 | - The client service isn't waiting for response. It can continue to operate normally. 92 | 93 | ** Autonomy 94 | 95 | Services can operate independently from each other. 96 | 97 | Autonomous services have enough information to resolve conflicts and repair failures. This means that they do not require other services to be operational all the time. Ideally all the time but in reality for a short time it guarantees some level of autonomy. 98 | 99 | Benefits: 100 | - Stronger scalability and availability. 101 | - Can be scaled indefinetly. 102 | - Operate independently. 103 | 104 | How: 105 | - Async messages. 106 | - Maintain enough internal state ofr the microservices to function in isolation. 107 | - Use eventual consistency. 108 | - Avoid direct, syncronous dependencies on external services. 109 | 110 | ** API Gateway Services 111 | 112 | Microservices can lead to complexity in the API. How can we manage complex APIs that access many microservices? 113 | 114 | A new microservice that is put between the client and the N-services that are required to fulfill that request. This new microservice is responsible for aggregating the responses moving the responsbility from the client itself. This way, the client *only* needs to manage failures from the gateway. 115 | 116 | This effectively creates an additional layer of isolation. 117 | 118 | !! This is specially useful for mobile applicatios where we cannot guarantee that the clients will update their app. 119 | -------------------------------------------------------------------------------- /org/monolith.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578d8 3 | :END: 4 | #+TITLE: Monolith 5 | #+filetags: system-design 6 | 7 | * Characteristics 8 | 9 | - Deployed as a single unit. 10 | - No Clear Isolation. 11 | - Complex Dependencies which in turn makes it hard to understand and modify. 12 | - Big Bang Style Releases 13 | - Long Cycle Times 14 | - Careful releases 15 | - Scalation is done with multiple copies and uses the database as consistency between them. 16 | 17 | * Advantages: 18 | - Easy Cross Module Refactor 19 | - Easier to maitain consistency 20 | - Single Deploy Process 21 | - Single thing to monitor 22 | - Simple Scalability Model 23 | 24 | * Disadvantages: 25 | - Limited by the maximum size of a single physical machine. 26 | - Only scales as the database allows. 27 | - Components are scaled as a group. 28 | - Deep coupling. 29 | - Long Dev Cycle. 30 | - Lack of reliability given that one failure may impact the whole monolith. 31 | 32 | * Tearing it up 33 | 34 | Introduce domain boundaries within the application itself (e.g., libraries). This falls within Service Oriented Architecture (SOA). 35 | -------------------------------------------------------------------------------- /org/nix.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578d9 3 | :END: 4 | #+TITLE: Nix 5 | #+filetags: nix dotfiles flakes 6 | 7 | [[https://nixos.org/][Nix]] is a declarative language aiming produce reproducible systems. In can be used to produce ~dotfiles~. 8 | 9 | Follows three tools: 10 | - [[https://github.com/nix-community/home-manager][Home Manager]]: Manages user's home. It can't be used to install fonts for example. 11 | - [[https://github.com/LnL7/nix-darwin/][Nix Darwin]]: Manages macOS systems. 12 | 13 | The goal is to compose these tools to produce a reproducible /generation/ of one's environment. To aid this, [[https://nixos.wiki/wiki/Flakes][Nix Flakes]] introduces another layer on top that aims to pin the versions the dependencies through a ~flake.lock~ file. 14 | 15 | My [[https://github.com/bphenriques/dotfiles][dotfiles]] leverage Nix. 16 | -------------------------------------------------------------------------------- /org/postgresql.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 9535c94a-83b7-4dda-abc0-1aa524842fbd 3 | :END: 4 | #+title: PostgreSQL 5 | #+filetags: psql postgres docker docker-compose 6 | 7 | Relational database. 8 | 9 | One can run in a docker-compose environment like this: 10 | #+begin_src yaml 11 | version: '3.8' 12 | 13 | services: 14 | postgres: 15 | image: postgres:13.3 # Version must be consistent with infrastructure 16 | environment: 17 | - "PGUSER=test" 18 | - "POSTGRES_USER=test" 19 | - "POSTGRES_PASSWORD=test" 20 | - "POSTGRES_DB=test" 21 | ports: 22 | - "5432:5432" 23 | healthcheck: 24 | test: ["CMD-SHELL", "pg_isready"] 25 | interval: 5s 26 | timeout: 5s 27 | retries: 5 28 | #+end_src 29 | -------------------------------------------------------------------------------- /org/reactive_streams.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578e0 3 | :END: 4 | #+TITLE: Reactive Streams 5 | #+filetags: system-design 6 | 7 | Components of a Reactive Stream: 8 | - *Publisher*: Publishes data to stream 9 | - *Subscriber*: Consumes data from the stream. 10 | - *Processor*: Acts as both a publisher and a subscriber, obeying the contract for each. 11 | - *Subscription*: Connects a subscriber to a publisher to initiate a message flow. 12 | 13 | [[id:2fa03d4f-948e-4a6e-a38b-178456b578b7][Akka Streams]] is built on these concepts but provides a different API (albeit is possible to bridge). 14 | -------------------------------------------------------------------------------- /org/reactive_systems.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578e1 3 | :END: 4 | #+TITLE: Reactive Systems 5 | #+filetags: system-design 6 | 7 | * Goal 8 | 9 | Provide an experience that is responsive under all conditions. Note that reactive programming is not the same as reactive systems. 10 | 11 | This requires: 12 | - Ability to scale from 10 users to million of users. 13 | - Consume _solely_ the resources required to support the current work-load. 14 | 15 | * Reactive Principles 16 | 17 | Systems that apply the following principles are considered reactive systems (see more [[https://github.com/reactivemanifesto/reactivemanifesto][here]]). 18 | 19 | ** Responsive 20 | 21 | Always respond in a timely manner. 22 | 23 | ** *Resiliency* 24 | 25 | Isolate failures on single components - Similar to how a boat is designed. 26 | 27 | ** *Elastic* 28 | 29 | - Keep responsive specially when the system load changes which provides a more efficient usage of resources. 30 | - Reduce contentions. 31 | - Avoid central bottlenecks. 32 | - Predictive Auto-Scaling policies. 33 | 34 | ** Message Driven 35 | 36 | - Losse coupling 37 | - Isolatation - Kinda disagree with loose coupling, as the systems _still_ will still depend on 3rd party behavior regardless of the communication medium. 38 | - Resources are only consumed while active 39 | 40 | Avoce all, make sure that the systems is not actively waiting so that it can do something else in the mean time which leads to a *Async and non-blocking* messages. 41 | 42 | * Git as example 43 | 44 | - Asyncronous and non-blocking as the work is submitted through PR and I can keep on working locally with no interruptions. Message based as it is basically "please review this". 45 | - Resiliency as each user has basically a copy of the whole repository locally. The local machine is isolated from the remote. 46 | - Elastic has we can have multiple copies and does not cause problemas having that repository sync on that many machines. 47 | - Responsive. 48 | 49 | 50 | * TODO https://www.lightbend.com/white-papers-and-reports/reactive-programming-versus-reactive-systems 51 | -------------------------------------------------------------------------------- /org/referential_transparency.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: ced9d57d-5884-4932-8753-3f9ca593f7f8 3 | :END: 4 | #+title: Referential Transparency 5 | #+filetags: functional-programming scala 6 | 7 | Referential transparency: replacing an expression by its bound value doesn't alter the behaviour of your program. This gives huge benefits, which I'll talk about later. 8 | 9 | 10 | * References 11 | - https://www.reddit.com/r/scala/comments/8ygjcq/can_someone_explain_to_me_the_benefits_of_io/ 12 | - https://www.youtube.com/watch?v=x3GLwl1FxcA&t=189s 13 | -------------------------------------------------------------------------------- /org/scala_experiments.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578e2 3 | :END: 4 | #+TITLE: Scala Experiments 5 | #+filetags: scala data-structures 6 | 7 | * Implementing ~Tree~ 8 | 9 | #+begin_src scala :eval never-export 10 | sealed class Tree[T] { 11 | def stream(): LazyList[T] = this match { 12 | case Leaf(value) => LazyList(value) 13 | case Branch(left, right) => left.stream() #::: right.stream() 14 | } 15 | } 16 | case class Branch[T](left: Tree[T], right: Tree[T]) extends Tree[T] 17 | case class Leaf[T](value: T) extends Tree[T] 18 | 19 | val tree = 20 | Branch( 21 | Leaf(1), 22 | Branch( 23 | Leaf(2), 24 | Leaf(3) 25 | ) 26 | ) 27 | 28 | val nextInOrder = tree.stream().dropWhile(_ != 2).drop(1).headOption 29 | println(nextInOrder) 30 | #+end_src 31 | -------------------------------------------------------------------------------- /org/sharding_or_partitioning.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578e3 3 | :END: 4 | #+TITLE: Sharding or Partitioning 5 | #+filetags: databases system-design 6 | 7 | Technique used by some data stores to reduce contention without sacrificing consistency. 8 | 9 | Records are distributed across nodes using a /Shard Key/ or a /Partition Key/ that will be used by the /Database Router/ that redirects requests to the correct shard/partition. 10 | 11 | Benefits: 12 | - Contention is isolated to a shard/partition. 13 | - Given that each shard stores a part of the dataset, it is only handling a small part of the overall load. 14 | - Improves elasticity. 15 | 16 | However, with time the database will still be a bottleneck because there may be too many users acesssing the shards/partitions (sometimes it may be a bad choice of partition key). 17 | -------------------------------------------------------------------------------- /org/sli_slo.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578e4 3 | :END: 4 | #+TITLE: SLI/SLO 5 | #+filetags: system-design 6 | 7 | - *Service Level Indicator (SLI)*: Performance indicator measured as a ratio of two numbers. 8 | - *Service Level Objective (SLO)*: Defines a target SLI as measurment of the systems's reliability. 9 | - *Service Level Agreement (SLA)*: Business contract regarding the expected SLO. 10 | 11 | It is far productive measuring them as use-histories as it defines the critical paths. 12 | -------------------------------------------------------------------------------- /org/stateless.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578e5 3 | :END: 4 | #+TITLE: Stateless 5 | #+filetags: system-design 6 | 7 | Stateless systems means that: 8 | - The system contains all the required information to complete requests. 9 | - Requests can be processed on any instance of of the application. 10 | 11 | Note that, some "stateless" systems can't be considered as such as the state is contained in a database. 12 | -------------------------------------------------------------------------------- /org/states-of-data.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578e6 3 | :END: 4 | #+TITLE: States of Data 5 | #+filetags: data 6 | 7 | States of data: 8 | - *At Rest*: Data that is not consumed at the time is ingested. It is stored and then consumed later in a batch process. 9 | - *In Transit*: Data that is travelling between point A and point B. 10 | - *In Use*: Data that is opened for treatment 11 | 12 | Reference: [[https://www.sealpath.com/protecting-the-three-states-of-data/][three-states-of-data]] 13 | -------------------------------------------------------------------------------- /org/thinking_tools.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578e7 3 | :END: 4 | #+TITLE: Thinking Tools 5 | #+filetags: work 6 | 7 | Follows some interesting resources: 8 | - https://untools.co/ 9 | -------------------------------------------------------------------------------- /org/using_psql_as_job_queue.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578e8 3 | :END: 4 | #+TITLE: Using PSQL as job queue 5 | #+filetags: psql queue 6 | #+HUGO_SECTION: posts 7 | 8 | Use-case: Fault Tolerant event queue in Postgres. 9 | 10 | Consider the following table: 11 | #+BEGIN_SRC SQL 12 | DROP TABLE IF EXISTS "queue"; 13 | CREATE TABLE "queue" ( 14 | "id" SERIAL, 15 | "dequeued_at" TIMESTAMP NULL, 16 | 17 | -- remaining fields relevant for the use-case. For the outbox pattern, one would be storing the event in JSON format, the event id and the event's timestamp. 18 | ); 19 | #+END_SRC 20 | 21 | This table represent a queue in the database. In order to fetch ~size~ elements from the queue: 22 | 23 | #+BEGIN_SRC sql 24 | UPDATE event_queue SET dequeued_at = current_timestamp 25 | WHERE id IN ( 26 | SELECT id FROM queue 27 | WHERE dequeued_at IS NULL OR current_timestamp >= (dequeued_at + interval '1 second' second * :seconds) 28 | ORDER BY event_timestamp ASC 29 | LIMIT :size 30 | FOR UPDATE SKIP LOCKED 31 | ) 32 | RETURNING * 33 | #+END_SRC 34 | 35 | Breaking down: 36 | - ~UPDATE event_queue SET dequeued_at = current_timestamp + ~FOR UPDATE SKIP LOCKED~ - Guarantees that multiple nodes reading N elements from the queue will have disjoint sets. 37 | - ~WHERE dequeued_at IS NULL~ - Check if the item was already or not dequeued. 38 | - ~current_timestamp >= (dequeued_at + interval '1 second' second * :seconds)~ - Guarantees that multiple nodes will not process the same elements. Does ~SKIP LOCKED~ already safe-guards this? 39 | -------------------------------------------------------------------------------- /org/way-of-work.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b578e9 3 | :END: 4 | #+TITLE: Way of work 5 | #+filetags: work 6 | 7 | * It's not about being right nor prove others wrong 8 | 9 | - *Corollary 1*: if it's wrong but it works, then it's not wrong. 10 | - *Corollary 2*: if you're right but it doesn't change the outcome, then it doesn't matter. 11 | - *Corollary 3*: if you're right, but it doesn't work, then you're wrong. 12 | - *Corollary 4*: if you prove someone else wrong, but their answer works and yours doesn't, then they're right and you're wrong. 13 | - *Corollary 5*: if you prove someone's solution to be wrong even though it does provide value, then you have not yet provided any value until you propose something better. 14 | 15 | * Balancing Work/Life 16 | 17 | There is much in life beyond working. Follows an interesting [[https://1x.engineer/][article]] that contrasts with the tendency. 18 | -------------------------------------------------------------------------------- /org/web-stack-enties.org: -------------------------------------------------------------------------------- 1 | :PROPERTIES: 2 | :ID: 2fa03d4f-948e-4a6e-a38b-178456b579a0 3 | :END: 4 | #+TITLE: Resources to (re)-read 5 | #+filetags: reference 6 | 7 | * Things I Wished More Developers Knew About Databases 8 | 9 | Source: https://rakyll.medium.com/things-i-wished-more-developers-knew-about-databases-2d0178464f78 10 | 11 | * How To Drive Change as a Software Engineer 12 | 13 | Source: https://www.lihaoyi.com/post/HowToDriveChangeasaSoftwareEngineer.html 14 | 15 | * The Dark Side of Events - YouTube 16 | 17 | Source: https://www.youtube.com/watch?v=URYPpY3SgS8&feature=youtu.be&t=1884 18 | 19 | * Ask HN: How to Take Good Notes? | Hacker News 20 | 21 | Source: https://news.ycombinator.com/item?id=22473209 22 | 23 | * Unlearning toxic behaviors in a code review culture | by Sandya Sankarram | Medium 24 | 25 | Source: https://medium.com/@sandya.sankarram/unlearning-toxic-behaviors-in-a-code-review-culture-b7c295452a3c 26 | 27 | * Nicolas Mattia – Nix: A Reproducible Setup for Linux and macOS 28 | 29 | Source: https://www.nmattia.com/posts/2018-03-21-nix-reproducible-setup-linux-macos.html 30 | 31 | * Write code that is easy to delete, not easy to... — programming is terrible 32 | 33 | Source: https://programmingisterrible.com/post/139222674273/how-to-write-disposable-code-in-large-systems 34 | 35 | * The Product-Minded Software Engineer - The Pragmatic Engineer 36 | 37 | Source: https://blog.pragmaticengineer.com/the-product-minded-engineer/ 38 | 39 | * Remembering what you Read: Zettelkasten vs P.A.R.A. 40 | 41 | Source: https://www.zainrizvi.io/blog/remembering-what-you-read-zettelkasten-vs-para/ 42 | 43 | * degoogle | A huge list of alternatives to Google products. Privacy tips, tricks, and links. 44 | 45 | Source: https://degoogle.jmoore.dev/#useful-links-tools-and-advice 46 | 47 | * How to teach yourself hard things 48 | 49 | Source: https://jvns.ca/blog/2018/09/01/learning-skills-you-can-practice/ 50 | 51 | -------------------------------------------------------------------------------- /shell.nix: -------------------------------------------------------------------------------- 1 | { pkgs }: 2 | 3 | with pkgs; 4 | mkShell { 5 | buildInputs = [hugo]; 6 | } 7 | -------------------------------------------------------------------------------- /static/.gitignore: -------------------------------------------------------------------------------- 1 | ox-hugo/ 2 | -------------------------------------------------------------------------------- /static/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bphenriques/knowledge-base/0490478c2ce82c257883eed1b6e6468fc66184f5/static/favicon.png -------------------------------------------------------------------------------- /tools/init.el: -------------------------------------------------------------------------------- 1 | ;;; build.el --- Minimal emacs installation to build the website -*- lexical-binding: t -*- 2 | ;; Authors: Bruno Henriques <4727729+bphenriques@users.noreply.github.com> 3 | 4 | ;;; Commentary: 5 | ;;; - Requires KNOWLEDGE_BASE_DIR environment variable 6 | ;; 7 | ;; Notes: 8 | ;; * This is not a package. 9 | ;;; Code: 10 | 11 | (require 'subr-x) 12 | 13 | (toggle-debug-on-error) ;; Show debug information as soon as error occurs. 14 | (setq 15 | make-backup-files nil ;; Disable "~" backups. 16 | ) 17 | 18 | ; Force emacs directory as setting the HOME beforehand is not enough. 19 | (setq user-init-file (or load-file-name (buffer-file-name))) 20 | (setq user-emacs-directory (file-name-directory user-init-file)) 21 | 22 | (defconst knowledge-base-dir 23 | (let* ((env-key "KNOWLEDGE_BASE_DIR") 24 | (env-value (getenv env-key))) 25 | (if (and env-value (file-directory-p env-value)) 26 | env-value 27 | (error (format "%s is not set or is not an existing directory (%s)" env-key env-value))))) 28 | 29 | ;;; 30 | ;;; Setup packages using straight.el: https://github.com/raxod502/straight.el 31 | ;;; 32 | ;;; Leads to better reproducible builds as the versions are pinned. 33 | (defvar bootstrap-version) 34 | (let ((bootstrap-file 35 | (expand-file-name "straight/repos/straight.el/bootstrap.el" user-emacs-directory)) 36 | (bootstrap-version 5)) 37 | (unless (file-exists-p bootstrap-file) 38 | (with-current-buffer 39 | (url-retrieve-synchronously 40 | "https://raw.githubusercontent.com/raxod502/straight.el/develop/install.el" 41 | 'silent 'inhibit-cookies) 42 | (goto-char (point-max)) 43 | (eval-print-last-sexp))) 44 | (load bootstrap-file nil 'nomessage)) 45 | 46 | (setq straight-use-package-by-default t) 47 | (straight-use-package 'use-package) 48 | 49 | ;;; 50 | ;;; Setup packages 51 | ;;; 52 | 53 | (use-package ox-hugo 54 | :straight (:type git :host github :repo "kaushalmodi/ox-hugo")) 55 | 56 | (setq 57 | org-hugo-base-dir knowledge-base-dir 58 | org-hugo-section "notes" 59 | org-id-extra-files (directory-files-recursively knowledge-base-dir "\.org$")) 60 | 61 | ;;; 62 | ;;; Public functions 63 | ;;; 64 | 65 | (defun build/export-all () 66 | "Export all org-files (including nested) under knowledge-base-org-files." 67 | (let ((search-path (concat (file-name-as-directory knowledge-base-dir) "org/"))) 68 | (message (format "[build] Looking for files at %s" search-path)) 69 | (dolist (org-file (directory-files-recursively search-path "\.org$")) 70 | (with-current-buffer (find-file org-file) 71 | (message (format "[build] Exporting %s" org-file)) 72 | (org-hugo-export-wim-to-md :all-subtrees nil nil nil))) 73 | (message "Done!"))) 74 | 75 | (provide 'build/export-all) 76 | 77 | ;;; init.el ends here 78 | -------------------------------------------------------------------------------- /tools/versions/default.el: -------------------------------------------------------------------------------- 1 | ; Must be consistent with: https://github.com/bphenriques/dotfiles/blob/master/emacs/.config/doom/packages.el 2 | ( 3 | ; https://github.com/kaushalmodi/ox-hugo 4 | ("ox-hugo" . "4cc8de4ec5f87bfa7fca97d61ae445ea0a75ae1e")) 5 | :beta 6 | --------------------------------------------------------------------------------