├── architecture ├── core_system │ ├── database │ │ └── url_service │ │ │ ├── cache.io │ │ │ └── database.io │ └── container.puml ├── analytics_system │ ├── database │ │ └── analytics_service │ │ │ ├── cache.io │ │ │ └── database.io │ └── container.puml └── context.puml ├── README.md ├── api └── rest_api.yml └── images └── diagrams ├── context.svg └── containers ├── core_system.svg └── analytics_system.svg /architecture/core_system/database/url_service/cache.io: -------------------------------------------------------------------------------- 1 | Table popular_url { 2 | id string [note: 'Tiny url id'] 3 | long_url string [not null, unique, note: 'Long url'] 4 | } -------------------------------------------------------------------------------- /architecture/analytics_system/database/analytics_service/cache.io: -------------------------------------------------------------------------------- 1 | Table aggregated_url_click { 2 | tiny_url_id string [not null, note: 'Tiny url id'] 3 | count integer [not null, note: 'Count of the clicks'] 4 | } -------------------------------------------------------------------------------- /architecture/core_system/database/url_service/database.io: -------------------------------------------------------------------------------- 1 | // Replication: 2 | // - master-slave (one sync + async) 3 | // - replication factor 3 4 | // 5 | // Sharding: 6 | // - key based by id 7 | 8 | Table url { 9 | id string [primary key, note: 'Tiny url id'] 10 | long_url string [not null, unique, note: 'Long url'] 11 | created_at timestamp [not null, note: 'Creation time'] 12 | } -------------------------------------------------------------------------------- /architecture/analytics_system/database/analytics_service/database.io: -------------------------------------------------------------------------------- 1 | // Replication: 2 | // - master-slave (async) 3 | // - replication factor 3 4 | // 5 | // Sharding: 6 | // - key based by tiny_url_id 7 | 8 | Table url_click { 9 | id id [primary key, note: 'Activity id'] 10 | tiny_url_id string [not null, note: 'Tiny url id'] 11 | clicked_at timestamp [not null, note: 'Time of the click'] 12 | ip_address int [note: 'IP address of the client'] 13 | device string [note: 'Name of the user device'] 14 | } -------------------------------------------------------------------------------- /architecture/context.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | !include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Container.puml 3 | 4 | Person(user, "User") 5 | Container(coreSystem, "Core system", "Software system", "Handles creation of tiny URL from long URL and redirections from tiny to long URL") 6 | Container(analyticsSystem, "Analytics system", "Software system", "Stores different analytic information about tiny URLs") 7 | 8 | Rel(user, coreSystem, "Creates tiny URL and redirects from tiny to long URL") 9 | Rel(user, analyticsSystem, "Gets different clicks information by tiny URLs") 10 | @enduml -------------------------------------------------------------------------------- /architecture/core_system/container.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | !include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Container.puml 3 | 4 | Person(user, "User") 5 | Container(loadBalancer, "Load Balancer", "Nginx", "Uses round robin") 6 | Container(analyticsSystem, "Analytics system", "Software system", "Stores different analytic information about tiny URLs") 7 | ContainerQueue(messageQueue, "Events queue", "Kafka", "Message queue for events by tiny URLs") 8 | 9 | System_Boundary(coreSystem, "Core system") { 10 | Container(urlService, "URL Service", "Go", "Handles creation tiny URL and redirects to long URL from tiny URL", $tags="webApp") 11 | ContainerDb(urlDatabase, "URL database", "PgSQL", "Stores long URLs by tiny URLs", $tags="db") 12 | ContainerDb(urlCache, "URL cache", "Tarantool", "Stores popular long URLs by tiny URLs", $tags="db") 13 | } 14 | 15 | Rel(user, loadBalancer, "Creates tiny URL and redirects from tiny URL to long URL", "REST") 16 | Rel(loadBalancer, urlService, "Creates tiny URL and redirects from tiny URL to long URL", "REST") 17 | 18 | Rel(urlService, urlDatabase, "Creates tiny URLs and gets long URLs by tiny URLs") 19 | Rel(urlService, urlCache, "Gets and puts least frequently used URLs") 20 | Rel(urlService, messageQueue, "Publishes click events by tiny URLs") 21 | Rel(analyticsSystem, messageQueue, "Subscribes on click events by tiny URLs") 22 | @enduml -------------------------------------------------------------------------------- /architecture/analytics_system/container.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | !include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Container.puml 3 | 4 | Person(user, "User") 5 | Container(loadBalancer, "Load Balancer", "Nginx", "Uses round robin") 6 | ContainerQueue(messageQueue, "Events queue", "Kafka", "Message queue for events by tiny URLs") 7 | Container(coreSystem, "Core system", "Software system", "Stores different analytic information about tiny URLs") 8 | 9 | System_Boundary(analyticsSystem, "Analytics system") { 10 | Container(analyticsService, "Analytics Service", "Go", "Handles requests by clicks information and each hour stores aggregated information about click numbers of each tiny URL to cache", $tags="webApp") 11 | ContainerDb(analyticsDatabase, "Analytics database", "ClickHouse", "Stores analytics information about clicks of each tiny URL", $tags="db") 12 | ContainerDb(analyticsCache, "Analytics cache", "Tarantool", "Stores aggregated information about number of clicks by each tiny URL", $tags="db") 13 | } 14 | 15 | Rel(user, loadBalancer, "Gets clicks information by tiny URLs", "REST") 16 | Rel(loadBalancer, analyticsService, "Gets clicks information by tiny URLs", "REST") 17 | 18 | Rel(coreSystem, messageQueue, "Publishes click events by tiny URLs") 19 | Rel(analyticsService, messageQueue, "Subscribes on click events by tiny URLs") 20 | Rel(analyticsService, analyticsDatabase, "Stores click events and also aggregates information about all clicks by tiny URLs") 21 | Rel(analyticsService, analyticsCache, "Gets clicks number of each tiny URL, also stores information about clicks number of each tiny URL") 22 | @enduml -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Tiny URL - System Design 2 | 3 | Example of the homework for [course by System Design](https://balun.courses/courses/system_design). 4 | TinyURL is a URL shortening web service, which provides short aliases for redirection 5 | of long URLs. 6 | 7 | ### Functional requirements: 8 | 9 | - creates tiny URL from long URL 10 | - redirects from long URL to tiny URL 11 | - gets analytics about tiny URL clicks 12 | 13 | 14 | ### Non-functional requirements: 15 | 16 | - 50 000 000 DAU 17 | - availability 99,95% 18 | - tiny URLs are always stored 19 | - service operation time 5 years 20 | - each user creates one tiny URL per week 21 | - on average, each tiny URL is accessed 20 times a day 22 | - geo distribution is not needed 23 | - no seasonality 24 | 25 | ## Design overview 26 | 27 | For system design I have used [C4 model](https://c4model.com/). The C4 model was created as a way 28 | to help software development teams describe and communicate software 29 | architecture, both during up-front design sessions and when retrospectively 30 | documenting an existing codebase. It's a way to create maps of your code, 31 | at various levels of detail, in the same way you would use something like 32 | Google Maps to zoom in and out of an area you are interested in. 33 | 34 |

35 |
Level 1. System context diagram

36 |

37 | 38 |

39 | 40 |

41 | 42 |

43 |
Level 2. Core system container diagram

44 |

45 | 46 |

47 | 48 |

49 | 50 |

51 |
Level 2. Analytics system container diagram

52 |

53 | 54 |

55 | 56 |

57 | 58 | ## Basic calculations 59 | 60 | RPS (create tiny URL): 61 | 62 | DAU = 50 000 000 63 | Each user creates one link per week 64 | RPS = 50 000 000 / 86 400 / 7 ~= 82 65 | 66 | RPS (redirect from long URL to tiny URL): 67 | 68 | DAU = 50 000 000 69 | Service operation time = 5 years 70 | Each user creates one link per week 71 | Created links for 5 years = 50 000 000 / 7 * 365 ~= 3e9 72 | On average, each tiny URL is accessed 20 times a day 73 | Maximum RPS = 3e9 * 20 / 86400 ~= 700 000 74 | 75 | Tiny URL length: 76 | 77 | Created links for 5 years = 50 000 000 / 7 * 365 ~= 3e9 78 | URL encoded base64 with length 5, will give ~= 916e6 79 | URL encoded base64 with length 6, will give ~= 56e9 80 | Tiny URL length = 6 81 | 82 | Required memory: 83 | 84 | Replication factor = 3 85 | Service operation time = 5 years 86 | Created links for 5 years = 50 000 000 / 7 * 365 ~= 3e9 87 | Each record size for link ~= 150B 88 | Required memory for 5 years = 3e9 * 150 * 3 ~= 1.5TB -------------------------------------------------------------------------------- /api/rest_api.yml: -------------------------------------------------------------------------------- 1 | openapi: 3.0.0 2 | 3 | tags: 4 | - name: TinyURL 5 | - name: Analytics 6 | 7 | info: 8 | title: TinyURL API 9 | description: Current API describes interaction with TinyURL 10 | version: 1.0.0 11 | 12 | paths: 13 | /tiny_urls: 14 | post: 15 | summary: Create tiny url 16 | description: Method creates tiny url from long url 17 | tags: 18 | - TinyURL 19 | requestBody: 20 | required: true 21 | content: 22 | application/json: 23 | schema: 24 | type: object 25 | properties: 26 | long_url: 27 | type: string 28 | description: long url 29 | required: 30 | - long_url 31 | examples: 32 | request: 33 | value: 34 | long_url: "https://test.com/long_long_long_url" 35 | responses: 36 | 200: 37 | description: Ok 38 | content: 39 | application/json: 40 | schema: 41 | type: object 42 | properties: 43 | id: 44 | type: string 45 | description: id of the tiny url 46 | tiny_url: 47 | type: string 48 | description: tiny url for long url 49 | required: 50 | - id 51 | - tiny_url 52 | examples: 53 | response: 54 | value: 55 | id: "12ae3c" 56 | tiny_url: "https://tiny_url.ru/12ae3c" 57 | 400: 58 | description: Bad Request 59 | 500: 60 | description: Server error 61 | 62 | /tiny_urls/{id}: 63 | delete: 64 | summary: Remove tiny url 65 | description: Method removes tiny url 66 | tags: 67 | - TinyURL 68 | parameters: 69 | - name: id 70 | in: path 71 | required: true 72 | description: Tiny url id (uri of the tiny url) 73 | schema: 74 | type : string 75 | responses: 76 | 200: 77 | description: Ok 78 | 404: 79 | description: Not found 80 | 500: 81 | description: Server error 82 | 83 | /{id}: 84 | get: 85 | summary: Redirect to long url 86 | description: Method redirects to long url by tiny url id 87 | tags: 88 | - TinyURL 89 | parameters: 90 | - name: id 91 | in: path 92 | required: true 93 | description: Tiny url id (uri of the tiny url) 94 | schema: 95 | type : string 96 | responses: 97 | 302: 98 | description: Redirect 99 | 404: 100 | description: Not found 101 | 500: 102 | description: Server error 103 | 104 | /clicks/{id}: 105 | get: 106 | summary: Get clicks info about tiny url 107 | description: Method returns clicks info about tiny url 108 | tags: 109 | - Analytics 110 | parameters: 111 | - name: id 112 | in: path 113 | required: true 114 | description: Tiny url id (uri of the tiny url) 115 | schema: 116 | type : string 117 | responses: 118 | 200: 119 | description: Ok 120 | content: 121 | application/json: 122 | schema: 123 | type: object 124 | properties: 125 | count: 126 | type: number 127 | description: Number of clicks 128 | required: 129 | - count 130 | examples: 131 | response: 132 | value: 133 | count: 143 134 | 404: 135 | description: Not found 136 | 500: 137 | description: Server error -------------------------------------------------------------------------------- /images/diagrams/context.svg: -------------------------------------------------------------------------------- 1 | «person»User«container»Core system[Software system] Handles creation of tiny URLfrom long URL andredirections from tiny to longURL«container»Analytics system[Software system] Stores different analyticinformation about tiny URLsCreates tiny URL andredirects from tiny tolong URLGets different clicksinformation by tinyURLs -------------------------------------------------------------------------------- /images/diagrams/containers/core_system.svg: -------------------------------------------------------------------------------- 1 | Core system[System]«webApp»«container»URL Service[Go] Handles creation tiny URLand redirects to long URLfrom tiny URL«db»«container»URL database[PgSQL] Stores long URLs by tinyURLs«db»«container»URL cache[Tarantool] Stores popular long URLs bytiny URLs«person»User«container»Load Balancer[Nginx] Uses round robin«container»Analytics system[Software system] Stores different analyticinformation about tiny URLs«container»Events queue[Kafka] Message queue for eventsby tiny URLsCreates tiny URL andredirects from tinyURL to long URL[REST]Creates tiny URL andredirects from tinyURL to long URL[REST]Creates tiny URLsand gets long URLsby tiny URLsGets and puts leastfrequently usedURLsPublishes clickevents by tiny URLsSubscribes on clickevents by tiny URLs -------------------------------------------------------------------------------- /images/diagrams/containers/analytics_system.svg: -------------------------------------------------------------------------------- 1 | Analytics system[System]«webApp»«container»Analytics Service[Go] Handles requests by clicksinformation and each hourstores aggregatedinformation about clicknumbers of each tiny URL tocache«db»«container»Analytics database[ClickHouse] Stores analytics informationabout clicks of each tiny URL«db»«container»Analytics cache[Tarantool] Stores aggregatedinformation about number ofclicks by each tiny URL«person»User«container»Load Balancer[Nginx] Uses round robin«container»Events queue[Kafka] Message queue for eventsby tiny URLs«container»Core system[Software system] Stores different analyticinformation about tiny URLsGets clicksinformation by tinyURLs[REST]Gets clicksinformation by tinyURLs[REST]Publishes clickevents by tiny URLsSubscribes on clickevents by tiny URLsStores click eventsand also aggregatesinformation about allclicks by tiny URLsGets clicks numberof each tiny URL,also storesinformation aboutclicks number ofeach tiny URL --------------------------------------------------------------------------------