├── .env ├── .gitignore ├── Readme.md ├── diagram.png ├── docker-compose.yaml ├── price-estimate-python ├── Dockerfile ├── constants.py ├── main.py ├── price_service.py ├── processed_vehicle_producer.py ├── requirements.txt ├── schema.py ├── schemas │ ├── processed-vehicle.json │ └── vehicle-stock.json └── vehicle_stock_consumer.py ├── schema-registry-balancer ├── Dockerfile └── nginx.conf ├── sql └── vehicles-stock-init.sql ├── vehicles-ads-nestjs ├── .dockerignore ├── .env ├── .eslintrc.js ├── .gitignore ├── .prettierrc ├── Dockerfile ├── nest-cli.json ├── package-lock.json ├── package.json ├── src │ ├── app.controller.ts │ ├── app.module.ts │ ├── config.ts │ ├── constants.ts │ ├── main.ts │ ├── models │ │ └── vehicle-ad.schema.ts │ └── types.ts ├── test │ ├── app.e2e-spec.ts │ └── jest-e2e.json ├── tsconfig.build.json └── tsconfig.json └── vehicles-stock-nestjs ├── .dockerignore ├── .env ├── .eslintrc.js ├── .gitignore ├── .prettierrc ├── Dockerfile ├── nest-cli.json ├── package-lock.json ├── package.json ├── prisma └── schema.prisma ├── src ├── app.controller.ts ├── app.module.ts ├── config.ts ├── constants.ts ├── dtos │ └── create-vehicle.dto.ts ├── main.ts ├── schemas │ └── vehicle-stock.json └── services │ ├── prisma.service.ts │ └── vehicle-created-producer.service.ts ├── test ├── app.e2e-spec.ts └── jest-e2e.json ├── tsconfig.build.json └── tsconfig.json /.env: -------------------------------------------------------------------------------- 1 | CONFLUENT_KAFKA_TAG=7.4.0 2 | POSTGRES_TAG=14.3 3 | MONGO_TAG=6 4 | 5 | VEHICLES_STOCK_DB_USR=postgres 6 | VEHICLES_STOCK_DB_PWD=postgres 7 | 8 | VEHICLES_ADS_DB_USR=root 9 | VEHICLES_ADS_DB_PWD=root 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.toptal.com/developers/gitignore/api/jetbrains+all,visualstudiocode 2 | # Edit at https://www.toptal.com/developers/gitignore?templates=jetbrains+all,visualstudiocode 3 | 4 | ### JetBrains+all ### 5 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider 6 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 7 | 8 | # User-specific stuff 9 | .idea/**/workspace.xml 10 | .idea/**/tasks.xml 11 | .idea/**/usage.statistics.xml 12 | .idea/**/dictionaries 13 | .idea/**/shelf 14 | 15 | # AWS User-specific 16 | .idea/**/aws.xml 17 | 18 | # Generated files 19 | .idea/**/contentModel.xml 20 | 21 | # Sensitive or high-churn files 22 | .idea/**/dataSources/ 23 | .idea/**/dataSources.ids 24 | .idea/**/dataSources.local.xml 25 | .idea/**/sqlDataSources.xml 26 | .idea/**/dynamic.xml 27 | .idea/**/uiDesigner.xml 28 | .idea/**/dbnavigator.xml 29 | 30 | # Gradle 31 | .idea/**/gradle.xml 32 | .idea/**/libraries 33 | 34 | # Gradle and Maven with auto-import 35 | # When using Gradle or Maven with auto-import, you should exclude module files, 36 | # since they will be recreated, and may cause churn. Uncomment if using 37 | # auto-import. 38 | # .idea/artifacts 39 | # .idea/compiler.xml 40 | # .idea/jarRepositories.xml 41 | # .idea/modules.xml 42 | # .idea/*.iml 43 | # .idea/modules 44 | # *.iml 45 | # *.ipr 46 | 47 | # CMake 48 | cmake-build-*/ 49 | 50 | # Mongo Explorer plugin 51 | .idea/**/mongoSettings.xml 52 | 53 | # File-based project format 54 | *.iws 55 | 56 | # IntelliJ 57 | out/ 58 | 59 | # mpeltonen/sbt-idea plugin 60 | .idea_modules/ 61 | 62 | # JIRA plugin 63 | atlassian-ide-plugin.xml 64 | 65 | # Cursive Clojure plugin 66 | .idea/replstate.xml 67 | 68 | # SonarLint plugin 69 | .idea/sonarlint/ 70 | 71 | # Crashlytics plugin (for Android Studio and IntelliJ) 72 | com_crashlytics_export_strings.xml 73 | crashlytics.properties 74 | crashlytics-build.properties 75 | fabric.properties 76 | 77 | # Editor-based Rest Client 78 | .idea/httpRequests 79 | 80 | # Android studio 3.1+ serialized cache file 81 | .idea/caches/build_file_checksums.ser 82 | 83 | ### JetBrains+all Patch ### 84 | # Ignore everything but code style settings and run configurations 85 | # that are supposed to be shared within teams. 86 | 87 | .idea/* 88 | 89 | !.idea/codeStyles 90 | !.idea/runConfigurations 91 | 92 | ### VisualStudioCode ### 93 | .vscode/* 94 | !.vscode/settings.json 95 | !.vscode/tasks.json 96 | !.vscode/launch.json 97 | !.vscode/extensions.json 98 | !.vscode/*.code-snippets 99 | 100 | # Local History for Visual Studio Code 101 | .history/ 102 | 103 | # Built Visual Studio Code Extensions 104 | *.vsix 105 | 106 | ### VisualStudioCode Patch ### 107 | # Ignore all local history of files 108 | .history 109 | .ionide 110 | 111 | # End of https://www.toptal.com/developers/gitignore/api/jetbrains+all,visualstudiocode 112 | -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | # Microservices architecture with NestJS, Python and Kafka 2 | 3 | The goal of this project is to practice a microservices architecture based on **Apache Kafka** with services written in different languages — **TypeScript** and **Python** — that communicate between them. The Kafka layer is configured to have a basic level of redundancy, with each component running on three instances. 4 | 5 | In total, the system is composed of 17 containers: 6 | - 3x **Apache Kafka** brokers 7 | - 3x **ZooKeeper** instances to orchestrate the Kafka infrastructure 8 | - 3x **Schema Registry** instances 9 | - 1x **NGINX** load balancer for Schema Registry 10 | - 1x **Apache Kafka UI**, a web-app to better interact with Kafka (exposed on http://localhost:8080, credentials `admin:password`) 11 | - 1x **NestJS** instance for the `vehicles-stock` service (exposed on http://localhost:3002) 12 | - 1x **PostgreSQL** database for the previous service 13 | - 2x **Python** instances for the `price-estimate` service 14 | - 1x **NestJS** instance for the `vehicles-ads` service (exposed on http://localhost:3003) 15 | - 1x **MongoDB** database for the previous service 16 | 17 | ![Infrastructure diagram](diagram.png) 18 | 19 | ## Services 20 | 21 | ### vehicles-stock 22 | This service exposes an HTTP REST API that allows users to create and list vehicles, and is also a **Kafka producer**. 23 | 24 | A POST request to `http://localhost:3002/stock` with the following body 25 | ```json 26 | { 27 | "vin": "vin", 28 | "model": "model", 29 | "manufacturer": "manufacturer", 30 | "year": 2022, 31 | "odometer": 1000, 32 | "odometerUnit": "odometerUnit" 33 | } 34 | ``` 35 | creates a new vehicle in the local Postgres database and emits data to the `stock.vehicle-created` topic in Kafka. 36 | 37 | A GET request to `http://localhost:3002/stock` retrieves vehicles. 38 | 39 | ### price-estimate 40 | This service consumes `stock.vehicle-created` to calculate a price for vehicles. The calculation simply generates a random number after 30 seconds to simulate some form of processing. 41 | 42 | When done, it emits the vehicle with its price to the `stock.vehicle-processed` topic in Kafka. This service is **both a consumer and a producer**. 43 | 44 | ### vehicles-ads 45 | This service only exposes an HTTP endpoint to get a list of ads — `http://localhost:3003/ads` — and is a **Kafka consumer**: it consumes `stock.vehicle-processed` to create vehicle ads in the local MongoDB database. 46 | 47 | ## Schema registry 48 | 49 | Messages are encoded and decoded using **JSON schemas** registered on the schema registry, to enforce structure: each topic requires messages to conform to a specific schema, and trying to emit one that does not will result in an error. 50 | 51 | Currently each service registers its own schema, so common schemas may be duplicated across services — this can be improved. 52 | 53 | ## Startup 54 | 55 | Run `docker compose up -d` to create the infrastructure and start the containers; the process may take some time. A few services need to wait for the schema registry servers to be up, so they use a *wait-for* script with a timeout of 90 seconds. 56 | 57 | The following services will then be exposed to the host machine: 58 | 59 | - Kafka Web UI on `http://localhost:8080` (use `admin:password` to access it) 60 | - Vehicles stock service on `http://localhost:3002` 61 | - Vehicles ads service on `http://localhost:3003` -------------------------------------------------------------------------------- /diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gispada/nestjs-python-kafka-microservices/5061d73e9fda99cbba7f1872a24203a08429499c/diagram.png -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3.9' 2 | name: npkm 3 | 4 | x-kafka-env: &kafka-env 5 | KAFKA_DEFAULT_REPLICATION_FACTOR: 3 6 | KAFKA_NUM_PARTITIONS: 3 7 | KAFKA_MIN_INSYNC_REPLICAS: 2 8 | ALLOW_PLAINTEXT_LISTENER: "yes" 9 | KAFKA_CONFLUENT_SCHEMA_REGISTRY_URL: http://ms-schema-registry-1:8081,http://ms-schema-registry-2:8082,http://ms-schema-registry-3:8083 10 | KAFKA_ZOOKEEPER_CONNECT: ms-zookeeper-1:2181,ms-zookeeper-2:2182,ms-zookeeper-3:2183 11 | KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" 12 | 13 | x-zookeeper-env: &zookeeper-env 14 | ZOOKEEPER_CLIENT_PORT: 2181 15 | ZOOKEEPER_TICK_TIME: 2000 16 | ZOOKEEPER_SERVERS: ms-zookeeper-1:2888:3888;ms-zookeeper-2:2888:3888;ms-zookeeper-3:2888:3888 17 | 18 | x-schema-registry-env: &schema-registry-env 19 | SCHEMA_REGISTRY_KAFKASTORE_GROUP_ID: ms-schema-registry 20 | SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: ms-kafka-1:9092 21 | SCHEMA_REGISTRY_LEADER_ELIGIBILITY: "true" 22 | SCHEMA_REGISTRY_KAFKASTORE_TOPIC_REPLICATION_FACTOR: 3 23 | 24 | services: 25 | # -------------------- Zookeeper -------------------- # 26 | zookeeper-1: 27 | container_name: ms-zookeeper-1 28 | image: "confluentinc/cp-zookeeper:${CONFLUENT_KAFKA_TAG}" 29 | environment: 30 | <<: *zookeeper-env 31 | ZOOKEEPER_SERVER_ID: 1 32 | 33 | zookeeper-2: 34 | container_name: ms-zookeeper-2 35 | image: "confluentinc/cp-zookeeper:${CONFLUENT_KAFKA_TAG}" 36 | environment: 37 | <<: *zookeeper-env 38 | ZOOKEEPER_SERVER_ID: 2 39 | 40 | zookeeper-3: 41 | container_name: ms-zookeeper-3 42 | image: "confluentinc/cp-zookeeper:${CONFLUENT_KAFKA_TAG}" 43 | environment: 44 | <<: *zookeeper-env 45 | ZOOKEEPER_SERVER_ID: 3 46 | 47 | # -------------------- Kafka brokers -------------------- # 48 | kafka-1: 49 | container_name: ms-kafka-1 50 | image: "confluentinc/cp-kafka:${CONFLUENT_KAFKA_TAG}" 51 | depends_on: 52 | - zookeeper-1 53 | - zookeeper-2 54 | - zookeeper-3 55 | environment: 56 | <<: *kafka-env 57 | KAFKA_BROKER_ID: 1 58 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://ms-kafka-1:9092 59 | 60 | kafka-2: 61 | container_name: ms-kafka-2 62 | image: "confluentinc/cp-kafka:${CONFLUENT_KAFKA_TAG}" 63 | depends_on: 64 | - zookeeper-1 65 | - zookeeper-2 66 | - zookeeper-3 67 | environment: 68 | <<: *kafka-env 69 | KAFKA_BROKER_ID: 2 70 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://ms-kafka-2:9093 71 | 72 | kafka-3: 73 | container_name: ms-kafka-3 74 | image: "confluentinc/cp-kafka:${CONFLUENT_KAFKA_TAG}" 75 | depends_on: 76 | - zookeeper-1 77 | - zookeeper-2 78 | - zookeeper-3 79 | environment: 80 | <<: *kafka-env 81 | KAFKA_BROKER_ID: 3 82 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://ms-kafka-3:9094 83 | 84 | # -------------------- Schema Registry -------------------- # 85 | schema-registry-balancer: 86 | container_name: ms-schema-registry-balancer 87 | build: 88 | context: './schema-registry-balancer' 89 | expose: 90 | - 8088 91 | 92 | schema-registry-1: 93 | image: "confluentinc/cp-schema-registry:${CONFLUENT_KAFKA_TAG}" 94 | container_name: ms-schema-registry-1 95 | restart: on-failure:3 96 | depends_on: 97 | - kafka-1 98 | - kafka-2 99 | - kafka-3 100 | environment: 101 | <<: *schema-registry-env 102 | SCHEMA_REGISTRY_HOST_NAME: ms-schema-registry-1 103 | SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 104 | 105 | schema-registry-2: 106 | image: "confluentinc/cp-schema-registry:${CONFLUENT_KAFKA_TAG}" 107 | container_name: ms-schema-registry-2 108 | restart: on-failure:3 109 | depends_on: 110 | - kafka-1 111 | - kafka-2 112 | - kafka-3 113 | environment: 114 | <<: *schema-registry-env 115 | SCHEMA_REGISTRY_HOST_NAME: ms-schema-registry-2 116 | SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8082 117 | 118 | schema-registry-3: 119 | image: "confluentinc/cp-schema-registry:${CONFLUENT_KAFKA_TAG}" 120 | container_name: ms-schema-registry-3 121 | restart: on-failure:3 122 | depends_on: 123 | - kafka-1 124 | - kafka-2 125 | - kafka-3 126 | environment: 127 | <<: *schema-registry-env 128 | SCHEMA_REGISTRY_HOST_NAME: ms-schema-registry-3 129 | SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8083 130 | 131 | # -------------------- Kafka Admin UI -------------------- # 132 | kafka-ui: 133 | image: provectuslabs/kafka-ui:latest 134 | container_name: ms-kafka-admin-ui 135 | depends_on: 136 | - kafka-1 137 | - kafka-2 138 | - kafka-3 139 | ports: 140 | - 8080:8080 141 | environment: 142 | AUTH_TYPE: LOGIN_FORM 143 | SPRING_SECURITY_USER_NAME: admin 144 | SPRING_SECURITY_USER_PASSWORD: password 145 | KAFKA_CLUSTERS_0_NAME: local 146 | KAFKA_CLUSTERS_0_ZOOKEEPER: ms-zookeeper-1:2181 147 | KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: ms-kafka-1:9092 148 | KAFKA_CLUSTERS_0_SCHEMAREGISTRY: http://ms-schema-registry-balancer:8088 149 | 150 | # -------------------- Microservices -------------------- # 151 | price-estimate-1: 152 | container_name: ms-price-estimate-1 153 | depends_on: 154 | - schema-registry-1 155 | - schema-registry-2 156 | - schema-registry-3 157 | build: 158 | context: ./price-estimate-python 159 | 160 | price-estimate-2: 161 | container_name: ms-price-estimate-2 162 | depends_on: 163 | - schema-registry-1 164 | - schema-registry-2 165 | - schema-registry-3 166 | build: 167 | context: ./price-estimate-python 168 | 169 | vehicles-stock: 170 | container_name: ms-vehicles-stock 171 | depends_on: 172 | - schema-registry-1 173 | - schema-registry-2 174 | - schema-registry-3 175 | - vehicles-stock-db 176 | build: 177 | context: ./vehicles-stock-nestjs 178 | ports: 179 | - 3002:3002 180 | 181 | vehicles-ads: 182 | container_name: ms-vehicles-ads 183 | depends_on: 184 | - schema-registry-1 185 | - schema-registry-2 186 | - schema-registry-3 187 | - vehicles-ads-db 188 | build: 189 | context: ./vehicles-ads-nestjs 190 | ports: 191 | - 3003:3003 192 | 193 | # -------------------- Databases -------------------- # 194 | vehicles-stock-db: 195 | image: "postgres:${POSTGRES_TAG}" 196 | container_name: ms-vehicles-stock-db 197 | expose: 198 | - 5432 199 | volumes: 200 | - ms-postgres:/var/lib/postgresql/data 201 | - ./sql/vehicles-stock-init.sql:/docker-entrypoint-initdb.d/vehicles-stock-init.sql 202 | environment: 203 | POSTGRES_USER: ${VEHICLES_STOCK_DB_USR} 204 | POSTGRES_PASSWORD: ${VEHICLES_STOCK_DB_PWD} 205 | 206 | vehicles-ads-db: 207 | image: "mongo:${MONGO_TAG}" 208 | container_name: ms-vehicles-ads-db 209 | expose: 210 | - 27017 211 | volumes: 212 | - ms-mongo:/data/db 213 | environment: 214 | MONGO_INITDB_ROOT_USERNAME: ${VEHICLES_ADS_DB_USR} 215 | MONGO_INITDB_ROOT_PASSWORD: ${VEHICLES_ADS_DB_PWD} 216 | MONGO_INITDB_DATABASE: vehicles 217 | 218 | volumes: 219 | ms-postgres: 220 | ms-mongo: 221 | -------------------------------------------------------------------------------- /price-estimate-python/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.9.13-slim 2 | 3 | COPY . . 4 | 5 | ENV KAFKA_BOOTSTRAP_SERVER=ms-kafka-1:9092 6 | ENV SCHEMA_REGISTRY_URL=http://ms-schema-registry-balancer:8088 7 | 8 | RUN apt-get update && apt-get install -y wget 9 | RUN wget -qO wait-for https://raw.githubusercontent.com/eficode/wait-for/master/wait-for 10 | RUN chmod +x wait-for 11 | RUN pip install -r requirements.txt 12 | 13 | # The schema registry takes a while to get up and running 14 | CMD ./wait-for http://ms-schema-registry-balancer:8088 -t 90 -- python -u main.py 15 | -------------------------------------------------------------------------------- /price-estimate-python/constants.py: -------------------------------------------------------------------------------- 1 | VEHICLE_CREATED_TOPIC = 'stock.vehicle-created' 2 | VEHICLE_PROCESSED_TOPIC = 'stock.vehicle-processed' 3 | CONSUMER_GROUP_ID = 'ms-price-estimate' 4 | -------------------------------------------------------------------------------- /price-estimate-python/main.py: -------------------------------------------------------------------------------- 1 | from price_service import PriceService 2 | 3 | def main(): 4 | print('Starting Price service') 5 | service = PriceService() 6 | service.start() 7 | 8 | if __name__ == '__main__': 9 | main() 10 | -------------------------------------------------------------------------------- /price-estimate-python/price_service.py: -------------------------------------------------------------------------------- 1 | import signal 2 | import json 3 | import threading 4 | from random import uniform 5 | from vehicle_stock_consumer import VehicleStockConsumer 6 | from processed_vehicle_producer import ProcessedVehicleProducer 7 | 8 | class PriceService: 9 | """ 10 | Consume messages from a vehicles stock topic, 11 | process those messages to add price information, 12 | then publish them to another topic. 13 | """ 14 | 15 | def __init__(self): 16 | signal.signal(signal.SIGINT, self.stop) 17 | signal.signal(signal.SIGTERM, self.stop) 18 | 19 | self.vehicle_stock_consumer = VehicleStockConsumer() 20 | self.processed_vehicle_producer = ProcessedVehicleProducer() 21 | 22 | def start(self): 23 | self.vehicle_stock_consumer.subscribe(self.onMessage) 24 | 25 | def stop(self, signalnum, handler): 26 | print('Stopping Price service') 27 | self.vehicle_stock_consumer.close() 28 | self.processed_vehicle_producer.close() 29 | 30 | def process_vehicle(self, v): 31 | processed_vehicle = { 32 | 'vin': v['vin'], 33 | 'model': v['model'], 34 | 'manufacturer': v['manufacturer'], 35 | 'year': v['year'], 36 | 'odometer': v['odometer'], 37 | 'odometerUnit': v['odometerUnit'], 38 | 'price': round(uniform(1000, 50000), 2), 39 | 'priceUnit': 'USD', 40 | } 41 | # Simulate a long processing time 42 | t = threading.Timer(30.0, 43 | lambda : self.processed_vehicle_producer.send(processed_vehicle)) 44 | t.daemon = True 45 | t.start() 46 | 47 | def onMessage(self, message): 48 | print(message) 49 | self.process_vehicle(message.value) 50 | -------------------------------------------------------------------------------- /price-estimate-python/processed_vehicle_producer.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | from kafka import KafkaProducer 4 | from schema import Schema 5 | import constants as c 6 | 7 | with open('./schemas/processed-vehicle.json') as f: 8 | producer_schema_file = json.load(f) 9 | 10 | class ProcessedVehicleProducer: 11 | def __init__(self): 12 | producer_schema = Schema(f'{c.VEHICLE_PROCESSED_TOPIC}-value', producer_schema_file) 13 | 14 | self.producer = KafkaProducer( 15 | bootstrap_servers=os.getenv('KAFKA_BOOTSTRAP_SERVER'), 16 | value_serializer=producer_schema.encode_message, 17 | api_version=(0, 10, 2)) 18 | 19 | def send(self, message): 20 | return self.producer.send(c.VEHICLE_PROCESSED_TOPIC, message) 21 | -------------------------------------------------------------------------------- /price-estimate-python/requirements.txt: -------------------------------------------------------------------------------- 1 | kafka-python==2.0.2 2 | python-schema-registry-client==2.4.0 -------------------------------------------------------------------------------- /price-estimate-python/schema.py: -------------------------------------------------------------------------------- 1 | import os 2 | from schema_registry.client import SchemaRegistryClient, schema as s 3 | from schema_registry.serializers import JsonMessageSerializer 4 | 5 | class Schema: 6 | def __init__(self, schema_name, schema): 7 | self.client = SchemaRegistryClient(url=os.getenv('SCHEMA_REGISTRY_URL')) 8 | 9 | self.json_message_serializer = JsonMessageSerializer( 10 | schemaregistry_client=self.client, reader_schema=schema) 11 | 12 | self.schema_id = self.client.register( 13 | schema_name, s.JsonSchema(schema)) 14 | 15 | print(f'Registered schema "{schema_name}" with ID {self.schema_id}') 16 | 17 | def encode_message(self, message): 18 | return self.json_message_serializer.encode_record_with_schema_id( 19 | self.schema_id, message) 20 | 21 | def decode_message(self, message): 22 | return self.json_message_serializer.decode_message(message) 23 | -------------------------------------------------------------------------------- /price-estimate-python/schemas/processed-vehicle.json: -------------------------------------------------------------------------------- 1 | { 2 | "$id": "nestjs-python-kafka/processed-vehicle.schema.json", 3 | "type": "object", 4 | "required": [ 5 | "vin", 6 | "model", 7 | "year", 8 | "manufacturer", 9 | "odometer", 10 | "odometerUnit", 11 | "price", 12 | "priceUnit" 13 | ], 14 | "properties": { 15 | "vin": { 16 | "type": "string" 17 | }, 18 | "model": { 19 | "type": "string" 20 | }, 21 | "year": { 22 | "type": "number" 23 | }, 24 | "manufacturer": { 25 | "type": "string" 26 | }, 27 | "odometer": { 28 | "type": "number" 29 | }, 30 | "odometerUnit": { 31 | "enum": [ 32 | "km", 33 | "mi" 34 | ] 35 | }, 36 | "price": { 37 | "type": "number" 38 | }, 39 | "priceUnit": { 40 | "type": "string" 41 | } 42 | } 43 | } -------------------------------------------------------------------------------- /price-estimate-python/schemas/vehicle-stock.json: -------------------------------------------------------------------------------- 1 | { 2 | "$id": "nestjs-python-kafka/vehicle-stock.schema.json", 3 | "type": "object", 4 | "required": [ 5 | "id", 6 | "vin", 7 | "model", 8 | "year", 9 | "manufacturer", 10 | "odometer", 11 | "odometerUnit" 12 | ], 13 | "properties": { 14 | "id": { 15 | "type": "number" 16 | }, 17 | "vin": { 18 | "type": "string" 19 | }, 20 | "model": { 21 | "type": "string" 22 | }, 23 | "year": { 24 | "type": "number" 25 | }, 26 | "manufacturer": { 27 | "type": "string" 28 | }, 29 | "odometer": { 30 | "type": "number" 31 | }, 32 | "odometerUnit": { 33 | "enum": [ 34 | "km", 35 | "mi" 36 | ] 37 | } 38 | } 39 | } -------------------------------------------------------------------------------- /price-estimate-python/vehicle_stock_consumer.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | from kafka import KafkaConsumer 4 | from schema import Schema 5 | import constants as c 6 | 7 | with open('./schemas/vehicle-stock.json') as f: 8 | consumer_schema_file = json.load(f) 9 | 10 | class VehicleStockConsumer: 11 | def __init__(self): 12 | consumer_schema = Schema(f'{c.VEHICLE_CREATED_TOPIC}-value', consumer_schema_file) 13 | 14 | self.consumer = KafkaConsumer( 15 | bootstrap_servers=os.getenv('KAFKA_BOOTSTRAP_SERVER'), 16 | group_id=c.CONSUMER_GROUP_ID, 17 | value_deserializer=consumer_schema.decode_message, 18 | api_version=(0, 10, 2)) 19 | 20 | def subscribe(self, onMessage): 21 | self.consumer.subscribe([c.VEHICLE_CREATED_TOPIC]) 22 | print(f'Subscribed to "{c.VEHICLE_CREATED_TOPIC}"') 23 | 24 | for message in self.consumer: 25 | onMessage(message) 26 | -------------------------------------------------------------------------------- /schema-registry-balancer/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nginx 2 | 3 | RUN curl https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh -o wait-for-it -s 4 | RUN chmod +x wait-for-it 5 | 6 | COPY nginx.conf /etc/nginx/nginx.conf 7 | 8 | CMD ./wait-for-it ms-schema-registry-1:8081 -t 90 -- nginx -g 'daemon off;' -------------------------------------------------------------------------------- /schema-registry-balancer/nginx.conf: -------------------------------------------------------------------------------- 1 | events { 2 | worker_connections 1024; 3 | } 4 | 5 | http { 6 | upstream load-balancer { 7 | server ms-schema-registry-1:8081; 8 | server ms-schema-registry-2:8082; 9 | server ms-schema-registry-3:8083; 10 | } 11 | 12 | server { 13 | listen 8088; 14 | location / { 15 | proxy_pass http://load-balancer; 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /sql/vehicles-stock-init.sql: -------------------------------------------------------------------------------- 1 | CREATE DATABASE stock; 2 | 3 | \c stock; 4 | 5 | CREATE TABLE IF NOT EXISTS "Vehicles" ( 6 | "id" SERIAL NOT NULL, 7 | "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, 8 | "vin" TEXT NOT NULL, 9 | "model" TEXT NOT NULL, 10 | "year" INTEGER NOT NULL, 11 | "manufacturer" TEXT NOT NULL, 12 | "odometer" DOUBLE PRECISION NOT NULL, 13 | "odometerUnit" TEXT NOT NULL, 14 | 15 | CONSTRAINT "Vehicles_pkey" PRIMARY KEY ("id") 16 | ); 17 | 18 | CREATE UNIQUE INDEX "Vehicles_vin_key" ON "Vehicles"("vin"); 19 | -------------------------------------------------------------------------------- /vehicles-ads-nestjs/.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules -------------------------------------------------------------------------------- /vehicles-ads-nestjs/.env: -------------------------------------------------------------------------------- 1 | DATABASE_URL=mongodb://root:root@ms-vehicles-ads-db:27017/vehicles?authSource=admin 2 | KAFKA_BOOTSTRAP_SERVER=ms-kafka-1:9092 3 | SCHEMA_REGISTRY_URL=http://ms-schema-registry-balancer:8088 4 | -------------------------------------------------------------------------------- /vehicles-ads-nestjs/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | parser: '@typescript-eslint/parser', 3 | parserOptions: { 4 | project: 'tsconfig.json', 5 | tsconfigRootDir : __dirname, 6 | sourceType: 'module', 7 | }, 8 | plugins: ['@typescript-eslint/eslint-plugin'], 9 | extends: [ 10 | 'plugin:@typescript-eslint/recommended', 11 | 'plugin:prettier/recommended', 12 | ], 13 | root: true, 14 | env: { 15 | node: true, 16 | jest: true, 17 | }, 18 | ignorePatterns: ['.eslintrc.js'], 19 | rules: { 20 | '@typescript-eslint/interface-name-prefix': 'off', 21 | '@typescript-eslint/explicit-function-return-type': 'off', 22 | '@typescript-eslint/explicit-module-boundary-types': 'off', 23 | '@typescript-eslint/no-explicit-any': 'off', 24 | }, 25 | }; 26 | -------------------------------------------------------------------------------- /vehicles-ads-nestjs/.gitignore: -------------------------------------------------------------------------------- 1 | # compiled output 2 | /dist 3 | /node_modules 4 | 5 | # Logs 6 | logs 7 | *.log 8 | npm-debug.log* 9 | pnpm-debug.log* 10 | yarn-debug.log* 11 | yarn-error.log* 12 | lerna-debug.log* 13 | 14 | # OS 15 | .DS_Store 16 | 17 | # Tests 18 | /coverage 19 | /.nyc_output 20 | 21 | # IDEs and editors 22 | /.idea 23 | .project 24 | .classpath 25 | .c9/ 26 | *.launch 27 | .settings/ 28 | *.sublime-workspace 29 | 30 | # IDE - VSCode 31 | .vscode/* 32 | !.vscode/settings.json 33 | !.vscode/tasks.json 34 | !.vscode/launch.json 35 | !.vscode/extensions.json -------------------------------------------------------------------------------- /vehicles-ads-nestjs/.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "semi": false, 3 | "singleQuote": true, 4 | "trailingComma": "all" 5 | } -------------------------------------------------------------------------------- /vehicles-ads-nestjs/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:16.15-alpine 2 | 3 | COPY . . 4 | 5 | RUN wget -qO wait-for https://raw.githubusercontent.com/eficode/wait-for/master/wait-for 6 | RUN chmod +x wait-for 7 | RUN npm install 8 | RUN npm run build 9 | 10 | CMD ./wait-for http://ms-schema-registry-balancer:8088 -t 90 -- npm run start:prod 11 | -------------------------------------------------------------------------------- /vehicles-ads-nestjs/nest-cli.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://json.schemastore.org/nest-cli", 3 | "collection": "@nestjs/schematics", 4 | "sourceRoot": "src" 5 | } 6 | -------------------------------------------------------------------------------- /vehicles-ads-nestjs/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "vehicles-ads", 3 | "version": "0.0.1", 4 | "description": "", 5 | "author": "", 6 | "private": true, 7 | "license": "UNLICENSED", 8 | "scripts": { 9 | "prebuild": "rimraf dist", 10 | "build": "nest build", 11 | "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"", 12 | "start": "nest start", 13 | "start:dev": "nest start --watch", 14 | "start:debug": "nest start --debug --watch", 15 | "start:prod": "node dist/main", 16 | "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix", 17 | "test": "jest", 18 | "test:watch": "jest --watch", 19 | "test:cov": "jest --coverage", 20 | "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", 21 | "test:e2e": "jest --config ./test/jest-e2e.json" 22 | }, 23 | "dependencies": { 24 | "@kafkajs/confluent-schema-registry": "^3.2.1", 25 | "@nestjs/common": "^8.4.7", 26 | "@nestjs/config": "^2.1.0", 27 | "@nestjs/core": "^8.4.7", 28 | "@nestjs/microservices": "^8.4.7", 29 | "@nestjs/mongoose": "^9.2.0", 30 | "@nestjs/platform-express": "^8.4.7", 31 | "kafkajs": "^2.1.0", 32 | "mongoose": "^6.4.4", 33 | "reflect-metadata": "^0.1.13", 34 | "rimraf": "^3.0.2", 35 | "rxjs": "^7.2.0" 36 | }, 37 | "devDependencies": { 38 | "@nestjs/cli": "^8.2.8", 39 | "@nestjs/schematics": "^8.0.11", 40 | "@nestjs/testing": "^8.4.7", 41 | "@types/express": "^4.17.13", 42 | "@types/jest": "27.5.0", 43 | "@types/node": "^16.0.0", 44 | "@types/supertest": "^2.0.11", 45 | "@typescript-eslint/eslint-plugin": "^5.0.0", 46 | "@typescript-eslint/parser": "^5.0.0", 47 | "eslint": "^8.0.1", 48 | "eslint-config-prettier": "^8.3.0", 49 | "eslint-plugin-prettier": "^4.0.0", 50 | "jest": "28.0.3", 51 | "prettier": "^2.3.2", 52 | "source-map-support": "^0.5.20", 53 | "supertest": "^6.1.3", 54 | "ts-jest": "28.0.1", 55 | "ts-loader": "^9.2.3", 56 | "ts-node": "^10.0.0", 57 | "tsconfig-paths": "4.0.0", 58 | "typescript": "^4.3.5" 59 | }, 60 | "jest": { 61 | "moduleFileExtensions": [ 62 | "js", 63 | "json", 64 | "ts" 65 | ], 66 | "rootDir": "src", 67 | "testRegex": ".*\\.spec\\.ts$", 68 | "transform": { 69 | "^.+\\.(t|j)s$": "ts-jest" 70 | }, 71 | "collectCoverageFrom": [ 72 | "**/*.(t|j)s" 73 | ], 74 | "coverageDirectory": "../coverage", 75 | "testEnvironment": "node" 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /vehicles-ads-nestjs/src/app.controller.ts: -------------------------------------------------------------------------------- 1 | import { Controller, Get, Param } from '@nestjs/common' 2 | import { EventPattern, Payload } from '@nestjs/microservices' 3 | import { InjectModel } from '@nestjs/mongoose' 4 | import type { Model } from 'mongoose' 5 | import { SchemaRegistry } from '@kafkajs/confluent-schema-registry' 6 | import { KafkaMessage } from 'kafkajs' 7 | import { VehicleAd, VehicleAdDocument } from './models/vehicle-ad.schema' 8 | import { VEHICLE_PROCESSED_TOPIC } from './constants' 9 | import { ProcessedVehicle } from './types' 10 | 11 | @Controller('ads') 12 | export class AppController { 13 | constructor( 14 | @InjectModel(VehicleAd.name) private vehicleAd: Model, 15 | private readonly schemaRegistry: SchemaRegistry, 16 | ) {} 17 | 18 | @Get() 19 | getAds() { 20 | return this.vehicleAd.find({}) 21 | } 22 | 23 | @Get(':id') 24 | getAdById(@Param('id') id: string) { 25 | return this.vehicleAd.findOne({ id }) 26 | } 27 | 28 | // Subscribe to Kafka topic for processed vehicles and save the message on MongoDB 29 | @EventPattern(VEHICLE_PROCESSED_TOPIC) 30 | async onVehicleProcessed(@Payload() message: KafkaMessage) { 31 | const decodedMessage: ProcessedVehicle = await this.schemaRegistry.decode( 32 | message.value, 33 | ) 34 | const newVehicleAd = new this.vehicleAd(decodedMessage) 35 | await newVehicleAd.save() 36 | console.log(`Successfully created vehicle ad with ID ${newVehicleAd._id}`) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /vehicles-ads-nestjs/src/app.module.ts: -------------------------------------------------------------------------------- 1 | import { Module } from '@nestjs/common' 2 | import { ConfigModule, ConfigService } from '@nestjs/config' 3 | import { MongooseModule } from '@nestjs/mongoose' 4 | import { SchemaRegistry } from '@kafkajs/confluent-schema-registry' 5 | import { AppController } from './app.controller' 6 | import { VehicleAd, VehicleAdSchema } from './models/vehicle-ad.schema' 7 | import configuration from './config' 8 | 9 | const imports = [ 10 | ConfigModule.forRoot({ 11 | load: [configuration], 12 | isGlobal: true, 13 | }), 14 | MongooseModule.forRootAsync({ 15 | useFactory: (config: ConfigService) => ({ 16 | uri: config.get('database.url'), 17 | }), 18 | inject: [ConfigService], 19 | }), 20 | MongooseModule.forFeature([ 21 | { name: VehicleAd.name, schema: VehicleAdSchema }, 22 | ]), 23 | ] 24 | 25 | const providers = [ 26 | { 27 | provide: SchemaRegistry, 28 | useFactory: (config: ConfigService) => 29 | new SchemaRegistry({ host: config.get('kafka.schemaRegistry') }), 30 | inject: [ConfigService], 31 | }, 32 | ] 33 | 34 | // Note: this app is only a consumer 35 | @Module({ 36 | imports, 37 | controllers: [AppController], 38 | providers, 39 | }) 40 | export class AppModule {} 41 | -------------------------------------------------------------------------------- /vehicles-ads-nestjs/src/config.ts: -------------------------------------------------------------------------------- 1 | export default () => ({ 2 | kafka: { 3 | bootstrapServer: process.env.KAFKA_BOOTSTRAP_SERVER, 4 | schemaRegistry: process.env.SCHEMA_REGISTRY_URL, 5 | }, 6 | database: { 7 | url: process.env.DATABASE_URL, 8 | }, 9 | }) 10 | -------------------------------------------------------------------------------- /vehicles-ads-nestjs/src/constants.ts: -------------------------------------------------------------------------------- 1 | export const CONSUMER_GROUP_ID = 'ms-vehicles-ads' 2 | export const VEHICLE_PROCESSED_TOPIC = 'stock.vehicle-processed' 3 | -------------------------------------------------------------------------------- /vehicles-ads-nestjs/src/main.ts: -------------------------------------------------------------------------------- 1 | import { MicroserviceOptions, Transport } from '@nestjs/microservices' 2 | import { NestFactory } from '@nestjs/core' 3 | import { AppModule } from './app.module' 4 | import { CONSUMER_GROUP_ID } from './constants' 5 | 6 | const DEFAULT_PORT = 3003 7 | 8 | async function bootstrap() { 9 | const app = await NestFactory.create(AppModule) 10 | 11 | app.connectMicroservice({ 12 | transport: Transport.KAFKA, 13 | options: { 14 | client: { 15 | clientId: 'vehicles-ads-app', 16 | brokers: [process.env.KAFKA_BOOTSTRAP_SERVER], 17 | }, 18 | consumer: { 19 | groupId: CONSUMER_GROUP_ID, 20 | }, 21 | subscribe: { 22 | fromBeginning: true, 23 | }, 24 | }, 25 | }) 26 | 27 | await app.startAllMicroservices() 28 | await app.listen(process.env.PORT || DEFAULT_PORT) 29 | } 30 | 31 | bootstrap() 32 | -------------------------------------------------------------------------------- /vehicles-ads-nestjs/src/models/vehicle-ad.schema.ts: -------------------------------------------------------------------------------- 1 | import { Prop, Schema, SchemaFactory } from '@nestjs/mongoose' 2 | import { Document, ObjectId } from 'mongoose' 3 | 4 | @Schema({ collection: 'ads' }) 5 | export class VehicleAd { 6 | @Prop({ default: () => new Date().toISOString() }) 7 | publishedAt: string 8 | 9 | @Prop({ required: true, unique: true }) 10 | vin: string 11 | 12 | @Prop({ required: true }) 13 | model: string 14 | 15 | @Prop({ required: true }) 16 | manufacturer: string 17 | 18 | @Prop({ required: true }) 19 | year: number 20 | 21 | @Prop({ required: true }) 22 | odometer: number 23 | 24 | @Prop({ required: true }) 25 | odometerUnit: string 26 | 27 | @Prop({ required: true }) 28 | price: number 29 | 30 | @Prop({ required: true }) 31 | priceUnit: string 32 | } 33 | 34 | export type VehicleAdDocument = VehicleAd & Document 35 | 36 | export const VehicleAdSchema = SchemaFactory.createForClass(VehicleAd) 37 | -------------------------------------------------------------------------------- /vehicles-ads-nestjs/src/types.ts: -------------------------------------------------------------------------------- 1 | export type ProcessedVehicle = { 2 | vin: string 3 | model: string 4 | manufacturer: string 5 | year: number 6 | odometer: number 7 | odometerUnit: string 8 | price: number 9 | priceUnit: string 10 | } 11 | -------------------------------------------------------------------------------- /vehicles-ads-nestjs/test/app.e2e-spec.ts: -------------------------------------------------------------------------------- 1 | import { Test, TestingModule } from '@nestjs/testing' 2 | import { INestApplication } from '@nestjs/common' 3 | import * as request from 'supertest' 4 | import { AppModule } from './../src/app.module' 5 | 6 | describe('AppController (e2e)', () => { 7 | let app: INestApplication 8 | 9 | beforeEach(async () => { 10 | const moduleFixture: TestingModule = await Test.createTestingModule({ 11 | imports: [AppModule], 12 | }).compile() 13 | 14 | app = moduleFixture.createNestApplication() 15 | await app.init() 16 | }) 17 | 18 | it('/ (GET)', () => { 19 | return request(app.getHttpServer()) 20 | .get('/') 21 | .expect(200) 22 | .expect('Hello World!') 23 | }) 24 | }) 25 | -------------------------------------------------------------------------------- /vehicles-ads-nestjs/test/jest-e2e.json: -------------------------------------------------------------------------------- 1 | { 2 | "moduleFileExtensions": ["js", "json", "ts"], 3 | "rootDir": ".", 4 | "testEnvironment": "node", 5 | "testRegex": ".e2e-spec.ts$", 6 | "transform": { 7 | "^.+\\.(t|j)s$": "ts-jest" 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /vehicles-ads-nestjs/tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "exclude": ["node_modules", "test", "dist", "**/*spec.ts"] 4 | } 5 | -------------------------------------------------------------------------------- /vehicles-ads-nestjs/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "declaration": true, 5 | "removeComments": true, 6 | "emitDecoratorMetadata": true, 7 | "experimentalDecorators": true, 8 | "allowSyntheticDefaultImports": true, 9 | "target": "es2017", 10 | "sourceMap": true, 11 | "outDir": "./dist", 12 | "baseUrl": "./", 13 | "incremental": true, 14 | "skipLibCheck": true, 15 | "strictNullChecks": false, 16 | "noImplicitAny": false, 17 | "strictBindCallApply": false, 18 | "forceConsistentCasingInFileNames": false, 19 | "noFallthroughCasesInSwitch": false, 20 | "resolveJsonModule": true 21 | }, 22 | "include": [ 23 | "src", 24 | "test" 25 | ] 26 | } -------------------------------------------------------------------------------- /vehicles-stock-nestjs/.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules -------------------------------------------------------------------------------- /vehicles-stock-nestjs/.env: -------------------------------------------------------------------------------- 1 | DATABASE_URL=postgres://postgres:postgres@ms-vehicles-stock-db:5432/stock 2 | KAFKA_BOOTSTRAP_SERVER=ms-kafka-1:9092 3 | SCHEMA_REGISTRY_URL=http://ms-schema-registry-balancer:8088 4 | -------------------------------------------------------------------------------- /vehicles-stock-nestjs/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | parser: '@typescript-eslint/parser', 3 | parserOptions: { 4 | project: 'tsconfig.json', 5 | tsconfigRootDir : __dirname, 6 | sourceType: 'module', 7 | }, 8 | plugins: ['@typescript-eslint/eslint-plugin'], 9 | extends: [ 10 | 'plugin:@typescript-eslint/recommended', 11 | 'plugin:prettier/recommended', 12 | ], 13 | root: true, 14 | env: { 15 | node: true, 16 | jest: true, 17 | }, 18 | ignorePatterns: ['.eslintrc.js'], 19 | rules: { 20 | '@typescript-eslint/interface-name-prefix': 'off', 21 | '@typescript-eslint/explicit-function-return-type': 'off', 22 | '@typescript-eslint/explicit-module-boundary-types': 'off', 23 | '@typescript-eslint/no-explicit-any': 'off', 24 | }, 25 | }; 26 | -------------------------------------------------------------------------------- /vehicles-stock-nestjs/.gitignore: -------------------------------------------------------------------------------- 1 | # compiled output 2 | /dist 3 | /node_modules 4 | 5 | # Logs 6 | logs 7 | *.log 8 | npm-debug.log* 9 | pnpm-debug.log* 10 | yarn-debug.log* 11 | yarn-error.log* 12 | lerna-debug.log* 13 | 14 | # OS 15 | .DS_Store 16 | 17 | # Tests 18 | /coverage 19 | /.nyc_output 20 | 21 | # IDEs and editors 22 | /.idea 23 | .project 24 | .classpath 25 | .c9/ 26 | *.launch 27 | .settings/ 28 | *.sublime-workspace 29 | 30 | # IDE - VSCode 31 | .vscode/* 32 | !.vscode/settings.json 33 | !.vscode/tasks.json 34 | !.vscode/launch.json 35 | !.vscode/extensions.json -------------------------------------------------------------------------------- /vehicles-stock-nestjs/.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "semi": false, 3 | "singleQuote": true, 4 | "trailingComma": "all" 5 | } -------------------------------------------------------------------------------- /vehicles-stock-nestjs/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:16.15-alpine 2 | 3 | COPY . . 4 | 5 | RUN wget -qO wait-for https://raw.githubusercontent.com/eficode/wait-for/master/wait-for 6 | RUN chmod +x wait-for 7 | RUN npm install 8 | RUN npx prisma generate 9 | RUN npm run build 10 | 11 | CMD ./wait-for http://ms-schema-registry-balancer:8088 -t 90 -- npm run start:prod 12 | -------------------------------------------------------------------------------- /vehicles-stock-nestjs/nest-cli.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://json.schemastore.org/nest-cli", 3 | "collection": "@nestjs/schematics", 4 | "sourceRoot": "src" 5 | } 6 | -------------------------------------------------------------------------------- /vehicles-stock-nestjs/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "vehicles-stock", 3 | "version": "0.0.1", 4 | "description": "", 5 | "author": "", 6 | "private": true, 7 | "license": "UNLICENSED", 8 | "scripts": { 9 | "prebuild": "rimraf dist", 10 | "build": "nest build", 11 | "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"", 12 | "start": "nest start", 13 | "start:dev": "nest start --watch", 14 | "start:debug": "nest start --debug --watch", 15 | "start:prod": "node dist/main", 16 | "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix", 17 | "test": "jest", 18 | "test:watch": "jest --watch", 19 | "test:cov": "jest --coverage", 20 | "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", 21 | "test:e2e": "jest --config ./test/jest-e2e.json", 22 | "db:init": "prisma migrate dev --name init" 23 | }, 24 | "dependencies": { 25 | "@kafkajs/confluent-schema-registry": "^3.2.1", 26 | "@nestjs/common": "^8.4.7", 27 | "@nestjs/config": "^2.1.0", 28 | "@nestjs/core": "^8.4.7", 29 | "@nestjs/microservices": "^8.4.7", 30 | "@nestjs/platform-express": "^8.4.7", 31 | "@prisma/client": "^4.0.0", 32 | "class-transformer": "^0.5.1", 33 | "class-validator": "^0.13.2", 34 | "kafkajs": "^2.1.0", 35 | "reflect-metadata": "^0.1.13", 36 | "rimraf": "^3.0.2", 37 | "rxjs": "^7.2.0" 38 | }, 39 | "devDependencies": { 40 | "@nestjs/cli": "^8.2.8", 41 | "@nestjs/schematics": "^8.0.11", 42 | "@nestjs/testing": "^8.4.7", 43 | "@types/express": "^4.17.13", 44 | "@types/jest": "27.5.0", 45 | "@types/node": "^16.0.0", 46 | "@types/supertest": "^2.0.11", 47 | "@typescript-eslint/eslint-plugin": "^5.0.0", 48 | "@typescript-eslint/parser": "^5.0.0", 49 | "eslint": "^8.0.1", 50 | "eslint-config-prettier": "^8.3.0", 51 | "eslint-plugin-prettier": "^4.0.0", 52 | "jest": "28.0.3", 53 | "prettier": "^2.3.2", 54 | "prisma": "^4.0.0", 55 | "source-map-support": "^0.5.20", 56 | "supertest": "^6.1.3", 57 | "ts-jest": "28.0.1", 58 | "ts-loader": "^9.2.3", 59 | "ts-node": "^10.0.0", 60 | "tsconfig-paths": "4.0.0", 61 | "typescript": "^4.3.5" 62 | }, 63 | "prisma": { 64 | "seed": "ts-node prisma/seed.ts" 65 | }, 66 | "jest": { 67 | "moduleFileExtensions": [ 68 | "js", 69 | "json", 70 | "ts" 71 | ], 72 | "rootDir": "src", 73 | "testRegex": ".*\\.spec\\.ts$", 74 | "transform": { 75 | "^.+\\.(t|j)s$": "ts-jest" 76 | }, 77 | "collectCoverageFrom": [ 78 | "**/*.(t|j)s" 79 | ], 80 | "coverageDirectory": "../coverage", 81 | "testEnvironment": "node" 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /vehicles-stock-nestjs/prisma/schema.prisma: -------------------------------------------------------------------------------- 1 | datasource db { 2 | url = env("DATABASE_URL") 3 | provider = "postgresql" 4 | } 5 | 6 | generator client { 7 | provider = "prisma-client-js" 8 | } 9 | 10 | model Vehicle { 11 | id Int @id @default(autoincrement()) 12 | createdAt DateTime @default(now()) 13 | vin String @unique 14 | model String 15 | year Int 16 | manufacturer String 17 | odometer Float 18 | odometerUnit String 19 | 20 | @@map("Vehicles") 21 | } 22 | -------------------------------------------------------------------------------- /vehicles-stock-nestjs/src/app.controller.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Body, 3 | Controller, 4 | Get, 5 | Param, 6 | ParseIntPipe, 7 | Post, 8 | } from '@nestjs/common' 9 | import { PrismaService } from './services/prisma.service' 10 | import { VehicleCreatedProducer } from './services/vehicle-created-producer.service' 11 | import { CreateVehicleDto } from './dtos/create-vehicle.dto' 12 | 13 | @Controller('stock') 14 | export class AppController { 15 | constructor( 16 | private readonly prisma: PrismaService, 17 | private readonly vehicleCreatedProducer: VehicleCreatedProducer, 18 | ) {} 19 | 20 | @Get() 21 | getVehicles() { 22 | return this.prisma.vehicle.findMany({}) 23 | } 24 | 25 | @Get(':id') 26 | getVehicleById(@Param('id', ParseIntPipe) vehicleId: number) { 27 | return this.prisma.vehicle.findFirst({ 28 | where: { id: vehicleId }, 29 | }) 30 | } 31 | 32 | @Post() 33 | async createVehicle(@Body() createVehicleDto: CreateVehicleDto) { 34 | const vehicle = await this.prisma.vehicle.create({ 35 | data: createVehicleDto, 36 | }) 37 | 38 | this.vehicleCreatedProducer.emit(vehicle).catch((error) => { 39 | console.log('ERROR emitting:', vehicle, error) 40 | }) 41 | 42 | return vehicle 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /vehicles-stock-nestjs/src/app.module.ts: -------------------------------------------------------------------------------- 1 | import { Inject, Module, OnApplicationBootstrap } from '@nestjs/common' 2 | import { ClientKafka, ClientsModule, Transport } from '@nestjs/microservices' 3 | import { ConfigModule, ConfigService } from '@nestjs/config' 4 | import { SchemaRegistry } from '@kafkajs/confluent-schema-registry' 5 | import { PrismaService } from './services/prisma.service' 6 | import { VehicleCreatedProducer } from './services/vehicle-created-producer.service' 7 | import { AppController } from './app.controller' 8 | import configuration from './config' 9 | import { KAFKA_SERVICE } from './constants' 10 | 11 | const imports = [ 12 | ConfigModule.forRoot({ 13 | load: [configuration], 14 | isGlobal: true, 15 | }), 16 | ClientsModule.registerAsync([ 17 | { 18 | name: KAFKA_SERVICE, 19 | useFactory: (config: ConfigService) => ({ 20 | transport: Transport.KAFKA, 21 | options: { 22 | client: { 23 | clientId: 'vehicles-stock-app', 24 | brokers: [config.get('kafka.bootstrapServer')], 25 | }, 26 | producer: { idempotent: true, allowAutoTopicCreation: true }, 27 | consumer: { groupId: 'ms-vehicles-stock' }, 28 | }, 29 | }), 30 | inject: [ConfigService], 31 | }, 32 | ]), 33 | ] 34 | 35 | const providers = [ 36 | PrismaService, 37 | { 38 | provide: SchemaRegistry, 39 | useFactory: (config: ConfigService) => 40 | new SchemaRegistry({ host: config.get('kafka.schemaRegistry') }), 41 | inject: [ConfigService], 42 | }, 43 | VehicleCreatedProducer, 44 | ] 45 | 46 | // Note: this app is only a producer 47 | @Module({ 48 | imports, 49 | controllers: [AppController], 50 | providers, 51 | }) 52 | export class AppModule implements OnApplicationBootstrap { 53 | constructor( 54 | @Inject(KAFKA_SERVICE) private readonly kafkaClient: ClientKafka, 55 | ) {} 56 | 57 | // Connect to Kafka immediately, not lazily at the first microservice call 58 | // https://docs.nestjs.com/microservices/basics#client 59 | async onApplicationBootstrap() { 60 | await this.kafkaClient.connect() 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /vehicles-stock-nestjs/src/config.ts: -------------------------------------------------------------------------------- 1 | export default () => ({ 2 | kafka: { 3 | bootstrapServer: process.env.KAFKA_BOOTSTRAP_SERVER, 4 | schemaRegistry: process.env.SCHEMA_REGISTRY_URL, 5 | }, 6 | database: { 7 | url: process.env.DATABASE_URL, 8 | }, 9 | }) 10 | -------------------------------------------------------------------------------- /vehicles-stock-nestjs/src/constants.ts: -------------------------------------------------------------------------------- 1 | export const KAFKA_SERVICE = 'KAFKA_SERVICE_TOKEN' 2 | export const VEHICLE_CREATED_TOPIC = 'stock.vehicle-created' 3 | -------------------------------------------------------------------------------- /vehicles-stock-nestjs/src/dtos/create-vehicle.dto.ts: -------------------------------------------------------------------------------- 1 | import { 2 | IsIn, 3 | IsNotEmpty, 4 | IsPositive, 5 | IsString, 6 | Max, 7 | Min, 8 | } from 'class-validator' 9 | 10 | export class CreateVehicleDto { 11 | @IsString() 12 | @IsNotEmpty() 13 | vin: string 14 | 15 | @IsString() 16 | @IsNotEmpty() 17 | model: string 18 | 19 | @Min(1900) 20 | @Max(new Date().getFullYear()) 21 | year: number 22 | 23 | @IsString() 24 | @IsNotEmpty() 25 | manufacturer: string 26 | 27 | @IsPositive() 28 | odometer: number 29 | 30 | @IsIn(['km', 'mi']) 31 | @IsNotEmpty() 32 | odometerUnit: string 33 | } 34 | -------------------------------------------------------------------------------- /vehicles-stock-nestjs/src/main.ts: -------------------------------------------------------------------------------- 1 | import { ValidationPipe } from '@nestjs/common' 2 | import { NestFactory } from '@nestjs/core' 3 | import { AppModule } from './app.module' 4 | 5 | const DEFAULT_PORT = 3002 6 | 7 | async function bootstrap() { 8 | const app = await NestFactory.create(AppModule) 9 | 10 | app.useGlobalPipes(new ValidationPipe()) 11 | 12 | await app.listen(process.env.PORT || DEFAULT_PORT) 13 | } 14 | 15 | bootstrap() 16 | -------------------------------------------------------------------------------- /vehicles-stock-nestjs/src/schemas/vehicle-stock.json: -------------------------------------------------------------------------------- 1 | { 2 | "$id": "nestjs-python-kafka/vehicle-stock.schema.json", 3 | "type": "object", 4 | "required": [ 5 | "id", 6 | "vin", 7 | "model", 8 | "year", 9 | "manufacturer", 10 | "odometer", 11 | "odometerUnit" 12 | ], 13 | "properties": { 14 | "id": { 15 | "type": "number" 16 | }, 17 | "vin": { 18 | "type": "string" 19 | }, 20 | "model": { 21 | "type": "string" 22 | }, 23 | "year": { 24 | "type": "number" 25 | }, 26 | "manufacturer": { 27 | "type": "string" 28 | }, 29 | "odometer": { 30 | "type": "number" 31 | }, 32 | "odometerUnit": { 33 | "enum": [ 34 | "km", 35 | "mi" 36 | ] 37 | } 38 | } 39 | } -------------------------------------------------------------------------------- /vehicles-stock-nestjs/src/services/prisma.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable, OnModuleInit, INestApplication } from '@nestjs/common' 2 | import { PrismaClient } from '@prisma/client' 3 | 4 | @Injectable() 5 | export class PrismaService extends PrismaClient implements OnModuleInit { 6 | async onModuleInit() { 7 | await this.$connect() 8 | } 9 | 10 | async enableShutdownHooks(app: INestApplication) { 11 | this.$on('beforeExit', async () => { 12 | await app.close() 13 | }) 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /vehicles-stock-nestjs/src/services/vehicle-created-producer.service.ts: -------------------------------------------------------------------------------- 1 | import { Inject, Injectable, OnModuleInit } from '@nestjs/common' 2 | import { ClientKafka } from '@nestjs/microservices' 3 | import { SchemaRegistry, SchemaType } from '@kafkajs/confluent-schema-registry' 4 | import { KAFKA_SERVICE, VEHICLE_CREATED_TOPIC } from 'src/constants' 5 | import * as vehicleStockSchema from '../schemas/vehicle-stock.json' 6 | 7 | @Injectable() 8 | export class VehicleCreatedProducer implements OnModuleInit { 9 | schemaId: number 10 | 11 | constructor( 12 | @Inject(KAFKA_SERVICE) private readonly kafkaClient: ClientKafka, 13 | private readonly schemaRegistry: SchemaRegistry, 14 | ) {} 15 | 16 | async onModuleInit() { 17 | // Register the schema 18 | const { id } = await this.schemaRegistry.register( 19 | { type: SchemaType.JSON, schema: JSON.stringify(vehicleStockSchema) }, 20 | { subject: `${VEHICLE_CREATED_TOPIC}-value` }, 21 | ) 22 | this.schemaId = id 23 | } 24 | 25 | async serialize(message: any) { 26 | const value = await this.schemaRegistry.encode(this.schemaId, message) 27 | return { value, headers: {} } 28 | } 29 | 30 | async emit(message: any) { 31 | const value = await this.serialize(message) 32 | return this.kafkaClient.emit(VEHICLE_CREATED_TOPIC, value) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /vehicles-stock-nestjs/test/app.e2e-spec.ts: -------------------------------------------------------------------------------- 1 | import { Test, TestingModule } from '@nestjs/testing' 2 | import { INestApplication } from '@nestjs/common' 3 | import * as request from 'supertest' 4 | import { AppModule } from './../src/app.module' 5 | 6 | describe('AppController (e2e)', () => { 7 | let app: INestApplication 8 | 9 | beforeEach(async () => { 10 | const moduleFixture: TestingModule = await Test.createTestingModule({ 11 | imports: [AppModule], 12 | }).compile() 13 | 14 | app = moduleFixture.createNestApplication() 15 | await app.init() 16 | }) 17 | 18 | it('/ (GET)', () => { 19 | return request(app.getHttpServer()) 20 | .get('/') 21 | .expect(200) 22 | .expect('Hello World!') 23 | }) 24 | }) 25 | -------------------------------------------------------------------------------- /vehicles-stock-nestjs/test/jest-e2e.json: -------------------------------------------------------------------------------- 1 | { 2 | "moduleFileExtensions": ["js", "json", "ts"], 3 | "rootDir": ".", 4 | "testEnvironment": "node", 5 | "testRegex": ".e2e-spec.ts$", 6 | "transform": { 7 | "^.+\\.(t|j)s$": "ts-jest" 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /vehicles-stock-nestjs/tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "exclude": ["node_modules", "test", "dist", "**/*spec.ts"] 4 | } 5 | -------------------------------------------------------------------------------- /vehicles-stock-nestjs/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "declaration": true, 5 | "removeComments": true, 6 | "emitDecoratorMetadata": true, 7 | "experimentalDecorators": true, 8 | "allowSyntheticDefaultImports": true, 9 | "target": "es2017", 10 | "sourceMap": true, 11 | "outDir": "./dist", 12 | "baseUrl": "./", 13 | "incremental": true, 14 | "skipLibCheck": true, 15 | "strictNullChecks": false, 16 | "noImplicitAny": false, 17 | "strictBindCallApply": false, 18 | "forceConsistentCasingInFileNames": false, 19 | "noFallthroughCasesInSwitch": false, 20 | "resolveJsonModule": true 21 | }, 22 | "include": [ 23 | "src", 24 | "test" 25 | ] 26 | } --------------------------------------------------------------------------------