├── assets └── images │ ├── cs1.png │ ├── cs2.png │ ├── cs3.png │ ├── dag.png │ ├── det.png │ ├── fs.png │ ├── tn.png │ ├── arch.png │ ├── det2.png │ ├── flink.png │ ├── graphana.png │ ├── proj_1.png │ ├── proj_2.png │ ├── secret.png │ ├── data_infra.png │ └── flink_ui_dag.png ├── container ├── datagen │ ├── requirements.txt │ └── Dockerfile └── flink │ ├── requirements.txt │ └── Dockerfile ├── grafana └── provisioning │ ├── dashboards │ ├── dashboards.yml │ └── Flink.json │ └── datasources │ └── prometheus.yml ├── prometheus └── prometheus.yml ├── code ├── source │ ├── users.sql │ ├── clicks.sql │ └── checkouts.sql ├── sink │ └── attributed_checkouts.sql ├── process │ └── attribute_checkouts.sql └── checkout_attribution.py ├── postgres └── init.sql ├── Makefile ├── docker-compose.yml ├── .gitignore ├── datagen └── gen_fake_data.py └── README.md /assets/images/cs1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmachado/beginner_de_project_stream/HEAD/assets/images/cs1.png -------------------------------------------------------------------------------- /assets/images/cs2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmachado/beginner_de_project_stream/HEAD/assets/images/cs2.png -------------------------------------------------------------------------------- /assets/images/cs3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmachado/beginner_de_project_stream/HEAD/assets/images/cs3.png -------------------------------------------------------------------------------- /assets/images/dag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmachado/beginner_de_project_stream/HEAD/assets/images/dag.png -------------------------------------------------------------------------------- /assets/images/det.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmachado/beginner_de_project_stream/HEAD/assets/images/det.png -------------------------------------------------------------------------------- /assets/images/fs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmachado/beginner_de_project_stream/HEAD/assets/images/fs.png -------------------------------------------------------------------------------- /assets/images/tn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmachado/beginner_de_project_stream/HEAD/assets/images/tn.png -------------------------------------------------------------------------------- /container/datagen/requirements.txt: -------------------------------------------------------------------------------- 1 | black==22.8.0 2 | flake8==5.0.4 3 | mypy==0.971 4 | isort==5.10.1 5 | Jinja2==3.1.2 -------------------------------------------------------------------------------- /assets/images/arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmachado/beginner_de_project_stream/HEAD/assets/images/arch.png -------------------------------------------------------------------------------- /assets/images/det2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmachado/beginner_de_project_stream/HEAD/assets/images/det2.png -------------------------------------------------------------------------------- /assets/images/flink.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmachado/beginner_de_project_stream/HEAD/assets/images/flink.png -------------------------------------------------------------------------------- /assets/images/graphana.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmachado/beginner_de_project_stream/HEAD/assets/images/graphana.png -------------------------------------------------------------------------------- /assets/images/proj_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmachado/beginner_de_project_stream/HEAD/assets/images/proj_1.png -------------------------------------------------------------------------------- /assets/images/proj_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmachado/beginner_de_project_stream/HEAD/assets/images/proj_2.png -------------------------------------------------------------------------------- /assets/images/secret.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmachado/beginner_de_project_stream/HEAD/assets/images/secret.png -------------------------------------------------------------------------------- /assets/images/data_infra.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmachado/beginner_de_project_stream/HEAD/assets/images/data_infra.png -------------------------------------------------------------------------------- /assets/images/flink_ui_dag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmachado/beginner_de_project_stream/HEAD/assets/images/flink_ui_dag.png -------------------------------------------------------------------------------- /container/flink/requirements.txt: -------------------------------------------------------------------------------- 1 | black==22.8.0 2 | flake8==5.0.4 3 | mypy==0.971 4 | isort==5.10.1 5 | Jinja2==3.1.2 6 | apache-flink==1.17.0 -------------------------------------------------------------------------------- /grafana/provisioning/dashboards/dashboards.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: 1 3 | 4 | providers: 5 | - name: 'default' 6 | orgId: 1 7 | folder: '' 8 | type: file 9 | disableDeletion: true 10 | editable: false 11 | updateIntervalSeconds: 3 12 | options: 13 | path: /etc/grafana/provisioning/dashboards -------------------------------------------------------------------------------- /prometheus/prometheus.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 5s 3 | evaluation_interval: 5s 4 | 5 | scrape_configs: 6 | - job_name: 'flink-jobmanager' 7 | static_configs: 8 | - targets: ['jobmanager:9249'] 9 | 10 | - job_name: 'flink-taskmanager' 11 | static_configs: 12 | - targets: ['taskmanager:9249'] -------------------------------------------------------------------------------- /grafana/provisioning/datasources/prometheus.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: 1 3 | 4 | deleteDatasources: 5 | - name: Prometheus 6 | orgId: 1 7 | 8 | datasources: 9 | - name: Prometheus 10 | type: prometheus 11 | access: proxy 12 | orgId: 1 13 | url: http://prometheus:9090 14 | isDefault: true 15 | version: 1 16 | editable: false -------------------------------------------------------------------------------- /container/datagen/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7-slim 2 | 3 | RUN pip install \ 4 | psycopg2-binary==2.9.3 \ 5 | faker==13.3.2 \ 6 | confluent-kafka 7 | 8 | 9 | WORKDIR /opt/datagen 10 | 11 | COPY requirements.txt /opt/datagen/ 12 | 13 | RUN pip install --no-cache-dir -r /opt/datagen/requirements.txt 14 | 15 | CMD ["tail" "-F" "anything"] -------------------------------------------------------------------------------- /code/source/users.sql: -------------------------------------------------------------------------------- 1 | CREATE TEMPORARY TABLE users ( 2 | id INT, 3 | username STRING, 4 | PASSWORD STRING, 5 | PRIMARY KEY (id) NOT ENFORCED 6 | ) WITH ( 7 | 'connector' = '{{ connector }}', 8 | 'url' = '{{ url }}', 9 | 'table-name' = '{{ table_name }}', 10 | 'username' = '{{ username }}', 11 | 'password' = '{{ password }}', 12 | 'driver' = '{{ driver }}' 13 | ); -------------------------------------------------------------------------------- /code/source/clicks.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE clicks ( 2 | click_id STRING, 3 | user_id INT, 4 | product_id STRING, 5 | product STRING, 6 | price DOUBLE, 7 | url STRING, 8 | user_agent STRING, 9 | ip_address STRING, 10 | datetime_occured TIMESTAMP(3), 11 | processing_time AS PROCTIME(), 12 | WATERMARK FOR datetime_occured AS datetime_occured - INTERVAL '15' SECOND 13 | ) WITH ( 14 | 'connector' = '{{ connector }}', 15 | 'topic' = '{{ topic }}', 16 | 'properties.bootstrap.servers' = '{{ bootstrap_servers }}', 17 | 'properties.group.id' = '{{ consumer_group_id }}', 18 | 'scan.startup.mode' = '{{ scan_stratup_mode }}', 19 | 'format' = '{{ format }}' 20 | ); -------------------------------------------------------------------------------- /code/sink/attributed_checkouts.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE attributed_checkouts ( 2 | checkout_id STRING, 3 | user_name STRING, 4 | click_id STRING, 5 | product_id STRING, 6 | payment_method STRING, 7 | total_amount DECIMAL(5, 2), 8 | shipping_address STRING, 9 | billing_address STRING, 10 | user_agent STRING, 11 | ip_address STRING, 12 | checkout_time TIMESTAMP(3), 13 | click_time TIMESTAMP, 14 | PRIMARY KEY (checkout_id) NOT ENFORCED 15 | ) WITH ( 16 | 'connector' = 'jdbc', 17 | 'url' = 'jdbc:postgresql://postgres:5432/postgres', 18 | 'table-name' = 'commerce.attributed_checkouts', 19 | 'username' = 'postgres', 20 | 'password' = 'postgres', 21 | 'driver' = 'org.postgresql.Driver' 22 | ) -------------------------------------------------------------------------------- /code/source/checkouts.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE checkouts ( 2 | checkout_id STRING, 3 | user_id INT, 4 | product_id STRING, 5 | payment_method STRING, 6 | total_amount DECIMAL(5, 2), 7 | shipping_address STRING, 8 | billing_address STRING, 9 | user_agent STRING, 10 | ip_address STRING, 11 | datetime_occured TIMESTAMP(3), 12 | processing_time AS PROCTIME(), 13 | WATERMARK FOR datetime_occured AS datetime_occured - INTERVAL '15' SECOND 14 | ) WITH ( 15 | 'connector' = '{{ connector }}', 16 | 'topic' = '{{ topic }}', 17 | 'properties.bootstrap.servers' = '{{ bootstrap_servers }}', 18 | 'properties.group.id' = '{{ consumer_group_id }}', 19 | 'scan.startup.mode' = '{{ scan_stratup_mode }}', 20 | 'format' = '{{ format }}' 21 | ); -------------------------------------------------------------------------------- /container/flink/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM flink:1.17.0 2 | 3 | 4 | # install python3 and pip3 5 | RUN apt-get update -y && \ 6 | apt-get install -y python3 python3-pip python3-dev && rm -rf /var/lib/apt/lists/* 7 | RUN ln -s /usr/bin/python3 /usr/bin/python 8 | 9 | RUN wget https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-connector-kafka/1.17.0/flink-sql-connector-kafka-1.17.0.jar && wget https://repo.maven.apache.org/maven2/org/apache/flink/flink-connector-jdbc/3.0.0-1.16/flink-connector-jdbc-3.0.0-1.16.jar && wget https://jdbc.postgresql.org/download/postgresql-42.6.0.jar 10 | 11 | RUN echo "metrics.reporters: prom" >> "$FLINK_HOME/conf/flink-conf.yaml"; \ 12 | echo "metrics.reporter.prom.factory.class: org.apache.flink.metrics.prometheus.PrometheusReporterFactory" >> "$FLINK_HOME/conf/flink-conf.yaml" 13 | 14 | COPY requirements.txt /opt/flink/ 15 | 16 | # Install py dependencies 17 | RUN pip install --no-cache-dir -r /opt/flink/requirements.txt -------------------------------------------------------------------------------- /postgres/init.sql: -------------------------------------------------------------------------------- 1 | -- create a commerce schema 2 | CREATE SCHEMA commerce; 3 | 4 | -- Use commerce schema 5 | SET 6 | search_path TO commerce; 7 | 8 | -- create a table named products 9 | CREATE TABLE products ( 10 | id int PRIMARY KEY, 11 | name VARCHAR(255) NOT NULL, 12 | description TEXT, 13 | price REAL NOT NULL 14 | ); 15 | 16 | -- create a users table 17 | CREATE TABLE users ( 18 | id int PRIMARY KEY, 19 | username VARCHAR(255) NOT NULL, 20 | PASSWORD VARCHAR(255) NOT NULL 21 | ); 22 | 23 | ALTER TABLE 24 | products REPLICA IDENTITY FULL; 25 | 26 | ALTER TABLE 27 | users REPLICA IDENTITY FULL; 28 | 29 | CREATE TABLE attributed_checkouts ( 30 | checkout_id VARCHAR PRIMARY KEY, 31 | user_name VARCHAR, 32 | click_id VARCHAR, 33 | product_id VARCHAR, 34 | payment_method VARCHAR, 35 | total_amount DECIMAL(5, 2), 36 | shipping_address VARCHAR, 37 | billing_address VARCHAR, 38 | user_agent VARCHAR, 39 | ip_address VARCHAR, 40 | checkout_time TIMESTAMP, 41 | click_time TIMESTAMP 42 | ); -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | up: 2 | docker compose up --build -d 3 | 4 | down: 5 | docker compose down 6 | 7 | run-checkout-attribution-job: 8 | docker exec jobmanager ./bin/flink run --python ./code/checkout_attribution.py 9 | 10 | sleep: 11 | sleep 20 12 | 13 | #################################################################################################################### 14 | # Testing, auto formatting, type checks, & Lint checks 15 | 16 | format: 17 | docker exec datagen python -m black -S --line-length 79 . 18 | 19 | isort: 20 | docker exec datagen isort . 21 | 22 | type: 23 | docker exec datagen mypy --ignore-missing-imports --no-implicit-optional /opt 24 | 25 | lint: 26 | docker exec datagen flake8 /opt 27 | 28 | ci: isort format type lint 29 | 30 | #################################################################################################################### 31 | # Run ETL 32 | 33 | pyflink: 34 | docker exec -ti jobmanager ./bin/pyflink-shell.sh local 35 | 36 | run: down up sleep ci run-checkout-attribution-job 37 | 38 | #################################################################################################################### 39 | # Monitoring 40 | 41 | viz: 42 | open http://localhost:3000 43 | 44 | ui: 45 | open http://localhost:8081/ -------------------------------------------------------------------------------- /code/process/attribute_checkouts.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO 2 | attributed_checkouts 3 | SELECT 4 | checkout_id, 5 | user_name, 6 | click_id, 7 | product_id, 8 | payment_method, 9 | total_amount, 10 | shipping_address, 11 | billing_address, 12 | user_agent, 13 | ip_address, 14 | checkout_time, 15 | click_time 16 | FROM 17 | ( 18 | SELECT 19 | co.checkout_id, 20 | u.username AS user_name, 21 | cl.click_id, 22 | co.product_id, 23 | co.payment_method, 24 | co.total_amount, 25 | co.shipping_address, 26 | co.billing_address, 27 | co.user_agent, 28 | co.ip_address, 29 | co.datetime_occured AS checkout_time, 30 | cl.datetime_occured AS click_time, 31 | ROW_NUMBER() OVER ( 32 | PARTITION BY cl.user_id, 33 | cl.product_id 34 | ORDER BY 35 | cl.datetime_occured 36 | ) AS rn 37 | FROM 38 | checkouts AS co 39 | JOIN users FOR SYSTEM_TIME AS OF co.processing_time AS u ON co.user_id = u.id 40 | LEFT JOIN clicks AS cl ON co.user_id = cl.user_id 41 | AND co.product_id = cl.product_id 42 | AND co.datetime_occured BETWEEN cl.datetime_occured 43 | AND cl.datetime_occured + INTERVAL '1' HOUR 44 | ) 45 | WHERE 46 | rn = 1; -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "2.2" 2 | services: 3 | 4 | jobmanager: 5 | container_name: jobmanager 6 | build: 7 | context: ./container/flink/ 8 | ports: 9 | - "8081:8081" 10 | - "9249:9249" 11 | command: jobmanager 12 | volumes: 13 | - ./code:/opt/flink/code 14 | environment: 15 | - | 16 | FLINK_PROPERTIES= 17 | jobmanager.rpc.address: jobmanager 18 | 19 | taskmanager: 20 | container_name: taskmanager 21 | build: 22 | context: ./container/flink/ 23 | depends_on: 24 | - jobmanager 25 | command: taskmanager 26 | ports: 27 | - "9250:9249" 28 | volumes: 29 | - ./code:/opt/flink/code 30 | scale: 1 31 | environment: 32 | - | 33 | FLINK_PROPERTIES= 34 | jobmanager.rpc.address: jobmanager 35 | taskmanager.numberOfTaskSlots: 2 36 | 37 | prometheus: 38 | image: prom/prometheus:v2.37.1 39 | container_name: prometheus 40 | ports: 41 | - "9090:9090" 42 | volumes: 43 | - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml 44 | 45 | grafana: 46 | image: grafana/grafana:8.4.0 47 | container_name: grafana 48 | ports: 49 | - "3000:3000" 50 | environment: 51 | - GF_SECURITY_ADMIN_PASSWORD=flink 52 | volumes: 53 | - ./grafana/provisioning/:/etc/grafana/provisioning/ 54 | 55 | zookeeper: 56 | image: docker.io/bitnami/zookeeper:3.8 57 | container_name: zookeeper 58 | ports: 59 | - "2181:2181" 60 | environment: 61 | - ALLOW_ANONYMOUS_LOGIN=yes 62 | 63 | kafka: 64 | image: docker.io/bitnami/kafka:3.4 65 | container_name: kafka 66 | ports: 67 | - "9093:9093" 68 | environment: 69 | - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181 70 | - KAFKA_ADVERTISED_LISTENERS=INSIDE://:9092,OUTSIDE://:9093 71 | - KAFKA_CFG_LISTENERS=INSIDE://:9092,OUTSIDE://:9093 72 | - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT 73 | - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INSIDE 74 | - ALLOW_PLAINTEXT_LISTENER=yes 75 | depends_on: 76 | - zookeeper 77 | 78 | postgres: 79 | image: debezium/postgres:15 80 | container_name: postgres 81 | hostname: postgres 82 | environment: 83 | - POSTGRES_USER=postgres 84 | - POSTGRES_DB=postgres 85 | - POSTGRES_PASSWORD=postgres 86 | ports: 87 | - "5432:5432" 88 | volumes: 89 | - ./postgres/init.sql:/docker-entrypoint-initdb.d/init.sql 90 | 91 | datagen: 92 | build: 93 | context: ./container/datagen/ 94 | command: python /opt/datagen/gen_fake_data.py 95 | volumes: 96 | - ./code:/opt/datagen/code 97 | - ./datagen:/opt/datagen 98 | container_name: datagen 99 | restart: on-failure 100 | depends_on: 101 | - postgres 102 | - kafka 103 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | *.pyc 131 | **/*.pyc 132 | 133 | 134 | # Spyder project settings 135 | .spyderproject 136 | .spyproject 137 | 138 | # Rope project settings 139 | .ropeproject 140 | 141 | # mkdocs documentation 142 | /site 143 | 144 | # mypy 145 | .mypy_cache/ 146 | .dmypy.json 147 | dmypy.json 148 | 149 | # Pyre type checker 150 | .pyre/ 151 | 152 | # pytype static type analyzer 153 | .pytype/ 154 | 155 | # Cython debug symbols 156 | cython_debug/ 157 | 158 | # PyCharm 159 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 160 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 161 | # and can be added to the global gitignore or merged into this file. For a more nuclear 162 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 163 | #.idea/ 164 | 165 | data/*.db 166 | 167 | .ruff_cache* 168 | 169 | *kafka* -------------------------------------------------------------------------------- /code/checkout_attribution.py: -------------------------------------------------------------------------------- 1 | from dataclasses import asdict, dataclass, field 2 | from typing import List, Tuple 3 | 4 | from jinja2 import Environment, FileSystemLoader 5 | from pyflink.datastream import StreamExecutionEnvironment 6 | from pyflink.table import StreamTableEnvironment 7 | 8 | # dependency jars to read data from kafka, and connect to postgres 9 | REQUIRED_JARS = [ 10 | "file:///opt/flink/flink-sql-connector-kafka-1.17.0.jar", 11 | "file:///opt/flink/flink-connector-jdbc-3.0.0-1.16.jar", 12 | "file:///opt/flink/postgresql-42.6.0.jar", 13 | ] 14 | 15 | 16 | @dataclass(frozen=True) 17 | class StreamJobConfig: 18 | job_name: str = 'checkout-attribution-job' 19 | jars: List[str] = field(default_factory=lambda: REQUIRED_JARS) 20 | checkpoint_interval: int = 10 21 | checkpoint_pause: int = 5 22 | checkpoint_timeout: int = 5 23 | parallelism: int = 2 24 | 25 | 26 | @dataclass(frozen=True) 27 | class KafkaConfig: 28 | connector: str = 'kafka' 29 | bootstrap_servers: str = 'kafka:9092' 30 | scan_stratup_mode: str = 'earliest-offset' 31 | consumer_group_id: str = 'flink-consumer-group-1' 32 | 33 | 34 | @dataclass(frozen=True) 35 | class ClickTopicConfig(KafkaConfig): 36 | topic: str = 'clicks' 37 | format: str = 'json' 38 | 39 | 40 | @dataclass(frozen=True) 41 | class CheckoutTopicConfig(KafkaConfig): 42 | topic: str = 'checkouts' 43 | format: str = 'json' 44 | 45 | 46 | @dataclass(frozen=True) 47 | class ApplicationDatabaseConfig: 48 | connector: str = 'jdbc' 49 | url: str = 'jdbc:postgresql://postgres:5432/postgres' 50 | username: str = 'postgres' 51 | password: str = 'postgres' 52 | driver: str = 'org.postgresql.Driver' 53 | 54 | 55 | @dataclass(frozen=True) 56 | class ApplicationUsersTableConfig(ApplicationDatabaseConfig): 57 | table_name: str = 'commerce.users' 58 | 59 | 60 | @dataclass(frozen=True) 61 | class ApplicationAttributedCheckoutsTableConfig(ApplicationDatabaseConfig): 62 | table_name: str = 'commerce.attributed_checkouts' 63 | 64 | 65 | def get_execution_environment( 66 | config: StreamJobConfig, 67 | ) -> Tuple[StreamExecutionEnvironment, StreamTableEnvironment]: 68 | s_env = StreamExecutionEnvironment.get_execution_environment() 69 | for jar in config.jars: 70 | s_env.add_jars(jar) 71 | # start a checkpoint every 10,000 ms (10 s) 72 | s_env.enable_checkpointing(config.checkpoint_interval * 1000) 73 | # make sure 5000 ms (5 s) of progress happen between checkpoints 74 | s_env.get_checkpoint_config().set_min_pause_between_checkpoints( 75 | config.checkpoint_pause * 1000 76 | ) 77 | # checkpoints have to complete within 5 minute, or are discarded 78 | s_env.get_checkpoint_config().set_checkpoint_timeout( 79 | config.checkpoint_timeout * 1000 80 | ) 81 | execution_config = s_env.get_config() 82 | execution_config.set_parallelism(config.parallelism) 83 | t_env = StreamTableEnvironment.create(s_env) 84 | job_config = t_env.get_config().get_configuration() 85 | job_config.set_string("pipeline.name", config.job_name) 86 | return s_env, t_env 87 | 88 | 89 | def get_sql_query( 90 | entity: str, 91 | type: str = 'source', 92 | template_env: Environment = Environment(loader=FileSystemLoader("code/")), 93 | ) -> str: 94 | config_map = { 95 | 'clicks': ClickTopicConfig(), 96 | 'checkouts': CheckoutTopicConfig(), 97 | 'users': ApplicationUsersTableConfig(), 98 | 'attributed_checkouts': ApplicationUsersTableConfig(), 99 | 'attribute_checkouts': ApplicationAttributedCheckoutsTableConfig(), 100 | } 101 | 102 | return template_env.get_template(f"{type}/{entity}.sql").render( 103 | asdict(config_map.get(entity)) 104 | ) 105 | 106 | 107 | def run_checkout_attribution_job( 108 | t_env: StreamTableEnvironment, 109 | get_sql_query=get_sql_query, 110 | ) -> None: 111 | # Create Source DDLs 112 | t_env.execute_sql(get_sql_query('clicks')) 113 | t_env.execute_sql(get_sql_query('checkouts')) 114 | t_env.execute_sql(get_sql_query('users')) 115 | 116 | # Create Sink DDL 117 | t_env.execute_sql(get_sql_query('attributed_checkouts', 'sink')) 118 | 119 | # Run processing query 120 | stmt_set = t_env.create_statement_set() 121 | stmt_set.add_insert_sql(get_sql_query('attribute_checkouts', 'process')) 122 | 123 | checkout_attribution_job = stmt_set.execute() 124 | print( 125 | f""" 126 | Async attributed checkouts sink job 127 | status: {checkout_attribution_job.get_job_client().get_job_status()} 128 | """ 129 | ) 130 | 131 | 132 | if __name__ == '__main__': 133 | _, t_env = get_execution_environment(StreamJobConfig()) 134 | run_checkout_attribution_job(t_env) 135 | -------------------------------------------------------------------------------- /datagen/gen_fake_data.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import random 4 | from datetime import datetime 5 | from uuid import uuid4 6 | 7 | import psycopg2 8 | from confluent_kafka import Producer 9 | from faker import Faker 10 | 11 | fake = Faker() 12 | 13 | 14 | # Generate user data 15 | def gen_user_data(num_user_records: int) -> None: 16 | for id in range(num_user_records): 17 | conn = psycopg2.connect( 18 | dbname="postgres", 19 | user="postgres", 20 | password="postgres", 21 | host="postgres", 22 | ) 23 | curr = conn.cursor() 24 | curr.execute( 25 | """INSERT INTO commerce.users 26 | (id, username, password) VALUES (%s, %s, %s)""", 27 | (id, fake.user_name(), fake.password()), 28 | ) 29 | curr.execute( 30 | """INSERT INTO commerce.products 31 | (id, name, description, price) VALUES (%s, %s, %s, %s)""", 32 | (id, fake.name(), fake.text(), fake.random_int(min=1, max=100)), 33 | ) 34 | conn.commit() 35 | 36 | # update 10 % of the time 37 | if random.randint(1, 100) >= 90: 38 | curr.execute( 39 | "UPDATE commerce.users SET username = %s WHERE id = %s", 40 | (fake.user_name(), id), 41 | ) 42 | curr.execute( 43 | "UPDATE commerce.products SET name = %s WHERE id = %s", 44 | (fake.name(), id), 45 | ) 46 | conn.commit() 47 | curr.close() 48 | return 49 | 50 | 51 | # Stream clicks and checkouts data 52 | 53 | 54 | # Generate a random user agent string 55 | def random_user_agent(): 56 | return fake.user_agent() 57 | 58 | 59 | # Generate a random IP address 60 | def random_ip(): 61 | return fake.ipv4() 62 | 63 | 64 | # Generate a click event with the current datetime_occured 65 | def generate_click_event(user_id, product_id=None): 66 | click_id = str(uuid4()) 67 | product_id = product_id or str(uuid4()) 68 | product = fake.word() 69 | price = fake.pyfloat(left_digits=2, right_digits=2, positive=True) 70 | url = fake.uri() 71 | user_agent = random_user_agent() 72 | ip_address = random_ip() 73 | datetime_occured = datetime.now() 74 | 75 | click_event = { 76 | "click_id": click_id, 77 | "user_id": user_id, 78 | "product_id": product_id, 79 | "product": product, 80 | "price": price, 81 | "url": url, 82 | "user_agent": user_agent, 83 | "ip_address": ip_address, 84 | "datetime_occured": datetime_occured.strftime("%Y-%m-%d %H:%M:%S.%f")[ 85 | :-3 86 | ], 87 | } 88 | 89 | return click_event 90 | 91 | 92 | # Generate a checkout event with the current datetime_occured 93 | def generate_checkout_event(user_id, product_id): 94 | payment_method = fake.credit_card_provider() 95 | total_amount = fake.pyfloat(left_digits=3, right_digits=2, positive=True) 96 | shipping_address = fake.address() 97 | billing_address = fake.address() 98 | user_agent = random_user_agent() 99 | ip_address = random_ip() 100 | datetime_occured = datetime.now() 101 | 102 | checkout_event = { 103 | "checkout_id": str(uuid4()), 104 | "user_id": user_id, 105 | "product_id": product_id, 106 | "payment_method": payment_method, 107 | "total_amount": total_amount, 108 | "shipping_address": shipping_address, 109 | "billing_address": billing_address, 110 | "user_agent": user_agent, 111 | "ip_address": ip_address, 112 | "datetime_occured": datetime_occured.strftime("%Y-%m-%d %H:%M:%S.%f")[ 113 | :-3 114 | ], 115 | } 116 | 117 | return checkout_event 118 | 119 | 120 | # Function to push the events to a Kafka topic 121 | def push_to_kafka(event, topic): 122 | producer = Producer({'bootstrap.servers': 'kafka:9092'}) 123 | producer.produce(topic, json.dumps(event).encode('utf-8')) 124 | producer.flush() 125 | 126 | 127 | def gen_clickstream_data(num_click_records: int) -> None: 128 | for _ in range(num_click_records): 129 | user_id = random.randint(1, 100) 130 | click_event = generate_click_event(user_id) 131 | push_to_kafka(click_event, 'clicks') 132 | 133 | # simulate multiple clicks & checkouts 50% of the time 134 | while random.randint(1, 100) >= 50: 135 | click_event = generate_click_event( 136 | user_id, click_event['product_id'] 137 | ) 138 | push_to_kafka(click_event, 'clicks') 139 | 140 | push_to_kafka( 141 | generate_checkout_event( 142 | click_event["user_id"], click_event["product_id"] 143 | ), 144 | 'checkouts', 145 | ) 146 | 147 | 148 | if __name__ == "__main__": 149 | parser = argparse.ArgumentParser() 150 | parser.add_argument( 151 | "-nu", 152 | "--num_user_records", 153 | type=int, 154 | help="Number of user records to generate", 155 | default=100, 156 | ) 157 | parser.add_argument( 158 | "-nc", 159 | "--num_click_records", 160 | type=int, 161 | help="Number of click records to generate", 162 | default=100000000, 163 | ) 164 | args = parser.parse_args() 165 | gen_user_data(args.num_user_records) 166 | gen_clickstream_data(args.num_click_records) 167 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | * [Beginner Data Engineering Project - Stream Version](#beginner-data-engineering-project---stream-version) 3 | * [Project](#project) 4 | * [Run on codespaces](#run-on-codespaces) 5 | * [Run locally](#run-locally) 6 | * [Prerequisites](#prerequisites) 7 | * [Architecture](#architecture) 8 | * [Code design](#code-design) 9 | * [Run streaming job](#run-streaming-job) 10 | * [Check output](#check-output) 11 | * [Tear down](#tear-down) 12 | * [Contributing](#contributing) 13 | * [References](#references) 14 | 15 | # Beginner Data Engineering Project - Stream Version 16 | 17 | Code for blog at [Data Engineering Project Stream Edition](https://www.startdataengineering.com/post/data-engineering-project-for-beginners-stream-edition/). 18 | 19 | ## Project 20 | 21 | Consider we run an e-commerce website. An everyday use case with e-commerce is to identify, for every product purchased, the click that led to this purchase. Attribution is the joining of checkout(purchase) of a product to a click. There are multiple types of **[attribution](https://www.shopify.com/blog/marketing-attribution#3)**; we will focus on `First Click Attribution`. 22 | 23 | Our objectives are: 24 | 1. Enrich checkout data with the user name. The user data is in a transactional database. 25 | 2. Identify which click leads to a checkout (aka attribution). For every product checkout, we consider **the earliest click a user made on that product in the previous hour to be the click that led to a checkout**. 26 | 3. Log the checkouts and their corresponding attributed clicks (if any) into a table. 27 | 28 | ## Run on codespaces 29 | 30 | You can run this data pipeline using GitHub codespaces. Follow the instructions below. 31 | 32 | 1. Create codespaces by going to the **[beginner_de_project_stream](https://github.com/josephmachado/beginner_de_project_stream)** repository, cloning(or fork) it and then clicking on `Create codespaces on main` button. 33 | 2. Wait for codespaces to start, then in the terminal type `make run`. 34 | 3. Wait for `make run` to complete. 35 | 4. Go to the `ports` tab and click on the link exposing port `8081` to access Flink UI and clicking on `Jobs -> Running Jobs -> checkout-attribution-job` to see our running job.. 36 | 37 | ![codespace start](./assets/images/cs1.png) 38 | ![codespace make up](./assets/images/cs2.png) 39 | ![codespace access ui](./assets/images/cs3.png) 40 | 41 | **Note** Make sure to switch off codespaces instance, you only have limited free usage; see docs [here](https://github.com/features/codespaces#pricing). 42 | 43 | ## Run locally 44 | 45 | ### Prerequisites 46 | 47 | To run the code, you'll need the following: 48 | 49 | 1. [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) 50 | 2. [Docker](https://docs.docker.com/engine/install/) with at least 4GB of RAM and [Docker Compose](https://docs.docker.com/compose/install/) v1.27.0 or later 51 | 3. [psql](https://blog.timescale.com/tutorials/how-to-install-psql-on-mac-ubuntu-debian-windows/) 52 | 53 | If you are using windows please setup WSL and a local Ubuntu Virtual machine following **[the instructions here](https://ubuntu.com/tutorials/install-ubuntu-on-wsl2-on-windows-10#1-overview)**. Install the above prerequisites on your ubuntu terminal, if you have trouble installing docker follow **[the steps here](https://www.digitalocean.com/community/tutorials/how-to-install-and-use-docker-on-ubuntu-22-04#step-1-installing-docker)**. 54 | 55 | ## Architecture 56 | 57 | Our streaming pipeline architecture is as follows (from left to right): 58 | 59 | 1. **`Application`**: Website generates clicks and checkout event data. 60 | 2. **`Queue`**: The clicks and checkout data are sent to their corresponding Kafka topics. 61 | 3. **`Stream processing`**: 62 | 1. Flink reads data from the Kafka topics. 63 | 2. The click data is stored in our cluster state. Note that we only store click information for the last hour, and we only store one click per user-product combination. 64 | 3. The checkout data is enriched with user information by querying the user table in Postgres. 65 | 4. The checkout data is left joined with the click data( in the cluster state) to see if the checkout can be attributed to a click. 66 | 5. The enriched and attributed checkout data is logged into a Postgres sink table. 67 | 4. **`Monitoring & Alerting`**: Apache Flink metrics are pulled by Prometheus and visualized using Graphana. 68 | 69 | ![Architecture](./assets/images/arch.png) 70 | 71 | ## Code design 72 | 73 | We use Apache Table API to 74 | 75 | 1. Define Source systems: **[clicks, checkouts and users](https://github.com/josephmachado/beginner_de_project_stream/tree/main/code/source)**. [This python script](https://github.com/josephmachado/beginner_de_project_stream/blob/main/datagen/gen_fake_data.py) generates fake click and checkout data. 76 | 2. Define how to process the data (enrich and attribute): **[Enriching with user data and attributing checkouts ](https://github.com/josephmachado/beginner_de_project_stream/blob/main/code/process/attribute_checkouts.sql)** 77 | 3. Define Sink system: **[sink](https://github.com/josephmachado/beginner_de_project_stream/blob/main/code/sink/attributed_checkouts.sql)** 78 | 79 | The function **[run_checkout_attribution_job](https://github.com/josephmachado/beginner_de_project_stream/blob/cddab5b4bb2bce80e59d3525a78a02598d88eac9/code/checkout_attribution.py#L107-L129)** creates the sources, and sink and runs the data processing. 80 | 81 | We store the SQL DDL and DML in the folders `source`, `process`, and `sink` corresponding to the above steps. We use [Jinja2](https://jinja.palletsprojects.com/en/3.1.x/) to replace placeholders with [config values](https://github.com/josephmachado/beginner_de_project_stream/blob/cddab5b4bb2bce80e59d3525a78a02598d88eac9/code/checkout_attribution.py#L16-L62). **The code is available [here](https://github.com/josephmachado/beginner_de_project_stream).** 82 | 83 | ## Run streaming job 84 | 85 | Clone and run the streaming job (via terminal) as shown below: 86 | 87 | ```bash 88 | git clone https://github.com/josephmachado/beginner_de_project_stream 89 | cd beginner_de_project_stream 90 | make run # restart all containers, & start streaming job 91 | ``` 92 | 93 | 1. **Apache Flink UI**: Open [http://localhost:8081/](http://localhost:8081/) or run `make ui` and click on `Jobs -> Running Jobs -> checkout-attribution-job` to see our running job. 94 | 2. **Graphana**: Visualize system metrics with Graphana, use the `make open` command or go to [http://localhost:3000](http://localhost:3000) via your browser (username: `admin`, password:`flink`). 95 | 96 | **Note**: Checkout [Makefile](https://github.com/josephmachado/beginner_de_project_stream/blob/main/Makefile) to see how/what commands are run. Use `make down` to spin down the containers. 97 | 98 | ## Check output 99 | 100 | Once we start the job, it will run asynchronously. We can check the Flink UI ([http://localhost:8081/](http://localhost:8081/) or `make ui`) and clicking on `Jobs -> Running Jobs -> checkout-attribution-job` to see our running job. 101 | 102 | ![Flink UI](assets/images/flink_ui_dag.png) 103 | 104 | We can check the output of our job, by looking at the attributed checkouts. 105 | 106 | Open a postgres terminal as shown below. 107 | 108 | ```bash 109 | pgcli -h localhost -p 5432 -U postgres -d postgres 110 | # password: postgres 111 | ``` 112 | 113 | Use the below query to check that the output updates every few seconds. 114 | 115 | ```sql 116 | SELECT checkout_id, click_id, checkout_time, click_time, user_name FROM commerce.attributed_checkouts order by checkout_time desc limit 5; 117 | ``` 118 | 119 | ## Tear down 120 | 121 | Use `make down` to spin down the containers. 122 | 123 | ## Contributing 124 | 125 | Contributions are welcome. If you would like to contribute you can help by opening a Github issue or putting up a PR. 126 | 127 | ## References 128 | 129 | 1. [Apache Flink docs](https://nightlies.apache.org/flink/flink-docs-release-1.17/) 130 | 2. [Flink Prometheus example project](https://github.com/mbode/flink-prometheus-example) 131 | -------------------------------------------------------------------------------- /grafana/provisioning/dashboards/Flink.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": "-- Grafana --", 7 | "enable": true, 8 | "hide": true, 9 | "iconColor": "rgba(0, 211, 255, 1)", 10 | "name": "Annotations & Alerts", 11 | "type": "dashboard" 12 | } 13 | ] 14 | }, 15 | "editable": true, 16 | "gnetId": null, 17 | "graphTooltip": 0, 18 | "iteration": 1535902255604, 19 | "links": [], 20 | "panels": [ 21 | { 22 | "collapsed": true, 23 | "gridPos": { 24 | "h": 1, 25 | "w": 24, 26 | "x": 0, 27 | "y": 0 28 | }, 29 | "id": 9, 30 | "panels": [ 31 | { 32 | "aliasColors": {}, 33 | "bars": false, 34 | "dashLength": 10, 35 | "dashes": false, 36 | "datasource": "Prometheus", 37 | "fill": 1, 38 | "gridPos": { 39 | "h": 9, 40 | "w": 24, 41 | "x": 0, 42 | "y": 1 43 | }, 44 | "id": 2, 45 | "legend": { 46 | "alignAsTable": true, 47 | "avg": true, 48 | "current": true, 49 | "max": true, 50 | "min": true, 51 | "rightSide": true, 52 | "show": true, 53 | "total": false, 54 | "values": true 55 | }, 56 | "lines": true, 57 | "linewidth": 1, 58 | "links": [], 59 | "nullPointMode": "null", 60 | "percentage": false, 61 | "pointradius": 5, 62 | "points": false, 63 | "renderer": "flot", 64 | "seriesOverrides": [], 65 | "spaceLength": 10, 66 | "stack": false, 67 | "steppedLine": false, 68 | "targets": [ 69 | { 70 | "expr": "flink_jobmanager_Status_JVM_Memory_Heap_Used", 71 | "format": "time_series", 72 | "intervalFactor": 1, 73 | "legendFormat": "heap memory [{{instance}}]", 74 | "refId": "C" 75 | }, 76 | { 77 | "expr": "flink_jobmanager_Status_JVM_Memory_NonHeap_Used", 78 | "format": "time_series", 79 | "intervalFactor": 1, 80 | "legendFormat": "non-heap memory [{{instance}}]", 81 | "refId": "D" 82 | } 83 | ], 84 | "thresholds": [], 85 | "timeFrom": null, 86 | "timeShift": null, 87 | "title": "Memory used", 88 | "tooltip": { 89 | "shared": true, 90 | "sort": 0, 91 | "value_type": "individual" 92 | }, 93 | "type": "graph", 94 | "xaxis": { 95 | "buckets": null, 96 | "mode": "time", 97 | "name": null, 98 | "show": true, 99 | "values": [] 100 | }, 101 | "yaxes": [ 102 | { 103 | "format": "bytes", 104 | "label": null, 105 | "logBase": 1, 106 | "max": null, 107 | "min": null, 108 | "show": true 109 | }, 110 | { 111 | "format": "short", 112 | "label": null, 113 | "logBase": 1, 114 | "max": null, 115 | "min": null, 116 | "show": true 117 | } 118 | ], 119 | "yaxis": { 120 | "align": false, 121 | "alignLevel": null 122 | } 123 | }, 124 | { 125 | "aliasColors": {}, 126 | "bars": false, 127 | "dashLength": 10, 128 | "dashes": false, 129 | "datasource": "Prometheus", 130 | "fill": 1, 131 | "gridPos": { 132 | "h": 9, 133 | "w": 12, 134 | "x": 0, 135 | "y": 10 136 | }, 137 | "id": 10, 138 | "legend": { 139 | "avg": false, 140 | "current": false, 141 | "max": false, 142 | "min": false, 143 | "show": true, 144 | "total": false, 145 | "values": false 146 | }, 147 | "lines": true, 148 | "linewidth": 1, 149 | "links": [], 150 | "nullPointMode": "null", 151 | "percentage": false, 152 | "pointradius": 5, 153 | "points": false, 154 | "renderer": "flot", 155 | "seriesOverrides": [], 156 | "spaceLength": 10, 157 | "stack": false, 158 | "steppedLine": false, 159 | "targets": [ 160 | { 161 | "expr": "rate(flink_jobmanager_Status_JVM_GarbageCollector_PS_MarkSweep_Time[5m])/1000 + rate(flink_jobmanager_Status_JVM_GarbageCollector_PS_Scavenge_Time[5m])/1000", 162 | "format": "time_series", 163 | "intervalFactor": 1, 164 | "legendFormat": "{{instance}}", 165 | "refId": "B" 166 | } 167 | ], 168 | "thresholds": [], 169 | "timeFrom": null, 170 | "timeShift": null, 171 | "title": "GC Overhead", 172 | "tooltip": { 173 | "shared": true, 174 | "sort": 0, 175 | "value_type": "individual" 176 | }, 177 | "type": "graph", 178 | "xaxis": { 179 | "buckets": null, 180 | "mode": "time", 181 | "name": null, 182 | "show": true, 183 | "values": [] 184 | }, 185 | "yaxes": [ 186 | { 187 | "format": "percentunit", 188 | "label": null, 189 | "logBase": 1, 190 | "max": null, 191 | "min": null, 192 | "show": true 193 | }, 194 | { 195 | "format": "short", 196 | "label": null, 197 | "logBase": 1, 198 | "max": null, 199 | "min": null, 200 | "show": true 201 | } 202 | ], 203 | "yaxis": { 204 | "align": false, 205 | "alignLevel": null 206 | } 207 | } 208 | ], 209 | "title": "JobManager", 210 | "type": "row" 211 | }, 212 | { 213 | "collapsed": true, 214 | "gridPos": { 215 | "h": 1, 216 | "w": 24, 217 | "x": 0, 218 | "y": 1 219 | }, 220 | "id": 6, 221 | "panels": [ 222 | { 223 | "aliasColors": {}, 224 | "bars": false, 225 | "dashLength": 10, 226 | "dashes": false, 227 | "datasource": "Prometheus", 228 | "fill": 1, 229 | "gridPos": { 230 | "h": 9, 231 | "w": 24, 232 | "x": 0, 233 | "y": 2 234 | }, 235 | "id": 7, 236 | "legend": { 237 | "alignAsTable": true, 238 | "avg": true, 239 | "current": true, 240 | "max": true, 241 | "min": true, 242 | "rightSide": true, 243 | "show": true, 244 | "total": false, 245 | "values": true 246 | }, 247 | "lines": true, 248 | "linewidth": 1, 249 | "links": [], 250 | "nullPointMode": "null", 251 | "percentage": false, 252 | "pointradius": 5, 253 | "points": false, 254 | "renderer": "flot", 255 | "seriesOverrides": [], 256 | "spaceLength": 10, 257 | "stack": false, 258 | "steppedLine": false, 259 | "targets": [ 260 | { 261 | "expr": "flink_taskmanager_Status_JVM_Memory_Heap_Used", 262 | "format": "time_series", 263 | "intervalFactor": 1, 264 | "legendFormat": "heap memory [{{instance}}]", 265 | "refId": "A" 266 | }, 267 | { 268 | "expr": "flink_taskmanager_Status_JVM_Memory_NonHeap_Used", 269 | "format": "time_series", 270 | "intervalFactor": 1, 271 | "legendFormat": "non-heap memory [{{instance}}]", 272 | "refId": "B" 273 | } 274 | ], 275 | "thresholds": [], 276 | "timeFrom": null, 277 | "timeShift": null, 278 | "title": "Memory used", 279 | "tooltip": { 280 | "shared": true, 281 | "sort": 0, 282 | "value_type": "individual" 283 | }, 284 | "type": "graph", 285 | "xaxis": { 286 | "buckets": null, 287 | "mode": "time", 288 | "name": null, 289 | "show": true, 290 | "values": [] 291 | }, 292 | "yaxes": [ 293 | { 294 | "format": "bytes", 295 | "label": null, 296 | "logBase": 1, 297 | "max": null, 298 | "min": null, 299 | "show": true 300 | }, 301 | { 302 | "format": "short", 303 | "label": null, 304 | "logBase": 1, 305 | "max": null, 306 | "min": null, 307 | "show": true 308 | } 309 | ], 310 | "yaxis": { 311 | "align": false, 312 | "alignLevel": null 313 | } 314 | }, 315 | { 316 | "aliasColors": {}, 317 | "bars": false, 318 | "dashLength": 10, 319 | "dashes": false, 320 | "datasource": "Prometheus", 321 | "fill": 1, 322 | "gridPos": { 323 | "h": 9, 324 | "w": 12, 325 | "x": 0, 326 | "y": 11 327 | }, 328 | "id": 4, 329 | "legend": { 330 | "avg": false, 331 | "current": false, 332 | "max": false, 333 | "min": false, 334 | "show": true, 335 | "total": false, 336 | "values": false 337 | }, 338 | "lines": true, 339 | "linewidth": 1, 340 | "links": [], 341 | "nullPointMode": "null", 342 | "percentage": false, 343 | "pointradius": 5, 344 | "points": false, 345 | "renderer": "flot", 346 | "seriesOverrides": [], 347 | "spaceLength": 10, 348 | "stack": false, 349 | "steppedLine": false, 350 | "targets": [ 351 | { 352 | "expr": "rate(flink_taskmanager_Status_JVM_GarbageCollector_G1_Old_Generation_Time[5m])/1000 + rate(flink_taskmanager_Status_JVM_GarbageCollector_G1_Young_Generation_Time[5m])/1000", 353 | "format": "time_series", 354 | "intervalFactor": 1, 355 | "legendFormat": "{{instance}}", 356 | "refId": "A" 357 | } 358 | ], 359 | "thresholds": [], 360 | "timeFrom": null, 361 | "timeShift": null, 362 | "title": "GC Overhead", 363 | "tooltip": { 364 | "shared": true, 365 | "sort": 0, 366 | "value_type": "individual" 367 | }, 368 | "type": "graph", 369 | "xaxis": { 370 | "buckets": null, 371 | "mode": "time", 372 | "name": null, 373 | "show": true, 374 | "values": [] 375 | }, 376 | "yaxes": [ 377 | { 378 | "format": "percentunit", 379 | "label": null, 380 | "logBase": 1, 381 | "max": null, 382 | "min": null, 383 | "show": true 384 | }, 385 | { 386 | "format": "short", 387 | "label": null, 388 | "logBase": 1, 389 | "max": null, 390 | "min": null, 391 | "show": true 392 | } 393 | ], 394 | "yaxis": { 395 | "align": false, 396 | "alignLevel": null 397 | } 398 | } 399 | ], 400 | "repeat": null, 401 | "title": "TaskManager", 402 | "type": "row" 403 | }, 404 | { 405 | "collapsed": false, 406 | "gridPos": { 407 | "h": 1, 408 | "w": 24, 409 | "x": 0, 410 | "y": 2 411 | }, 412 | "id": 25, 413 | "panels": [], 414 | "title": "checkout_attribution_job", 415 | "type": "row" 416 | }, 417 | { 418 | "aliasColors": {}, 419 | "bars": false, 420 | "dashLength": 10, 421 | "dashes": false, 422 | "datasource": null, 423 | "fill": 1, 424 | "gridPos": { 425 | "h": 9, 426 | "w": 12, 427 | "x": 0, 428 | "y": 3 429 | }, 430 | "id": 20, 431 | "legend": { 432 | "avg": false, 433 | "current": false, 434 | "max": false, 435 | "min": false, 436 | "show": true, 437 | "total": false, 438 | "values": false 439 | }, 440 | "lines": true, 441 | "linewidth": 1, 442 | "links": [], 443 | "nullPointMode": "null", 444 | "percentage": false, 445 | "pointradius": 5, 446 | "points": false, 447 | "renderer": "flot", 448 | "seriesOverrides": [], 449 | "spaceLength": 10, 450 | "stack": false, 451 | "steppedLine": false, 452 | "targets": [ 453 | { 454 | "expr": "flink_taskmanager_job_task_operator_KafkaSourceReader_KafkaConsumer_bytes_consumed_total", 455 | "format": "time_series", 456 | "intervalFactor": 1, 457 | "legendFormat": "{{job}}", 458 | "refId": "A" 459 | } 460 | ], 461 | "thresholds": [], 462 | "timeFrom": null, 463 | "timeShift": null, 464 | "title": "Kafka Bytes Consumed", 465 | "tooltip": { 466 | "shared": true, 467 | "sort": 0, 468 | "value_type": "individual" 469 | }, 470 | "type": "graph", 471 | "xaxis": { 472 | "buckets": null, 473 | "mode": "time", 474 | "name": null, 475 | "show": true, 476 | "values": [] 477 | }, 478 | "yaxes": [ 479 | { 480 | "format": "short", 481 | "label": null, 482 | "logBase": 1, 483 | "max": null, 484 | "min": null, 485 | "show": true 486 | }, 487 | { 488 | "format": "short", 489 | "label": null, 490 | "logBase": 1, 491 | "max": null, 492 | "min": null, 493 | "show": true 494 | } 495 | ], 496 | "yaxis": { 497 | "align": false, 498 | "alignLevel": null 499 | } 500 | }, 501 | { 502 | "aliasColors": {}, 503 | "bars": false, 504 | "dashLength": 10, 505 | "dashes": false, 506 | "datasource": null, 507 | "fill": 1, 508 | "gridPos": { 509 | "h": 9, 510 | "w": 12, 511 | "x": 12, 512 | "y": 3 513 | }, 514 | "id": 22, 515 | "legend": { 516 | "alignAsTable": false, 517 | "avg": false, 518 | "current": false, 519 | "hideZero": false, 520 | "max": false, 521 | "min": false, 522 | "show": true, 523 | "total": false, 524 | "values": false 525 | }, 526 | "lines": true, 527 | "linewidth": 1, 528 | "links": [], 529 | "nullPointMode": "null", 530 | "percentage": false, 531 | "pointradius": 5, 532 | "points": false, 533 | "renderer": "flot", 534 | "seriesOverrides": [], 535 | "spaceLength": 10, 536 | "stack": false, 537 | "steppedLine": false, 538 | "targets": [ 539 | { 540 | "expr": "flink_taskmanager_job_task_operator_currentEmitEventTimeLag", 541 | "format": "time_series", 542 | "intervalFactor": 1, 543 | "legendFormat": "{{quantile}}", 544 | "refId": "A" 545 | } 546 | ], 547 | "thresholds": [], 548 | "timeFrom": null, 549 | "timeShift": null, 550 | "title": "Event Time lag", 551 | "tooltip": { 552 | "shared": true, 553 | "sort": 0, 554 | "value_type": "individual" 555 | }, 556 | "type": "graph", 557 | "xaxis": { 558 | "buckets": null, 559 | "mode": "time", 560 | "name": null, 561 | "show": true, 562 | "values": [] 563 | }, 564 | "yaxes": [ 565 | { 566 | "format": "short", 567 | "label": null, 568 | "logBase": 1, 569 | "max": null, 570 | "min": null, 571 | "show": true 572 | }, 573 | { 574 | "format": "short", 575 | "label": null, 576 | "logBase": 1, 577 | "max": null, 578 | "min": null, 579 | "show": true 580 | } 581 | ], 582 | "yaxis": { 583 | "align": false, 584 | "alignLevel": null 585 | } 586 | }, 587 | { 588 | "aliasColors": {}, 589 | "bars": false, 590 | "dashLength": 10, 591 | "dashes": false, 592 | "datasource": null, 593 | "fill": 1, 594 | "gridPos": { 595 | "h": 9, 596 | "w": 12, 597 | "x": 0, 598 | "y": 12 599 | }, 600 | "id": 28, 601 | "legend": { 602 | "avg": false, 603 | "current": false, 604 | "max": false, 605 | "min": false, 606 | "show": true, 607 | "total": false, 608 | "values": false 609 | }, 610 | "lines": true, 611 | "linewidth": 1, 612 | "links": [], 613 | "nullPointMode": "null", 614 | "percentage": false, 615 | "pointradius": 5, 616 | "points": false, 617 | "renderer": "flot", 618 | "seriesOverrides": [], 619 | "spaceLength": 10, 620 | "stack": false, 621 | "steppedLine": false, 622 | "targets": [ 623 | { 624 | "expr": "flink_taskmanager_job_task_numBytesInLocalPerSecond{job_name=\"checkout_attribution_job\"}", 625 | "format": "time_series", 626 | "interval": "", 627 | "intervalFactor": 1, 628 | "legendFormat": "{{task_name}}", 629 | "refId": "A" 630 | } 631 | ], 632 | "thresholds": [], 633 | "timeFrom": null, 634 | "timeShift": null, 635 | "title": "Bytes in", 636 | "tooltip": { 637 | "shared": true, 638 | "sort": 0, 639 | "value_type": "individual" 640 | }, 641 | "type": "graph", 642 | "xaxis": { 643 | "buckets": null, 644 | "mode": "time", 645 | "name": null, 646 | "show": true, 647 | "values": [] 648 | }, 649 | "yaxes": [ 650 | { 651 | "format": "Bps", 652 | "label": null, 653 | "logBase": 1, 654 | "max": null, 655 | "min": null, 656 | "show": true 657 | }, 658 | { 659 | "format": "short", 660 | "label": null, 661 | "logBase": 1, 662 | "max": null, 663 | "min": null, 664 | "show": true 665 | } 666 | ], 667 | "yaxis": { 668 | "align": false, 669 | "alignLevel": null 670 | } 671 | }, 672 | { 673 | "aliasColors": {}, 674 | "bars": false, 675 | "dashLength": 10, 676 | "dashes": false, 677 | "datasource": null, 678 | "fill": 1, 679 | "gridPos": { 680 | "h": 9, 681 | "w": 12, 682 | "x": 12, 683 | "y": 12 684 | }, 685 | "id": 29, 686 | "legend": { 687 | "avg": false, 688 | "current": false, 689 | "max": false, 690 | "min": false, 691 | "show": true, 692 | "total": false, 693 | "values": false 694 | }, 695 | "lines": true, 696 | "linewidth": 1, 697 | "links": [], 698 | "nullPointMode": "null", 699 | "percentage": false, 700 | "pointradius": 5, 701 | "points": false, 702 | "renderer": "flot", 703 | "seriesOverrides": [], 704 | "spaceLength": 10, 705 | "stack": false, 706 | "steppedLine": false, 707 | "targets": [ 708 | { 709 | "expr": "flink_taskmanager_job_task_numBytesOutPerSecond{job_name=\"checkout_attribution_job\"}", 710 | "format": "time_series", 711 | "interval": "", 712 | "intervalFactor": 1, 713 | "legendFormat": "{{task_name}}", 714 | "refId": "A" 715 | } 716 | ], 717 | "thresholds": [], 718 | "timeFrom": null, 719 | "timeShift": null, 720 | "title": "Bytes out", 721 | "tooltip": { 722 | "shared": true, 723 | "sort": 0, 724 | "value_type": "individual" 725 | }, 726 | "type": "graph", 727 | "xaxis": { 728 | "buckets": null, 729 | "mode": "time", 730 | "name": null, 731 | "show": true, 732 | "values": [] 733 | }, 734 | "yaxes": [ 735 | { 736 | "format": "Bps", 737 | "label": null, 738 | "logBase": 1, 739 | "max": null, 740 | "min": null, 741 | "show": true 742 | }, 743 | { 744 | "format": "short", 745 | "label": null, 746 | "logBase": 1, 747 | "max": null, 748 | "min": null, 749 | "show": true 750 | } 751 | ], 752 | "yaxis": { 753 | "align": false, 754 | "alignLevel": null 755 | } 756 | }, 757 | { 758 | "cacheTimeout": null, 759 | "colorBackground": false, 760 | "colorValue": false, 761 | "colors": [ 762 | "#d44a3a", 763 | "rgba(237, 129, 40, 0.89)", 764 | "#299c46" 765 | ], 766 | "datasource": "Prometheus", 767 | "decimals": 1, 768 | "format": "ms", 769 | "gauge": { 770 | "maxValue": 100, 771 | "minValue": 0, 772 | "show": false, 773 | "thresholdLabels": false, 774 | "thresholdMarkers": true 775 | }, 776 | "gridPos": { 777 | "h": 3, 778 | "w": 5, 779 | "x": 0, 780 | "y": 21 781 | }, 782 | "id": 14, 783 | "interval": null, 784 | "links": [], 785 | "mappingType": 1, 786 | "mappingTypes": [ 787 | { 788 | "name": "value to text", 789 | "value": 1 790 | }, 791 | { 792 | "name": "range to text", 793 | "value": 2 794 | } 795 | ], 796 | "maxDataPoints": 100, 797 | "nullPointMode": "connected", 798 | "nullText": null, 799 | "postfix": "", 800 | "postfixFontSize": "50%", 801 | "prefix": "", 802 | "prefixFontSize": "50%", 803 | "rangeMaps": [ 804 | { 805 | "from": "null", 806 | "text": "N/A", 807 | "to": "null" 808 | } 809 | ], 810 | "repeat": "job", 811 | "repeatDirection": "h", 812 | "scopedVars": { 813 | "job": { 814 | "selected": false, 815 | "text": "checkout_attribution_job", 816 | "value": "checkout_attribution_job" 817 | } 818 | }, 819 | "sparkline": { 820 | "fillColor": "rgba(31, 118, 189, 0.18)", 821 | "full": false, 822 | "lineColor": "rgb(31, 120, 193)", 823 | "show": false 824 | }, 825 | "tableColumn": "", 826 | "targets": [ 827 | { 828 | "expr": "flink_jobmanager_job_uptime{job_name=\"$job\"}", 829 | "format": "time_series", 830 | "intervalFactor": 1, 831 | "refId": "A" 832 | } 833 | ], 834 | "thresholds": ".5,.8", 835 | "title": "Uptime", 836 | "type": "singlestat", 837 | "valueFontSize": "80%", 838 | "valueMaps": [ 839 | { 840 | "op": "=", 841 | "text": "N/A", 842 | "value": "null" 843 | } 844 | ], 845 | "valueName": "current" 846 | } 847 | ], 848 | "refresh": "5s", 849 | "schemaVersion": 16, 850 | "style": "dark", 851 | "tags": [], 852 | "templating": { 853 | "list": [] 854 | }, 855 | "time": { 856 | "from": "now-5m", 857 | "to": "now" 858 | }, 859 | "timepicker": { 860 | "refresh_intervals": [ 861 | "5s", 862 | "10s", 863 | "30s", 864 | "1m", 865 | "5m", 866 | "15m", 867 | "30m", 868 | "1h", 869 | "2h", 870 | "1d" 871 | ], 872 | "time_options": [ 873 | "5m", 874 | "15m", 875 | "1h", 876 | "6h", 877 | "12h", 878 | "24h", 879 | "2d", 880 | "7d", 881 | "30d" 882 | ] 883 | }, 884 | "timezone": "", 885 | "title": "Flink", 886 | "uid": "veLveEOiz", 887 | "version": 1 888 | } --------------------------------------------------------------------------------