├── Images ├── Flow.png ├── Workflow_Regression.png └── graph.png ├── ML_model ├── dags │ ├── __pycache__ │ │ ├── basic-pipeline.cpython-37.pyc │ │ └── tutorial.cpython-37.pyc │ ├── basic-pipeline.py │ └── variables_config.json └── models │ ├── __pycache__ │ └── data_is_available.cpython-37.pyc │ ├── data │ ├── .ipynb_checkpoints │ │ └── Untitled-checkpoint.ipynb │ ├── 50_Startups.csv │ ├── preprocessed_data.csv │ ├── scaled_data.csv │ ├── testing_data.csv │ └── training_data.csv │ ├── data_is_available.py │ ├── data_preprocessing.py │ └── results │ ├── BestModel_results.csv │ ├── DT_results.csv │ ├── GBR_results.csv │ ├── LR_results.csv │ ├── RF_results.csv │ └── SVR_results.csv ├── README.md ├── docker-airflow ├── Dockerfile ├── config │ └── airflow.cfg ├── docker-compose-CeleryExecutor.yml ├── docker-compose-LocalExecutor.yml └── script │ └── entrypoint.sh └── docker-compose.yml /Images/Flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rahul765/Machine-Learning-Pipelines/27ee52614b755912bcd5038a42fe67e148d79c57/Images/Flow.png -------------------------------------------------------------------------------- /Images/Workflow_Regression.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rahul765/Machine-Learning-Pipelines/27ee52614b755912bcd5038a42fe67e148d79c57/Images/Workflow_Regression.png -------------------------------------------------------------------------------- /Images/graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rahul765/Machine-Learning-Pipelines/27ee52614b755912bcd5038a42fe67e148d79c57/Images/graph.png -------------------------------------------------------------------------------- /ML_model/dags/__pycache__/basic-pipeline.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rahul765/Machine-Learning-Pipelines/27ee52614b755912bcd5038a42fe67e148d79c57/ML_model/dags/__pycache__/basic-pipeline.cpython-37.pyc -------------------------------------------------------------------------------- /ML_model/dags/__pycache__/tutorial.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rahul765/Machine-Learning-Pipelines/27ee52614b755912bcd5038a42fe67e148d79c57/ML_model/dags/__pycache__/tutorial.cpython-37.pyc -------------------------------------------------------------------------------- /ML_model/dags/basic-pipeline.py: -------------------------------------------------------------------------------- 1 | from airflow import DAG 2 | from airflow.operators.python_operator import PythonOperator 3 | from datetime import datetime, timedelta 4 | import pandas as pd 5 | import numpy as np 6 | import os 7 | import sqlite3 8 | 9 | from airflow.models import Variable 10 | from sklearn.model_selection import train_test_split 11 | from sklearn.preprocessing import MinMaxScaler 12 | from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score 13 | from sklearn.linear_model import LinearRegression 14 | from sklearn.tree import DecisionTreeRegressor 15 | from sklearn.ensemble import RandomForestRegressor 16 | from sklearn.svm import SVR 17 | from sklearn.ensemble import GradientBoostingRegressor 18 | 19 | # Fetching environment variable 20 | dag_config = Variable.get("variables_config", deserialize_json=True) 21 | data_path = dag_config["data_path"] 22 | preprocessed_data = dag_config["preprocessed_data"] 23 | scaled_data = dag_config["scaled_data"] 24 | training_data = dag_config["training_data"] 25 | testing_data = dag_config["testing_data"] 26 | LR_results = dag_config["LR_results"] 27 | DT_results = dag_config["DT_results"] 28 | RF_results = dag_config["RF_results"] 29 | SVR_results = dag_config["SVR_results"] 30 | GBR_results = dag_config["GBR_results"] 31 | BestModel_results = dag_config["BestModel_results"] 32 | 33 | # Checking if Data is availabe 34 | def data_is_available(_file_name=data_path, **kwargs): 35 | dataset = pd.read_csv(_file_name) 36 | if dataset.empty: 37 | print("No Data Fetched") 38 | else: 39 | print("{} records have been fetched".format(dataset.shape[0])) 40 | return "{} records have been fetched".format(dataset.shape[0]) 41 | 42 | # Preprocessing the dataset 43 | def preprocessing(_file_name=data_path, **kwargs): 44 | dataset = pd.read_csv(_file_name) 45 | dum_df = pd.get_dummies(dataset, columns=["State"], prefix=["State_is"] ) 46 | dum_df.drop(['State_is_Florida'], axis=1, inplace=True) 47 | dum_df.to_csv(preprocessed_data) 48 | print(dum_df.head()) 49 | 50 | def scaling(_file_name=preprocessed_data, **kwargs): 51 | dataset = pd.read_csv(_file_name) 52 | y = dataset['Profit'] 53 | X = dataset.drop(['Profit'], axis=1) 54 | X_colums = X.columns 55 | # Feature Scaling 56 | sc_X = MinMaxScaler() 57 | X = sc_X.fit_transform(X) 58 | scaled_X = pd.DataFrame(X, columns=X_colums) 59 | scaled_X['Profit'] = y 60 | scaled_X.to_csv(scaled_data) 61 | print(scaled_X.head()) 62 | 63 | def splitting_data(_file_name=scaled_data, **kwargs): 64 | dataset = pd.read_csv(_file_name) 65 | y = dataset['Profit'] 66 | X = dataset.drop(['Profit'], axis=1) 67 | X_colums = X.columns 68 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0) 69 | X_train = pd.DataFrame(X_train,columns=X_colums) 70 | X_train['Profit'] = y_train 71 | X_test = pd.DataFrame(X_test,columns=X_colums) 72 | X_test['Profit'] = y_test 73 | X_train.to_csv(training_data) 74 | X_test.to_csv(testing_data) 75 | print("Ratio is 80:20") 76 | 77 | def evaluate(y_test,y_pred,test_train): 78 | MSE_test = mean_squared_error(y_test, y_pred) 79 | RMSE_test = np.sqrt(MSE_test) 80 | Adjusted_RSquare_test = r2_score(y_test, y_pred) 81 | MAE = mean_absolute_error(y_test,y_pred) 82 | print('Model Performance for {:s}'.format(test_train)) 83 | print('Mean Sqaure Error(MSE): {:0.4f}.'.format(MSE_test)) 84 | print('Root Mean Sqaure Error(RMSE): {:0.4f}.'.format(RMSE_test)) 85 | print('Adjusted R Square = {:0.2f}.'.format(Adjusted_RSquare_test)) 86 | print('MAE = {:0.2f}.'.format(MAE)) 87 | return {"MSE": MSE_test, "RMSE" : RMSE_test, "Adjusted_RSquare_test": Adjusted_RSquare_test, "MAE": MAE} 88 | 89 | 90 | def model_1_LR(_file_name=training_data, _test_file=testing_data, **kwargs): 91 | dataset = pd.read_csv(_file_name) 92 | y_train = dataset['Profit'] 93 | X_train = dataset.drop(['Profit'], axis=1) 94 | regressor = LinearRegression() 95 | regressor.fit(X_train, y_train) 96 | test = pd.read_csv(_test_file) 97 | y_test = test['Profit'] 98 | X_test = test.drop(['Profit'], axis=1) 99 | y_pred = regressor.predict(X_test) 100 | test_results = evaluate(y_test,y_pred, 'Linear Regression') 101 | test_results = pd.DataFrame.from_dict(test_results, orient='index',columns=['Test Values']) 102 | test_results.to_csv(LR_results) 103 | print(test_results) 104 | 105 | def model_2_DT(_file_name=training_data, _test_file=testing_data, **kwargs): 106 | dataset = pd.read_csv(_file_name) 107 | y_train = dataset['Profit'] 108 | X_train = dataset.drop(['Profit'], axis=1) 109 | regressor = DecisionTreeRegressor(random_state=123) 110 | regressor.fit(X_train, y_train) 111 | test = pd.read_csv(_test_file) 112 | y_test = test['Profit'] 113 | X_test = test.drop(['Profit'], axis=1) 114 | y_pred = regressor.predict(X_test) 115 | test_results = evaluate(y_test,y_pred,'Decision Tree Regression') 116 | test_results = pd.DataFrame.from_dict(test_results, orient='index',columns=['Test Values']) 117 | test_results.to_csv(DT_results) 118 | print(test_results) 119 | 120 | def model_3_RF(_file_name=training_data, _test_file=testing_data, **kwargs): 121 | dataset = pd.read_csv(_file_name) 122 | y_train = dataset['Profit'] 123 | X_train = dataset.drop(['Profit'], axis=1) 124 | regressor = RandomForestRegressor(random_state=123) 125 | regressor.fit(X_train, y_train) 126 | test = pd.read_csv(_test_file) 127 | y_test = test['Profit'] 128 | X_test = test.drop(['Profit'], axis=1) 129 | y_pred = regressor.predict(X_test) 130 | test_results = evaluate(y_test,y_pred,'Random Forest Regression') 131 | test_results = pd.DataFrame.from_dict(test_results, orient='index',columns=['Test Values']) 132 | test_results.to_csv(RF_results) 133 | print(test_results) 134 | 135 | def model_4_SVR(_file_name=training_data, _test_file=testing_data, **kwargs): 136 | dataset = pd.read_csv(_file_name) 137 | y_train = dataset['Profit'] 138 | X_train = dataset.drop(['Profit'], axis=1) 139 | regressor = SVR() 140 | regressor.fit(X_train, y_train) 141 | test = pd.read_csv(_test_file) 142 | y_test = test['Profit'] 143 | X_test = test.drop(['Profit'], axis=1) 144 | y_pred = regressor.predict(X_test) 145 | test_results = evaluate(y_test,y_pred,'Support Vector Regression') 146 | test_results = pd.DataFrame.from_dict(test_results, orient='index',columns=['Test Values']) 147 | test_results.to_csv(SVR_results) 148 | print(test_results) 149 | 150 | def model_5_GBR(_file_name=training_data, _test_file=testing_data, **kwargs): 151 | dataset = pd.read_csv(_file_name) 152 | y_train = dataset['Profit'] 153 | X_train = dataset.drop(['Profit'], axis=1) 154 | regressor = GradientBoostingRegressor(random_state=123) 155 | regressor.fit(X_train, y_train) 156 | test = pd.read_csv(_test_file) 157 | y_test = test['Profit'] 158 | X_test = test.drop(['Profit'], axis=1) 159 | y_pred = regressor.predict(X_test) 160 | test_results = evaluate(y_test,y_pred,'Gradient Boosting Regression') 161 | test_results = pd.DataFrame.from_dict(test_results, orient='index',columns=['Test Values']) 162 | test_results.to_csv(GBR_results) 163 | print(test_results) 164 | 165 | def best_model(m1=LR_results, m2=DT_results, m3=RF_results, m4=SVR_results, m5=GBR_results, best=BestModel_results, **kwargs): 166 | dt = pd.read_csv(m2) 167 | rf = pd.read_csv(m3) 168 | gbr = pd.read_csv(m5) 169 | lr = pd.read_csv(m1) 170 | svr = pd.read_csv(m4) 171 | conn = sqlite3.connect(':memory:') 172 | dt.to_sql('dt',conn,index=False) 173 | rf.to_sql('rf',conn,index=False) 174 | gbr.to_sql('gbr',conn,index=False) 175 | lr.to_sql('lr',conn,index=False) 176 | svr.to_sql('svr',conn,index=False) 177 | tmp=pd.read_sql_query('''select dt."Unnamed: 0" as ErrorMetric, 178 | case 179 | when dt."Unnamed: 0"<>'Adjusted_RSquare_test' and dt."Test Values"'Adjusted_RSquare_test' and rf."Test Values"'Adjusted_RSquare_test' and gbr."Test Values"'Adjusted_RSquare_test' and lr."Test Values"'Adjusted_RSquare_test' and svr."Test Values"rf."Test Values" and dt."Test Values">gbr."Test Values" and dt."Test Values">lr."Test Values" and dt."Test Values">svr."Test Values" 190 | then 'DT' 191 | when dt."Unnamed: 0"='Adjusted_RSquare_test' and rf."Test Values">dt."Test Values" and rf."Test Values">gbr."Test Values" and rf."Test Values">lr."Test Values" and rf."Test Values">svr."Test Values" 192 | then 'RF' 193 | when dt."Unnamed: 0"='Adjusted_RSquare_test' and gbr."Test Values">dt."Test Values" and gbr."Test Values">rf."Test Values" and gbr."Test Values">lr."Test Values" and gbr."Test Values">svr."Test Values" 194 | then 'GBR' 195 | when dt."Unnamed: 0"='Adjusted_RSquare_test' and lr."Test Values">dt."Test Values" and lr."Test Values">rf."Test Values" and lr."Test Values">gbr."Test Values" and lr."Test Values">svr."Test Values" 196 | then 'LR' 197 | when dt."Unnamed: 0"='Adjusted_RSquare_test' and svr."Test Values">dt."Test Values" and svr."Test Values">rf."Test Values" and svr."Test Values">lr."Test Values" and svr."Test Values">gbr."Test Values" 198 | then 'SVR' 199 | end as BestModel 200 | from dt 201 | inner join rf on dt."Unnamed: 0"=rf."Unnamed: 0" 202 | inner join gbr on gbr."Unnamed: 0"=rf."Unnamed: 0" 203 | inner join lr on lr."Unnamed: 0"=gbr."Unnamed: 0" 204 | inner join svr on svr."Unnamed: 0"=lr."Unnamed: 0"''',conn) 205 | tmp.to_csv(best) 206 | 207 | project_cfg = { 208 | 'owner': 'airflow', 209 | 'email': ['your-email@example.com'], 210 | 'email_on_failure': False, 211 | 'start_date': datetime(2019, 8, 1), 212 | 'retries': 1, 213 | 'retry_delay': timedelta(hours=1), 214 | } 215 | 216 | dag = DAG('Regression_ML_Pipeline', 217 | default_args=project_cfg, 218 | schedule_interval=timedelta(days=1)) 219 | 220 | task_1 = PythonOperator( 221 | task_id='is_data_available', 222 | provide_context=True, 223 | python_callable=data_is_available, 224 | # op_kwargs={'_file_name': '50_Startups.csv'}, 225 | dag=dag, 226 | ) 227 | 228 | task_2 = PythonOperator( 229 | task_id='preprocessing', 230 | provide_context=True, 231 | python_callable=preprocessing, 232 | dag=dag, 233 | ) 234 | 235 | task_3 = PythonOperator( 236 | task_id='scaling', 237 | provide_context=True, 238 | python_callable=scaling, 239 | dag=dag, 240 | ) 241 | 242 | task_4 = PythonOperator( 243 | task_id='splitting_data', 244 | provide_context=True, 245 | python_callable=splitting_data, 246 | dag=dag, 247 | ) 248 | 249 | task_5_1 = PythonOperator( 250 | task_id='linear_regression', 251 | provide_context=True, 252 | python_callable=model_1_LR, 253 | dag=dag, 254 | ) 255 | 256 | task_5_2 = PythonOperator( 257 | task_id='decision_tree_regression', 258 | provide_context=True, 259 | python_callable=model_2_DT, 260 | dag=dag, 261 | ) 262 | 263 | 264 | 265 | task_5_3 = PythonOperator( 266 | task_id='random_forest_regression', 267 | provide_context=True, 268 | python_callable=model_3_RF, 269 | dag=dag, 270 | ) 271 | 272 | task_5_4 = PythonOperator( 273 | task_id='support_vector_regression', 274 | provide_context=True, 275 | python_callable=model_4_SVR, 276 | dag=dag, 277 | ) 278 | 279 | task_5_5 = PythonOperator( 280 | task_id='gradient_boosting_regression', 281 | provide_context=True, 282 | python_callable=model_5_GBR, 283 | dag=dag, 284 | ) 285 | 286 | task_6 = PythonOperator( 287 | task_id='best_model', 288 | provide_context=True, 289 | python_callable=best_model, 290 | dag=dag, 291 | ) 292 | 293 | task_1 >> task_2 >> task_3 >> task_4 >> [task_5_1, task_5_2, task_5_3, task_5_4, task_5_5] >> task_6 294 | -------------------------------------------------------------------------------- /ML_model/dags/variables_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "variables_config": { 3 | "data_path": "/usr/local/airflow/models/data/50_startups.csv", 4 | "preprocessed_data": "/usr/local/airflow/models/data/preprocessed_data.csv", 5 | "scaled_data": "/usr/local/airflow/models/data/scaled_data.csv", 6 | "training_data": "/usr/local/airflow/models/data/training_data.csv", 7 | "testing_data": "/usr/local/airflow/models/data/testing_data.csv", 8 | "LR_results": "/usr/local/airflow/models/results/LR_results.csv", 9 | "DT_results": "/usr/local/airflow/models/results/DT_results.csv", 10 | "RF_results": "/usr/local/airflow/models/results/RF_results.csv", 11 | "SVR_results": "/usr/local/airflow/models/results/SVR_results.csv", 12 | "GBR_results": "/usr/local/airflow/models/results/GBR_results.csv", 13 | "BestModel_results": "/usr/local/airflow/models/results/BestModel_results.csv", 14 | "var2": [1, 2, 3], 15 | "var3": {"k": "value3"} 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /ML_model/models/__pycache__/data_is_available.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rahul765/Machine-Learning-Pipelines/27ee52614b755912bcd5038a42fe67e148d79c57/ML_model/models/__pycache__/data_is_available.cpython-37.pyc -------------------------------------------------------------------------------- /ML_model/models/data/.ipynb_checkpoints/Untitled-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [], 3 | "metadata": {}, 4 | "nbformat": 4, 5 | "nbformat_minor": 4 6 | } 7 | -------------------------------------------------------------------------------- /ML_model/models/data/50_Startups.csv: -------------------------------------------------------------------------------- 1 | R&D Spend,Administration,Marketing Spend,State,Profit 2 | 165349.2,136897.8,471784.1,New York,192261.83 3 | 162597.7,151377.59,443898.53,California,191792.06 4 | 153441.51,101145.55,407934.54,Florida,191050.39 5 | 144372.41,118671.85,383199.62,New York,182901.99 6 | 142107.34,91391.77,366168.42,Florida,166187.94 7 | 131876.9,99814.71,362861.36,New York,156991.12 8 | 134615.46,147198.87,127716.82,California,156122.51 9 | 130298.13,145530.06,323876.68,Florida,155752.6 10 | 120542.52,148718.95,311613.29,New York,152211.77 11 | 123334.88,108679.17,304981.62,California,149759.96 12 | 101913.08,110594.11,229160.95,Florida,146121.95 13 | 100671.96,91790.61,249744.55,California,144259.4 14 | 93863.75,127320.38,249839.44,Florida,141585.52 15 | 91992.39,135495.07,252664.93,California,134307.35 16 | 119943.24,156547.42,256512.92,Florida,132602.65 17 | 114523.61,122616.84,261776.23,New York,129917.04 18 | 78013.11,121597.55,264346.06,California,126992.93 19 | 94657.16,145077.58,282574.31,New York,125370.37 20 | 91749.16,114175.79,294919.57,Florida,124266.9 21 | 86419.7,153514.11,0,New York,122776.86 22 | 76253.86,113867.3,298664.47,California,118474.03 23 | 78389.47,153773.43,299737.29,New York,111313.02 24 | 73994.56,122782.75,303319.26,Florida,110352.25 25 | 67532.53,105751.03,304768.73,Florida,108733.99 26 | 77044.01,99281.34,140574.81,New York,108552.04 27 | 64664.71,139553.16,137962.62,California,107404.34 28 | 75328.87,144135.98,134050.07,Florida,105733.54 29 | 72107.6,127864.55,353183.81,New York,105008.31 30 | 66051.52,182645.56,118148.2,Florida,103282.38 31 | 65605.48,153032.06,107138.38,New York,101004.64 32 | 61994.48,115641.28,91131.24,Florida,99937.59 33 | 61136.38,152701.92,88218.23,New York,97483.56 34 | 63408.86,129219.61,46085.25,California,97427.84 35 | 55493.95,103057.49,214634.81,Florida,96778.92 36 | 46426.07,157693.92,210797.67,California,96712.8 37 | 46014.02,85047.44,205517.64,New York,96479.51 38 | 28663.76,127056.21,201126.82,Florida,90708.19 39 | 44069.95,51283.14,197029.42,California,89949.14 40 | 20229.59,65947.93,185265.1,New York,81229.06 41 | 38558.51,82982.09,174999.3,California,81005.76 42 | 28754.33,118546.05,172795.67,California,78239.91 43 | 27892.92,84710.77,164470.71,Florida,77798.83 44 | 23640.93,96189.63,148001.11,California,71498.49 45 | 15505.73,127382.3,35534.17,New York,69758.98 46 | 22177.74,154806.14,28334.72,California,65200.33 47 | 1000.23,124153.04,1903.93,New York,64926.08 48 | 1315.46,115816.21,297114.46,Florida,49490.75 49 | 0,135426.92,0,California,42559.73 50 | 542.05,51743.15,0,New York,35673.41 51 | 0,116983.8,45173.06,California,14681.4 -------------------------------------------------------------------------------- /ML_model/models/data/preprocessed_data.csv: -------------------------------------------------------------------------------- 1 | ,R&D Spend,Administration,Marketing Spend,Profit,State_is_California,State_is_New York 2 | 0,165349.2,136897.8,471784.1,192261.83,0,1 3 | 1,162597.7,151377.59,443898.53,191792.06,1,0 4 | 2,153441.51,101145.55,407934.54,191050.39,0,0 5 | 3,144372.41,118671.85,383199.62,182901.99,0,1 6 | 4,142107.34,91391.77,366168.42,166187.94,0,0 7 | 5,131876.9,99814.71,362861.36,156991.12,0,1 8 | 6,134615.46,147198.87,127716.82,156122.51,1,0 9 | 7,130298.13,145530.06,323876.68,155752.6,0,0 10 | 8,120542.52,148718.95,311613.29,152211.77,0,1 11 | 9,123334.88,108679.17,304981.62,149759.96,1,0 12 | 10,101913.08,110594.11,229160.95,146121.95,0,0 13 | 11,100671.96,91790.61,249744.55,144259.4,1,0 14 | 12,93863.75,127320.38,249839.44,141585.52,0,0 15 | 13,91992.39,135495.07,252664.93,134307.35,1,0 16 | 14,119943.24,156547.42,256512.92,132602.65,0,0 17 | 15,114523.61,122616.84,261776.23,129917.04,0,1 18 | 16,78013.11,121597.55,264346.06,126992.93,1,0 19 | 17,94657.16,145077.58,282574.31,125370.37,0,1 20 | 18,91749.16,114175.79,294919.57,124266.9,0,0 21 | 19,86419.7,153514.11,0.0,122776.86,0,1 22 | 20,76253.86,113867.3,298664.47,118474.03,1,0 23 | 21,78389.47,153773.43,299737.29,111313.02,0,1 24 | 22,73994.56,122782.75,303319.26,110352.25,0,0 25 | 23,67532.53,105751.03,304768.73,108733.99,0,0 26 | 24,77044.01,99281.34,140574.81,108552.04,0,1 27 | 25,64664.71,139553.16,137962.62,107404.34,1,0 28 | 26,75328.87,144135.98,134050.07,105733.54,0,0 29 | 27,72107.6,127864.55,353183.81,105008.31,0,1 30 | 28,66051.52,182645.56,118148.2,103282.38,0,0 31 | 29,65605.48,153032.06,107138.38,101004.64,0,1 32 | 30,61994.48,115641.28,91131.24,99937.59,0,0 33 | 31,61136.38,152701.92,88218.23,97483.56,0,1 34 | 32,63408.86,129219.61,46085.25,97427.84,1,0 35 | 33,55493.95,103057.49,214634.81,96778.92,0,0 36 | 34,46426.07,157693.92,210797.67,96712.8,1,0 37 | 35,46014.02,85047.44,205517.64,96479.51,0,1 38 | 36,28663.76,127056.21,201126.82,90708.19,0,0 39 | 37,44069.95,51283.14,197029.42,89949.14,1,0 40 | 38,20229.59,65947.93,185265.1,81229.06,0,1 41 | 39,38558.51,82982.09,174999.3,81005.76,1,0 42 | 40,28754.33,118546.05,172795.67,78239.91,1,0 43 | 41,27892.92,84710.77,164470.71,77798.83,0,0 44 | 42,23640.93,96189.63,148001.11,71498.49,1,0 45 | 43,15505.73,127382.3,35534.17,69758.98,0,1 46 | 44,22177.74,154806.14,28334.72,65200.33,1,0 47 | 45,1000.23,124153.04,1903.93,64926.08,0,1 48 | 46,1315.46,115816.21,297114.46,49490.75,0,0 49 | 47,0.0,135426.92,0.0,42559.73,1,0 50 | 48,542.05,51743.15,0.0,35673.41,0,1 51 | 49,0.0,116983.8,45173.06,14681.4,1,0 52 | -------------------------------------------------------------------------------- /ML_model/models/data/scaled_data.csv: -------------------------------------------------------------------------------- 1 | ,Unnamed: 0,R&D Spend,Administration,Marketing Spend,State_is_California,State_is_New York,Profit 2 | 0,0.0,1.0,0.6517439310268491,1.0,0.0,1.0,192261.83 3 | 1,0.02040816326530612,0.9833594598582879,0.7619717267693455,0.9408933662664767,1.0,0.0,191792.06 4 | 2,0.04081632653061224,0.927984592607645,0.3795789541636034,0.864663603542383,0.0,0.0,191050.39 5 | 3,0.061224489795918366,0.8731364288427159,0.5129983902549908,0.8122351304336031,0.0,1.0,182901.99 6 | 4,0.08163265306122448,0.8594377233152625,0.3053280382623889,0.776135567095203,0.0,0.0,166187.94 7 | 5,0.1020408163265306,0.7975659997145434,0.36944789841721865,0.7691258777055013,0.0,1.0,156991.12 8 | 6,0.12244897959183673,0.8141282812375263,0.7301611069589005,0.27071031007615565,1.0,0.0,156122.51 9 | 7,0.14285714285714285,0.7880179039269618,0.7174572453826598,0.6864934193416015,0.0,0.0,155752.6 10 | 8,0.16326530612244897,0.7290178603827536,0.7417327573593728,0.6604997709757493,0.0,1.0,152211.77 11 | 9,0.18367346938775508,0.7459055139063268,0.4369288415971631,0.6464431929774658,1.0,0.0,149759.96 12 | 10,0.2040816326530612,0.6163506082883982,0.4515063745019315,0.48573266882033544,0.0,0.0,146121.95 13 | 11,0.22448979591836732,0.6088445544338891,0.3083642186250832,0.5293619475518568,1.0,0.0,144259.4 14 | 12,0.24489795918367346,0.5676698163643973,0.5788355604289264,0.5295630776874423,0.0,0.0,141585.52 15 | 13,0.26530612244897955,0.5563521928137541,0.6410656106974888,0.5355520247503043,1.0,0.0,134307.35 16 | 14,0.2857142857142857,0.7253935307821265,0.8013271984483843,0.5437082767308182,0.0,0.0,132602.65 17 | 15,0.3061224489795918,0.692616656143483,0.5430297340746311,0.5548644602478126,0.0,1.0,129917.04 18 | 16,0.32653061224489793,0.4718082095347301,0.5352703611885348,0.5603115068947851,1.0,0.0,126992.93 19 | 17,0.3469387755102041,0.5724682066801654,0.7140127290590412,0.5989483537067062,0.0,1.0,125370.37 20 | 18,0.36734693877551017,0.5548811847895242,0.4787720110515626,0.6251155348389232,0.0,0.0,124266.9 21 | 19,0.3877551020408163,0.522649640881238,0.7782360434590043,0.0,0.0,1.0,122776.86 22 | 20,0.4081632653061224,0.46116860559349543,0.4764236225246156,0.6330532758522384,1.0,0.0,118474.03 23 | 21,0.42857142857142855,0.4740843620652534,0.7802101240217711,0.6353272397268157,0.0,1.0,111313.02 24 | 22,0.44897959183673464,0.447504795910715,0.5442927284683095,0.6429196320944263,0.0,0.0,110352.25 25 | 23,0.4693877551020408,0.4084236875654675,0.4146382960971639,0.6459919484357357,0.0,0.0,108733.99 26 | 24,0.4897959183673469,0.4659472800594136,0.36538760476550297,0.2979642806953435,0.0,1.0,108552.04 27 | 25,0.5102040816326531,0.39107966654812965,0.6719579313474888,0.29242744721579217,1.0,0.0,107404.34 28 | 26,0.5306122448979591,0.45557444487182275,0.7068447734138883,0.28413435298052653,0.0,0.0,105733.54 29 | 27,0.5510204081632653,0.4360928265755141,0.5829780693747877,0.7486132110005403,0.0,1.0,105008.31 30 | 28,0.5714285714285714,0.39946682536111455,1.0,0.2504285328818839,0.0,0.0,103282.38 31 | 29,0.5918367346938775,0.3967692616595665,0.7745664247050259,0.22709196855086894,0.0,1.0,101004.64 32 | 30,0.6122448979591836,0.37493063165712326,0.48992809359023687,0.19316301672735475,0.0,0.0,99937.59 33 | 31,0.6326530612244897,0.3697410087257755,0.7720532249634258,0.1869885610812234,0.0,1.0,97483.56 34 | 32,0.6530612244897959,0.3834845285008939,0.5932935005308215,0.09768292318456684,1.0,0.0,97427.84 35 | 33,0.673469387755102,0.33561668275383244,0.3941336494866646,0.45494286475529805,0.0,0.0,96778.92 36 | 34,0.6938775510204082,0.28077589731308045,0.8100549609241365,0.44680961058246776,1.0,0.0,96712.8 37 | 35,0.7142857142857142,0.2782838985613477,0.25703165334499783,0.4356179871258909,0.0,1.0,96479.51 38 | 36,0.7346938775510203,0.17335287984459555,0.5768245591090664,0.42631114528870306,0.0,0.0,90708.19 39 | 37,0.7551020408163265,0.26652653898537154,0.0,0.41762624047737096,1.0,0.0,89949.14 40 | 38,0.7755102040816326,0.1223446499892349,0.11163611328110429,0.39269042767655804,0.0,1.0,81229.06 41 | 39,0.7959183673469387,0.23319441521337872,0.24130912021870482,0.37093089826469355,1.0,0.0,81005.76 42 | 40,0.8163265306122448,0.17390062969763387,0.5120407343287374,0.366260054122214,1.0,0.0,78239.91 43 | 41,0.836734693877551,0.16869098852610112,0.25446874380054824,0.34861435559189047,0.0,0.0,77798.83 44 | 42,0.8571428571428571,0.14297577490547278,0.34185187818555723,0.31370516725765024,1.0,0.0,71498.49 45 | 43,0.8775510204081632,0.09377565781993502,0.5793069281153622,0.07531871040164347,0.0,1.0,69758.98 46 | 44,0.8979591836734693,0.13412668461655697,0.788071657023371,0.06005865818708177,1.0,0.0,65200.33 47 | 45,0.9183673469387754,0.006049197697962857,0.554724098414143,0.004035595943144333,0.0,1.0,64926.08 48 | 46,0.9387755102040816,0.007955647804767122,0.4912597529795813,0.6297678535584392,0.0,0.0,49490.75 49 | 47,0.9591836734693877,0.0,0.6405468169663746,0.0,1.0,0.0,42559.73 50 | 48,0.9795918367346939,0.003278213623047465,0.0035018386537032375,0.0,0.0,1.0,35673.41 51 | 49,0.9999999999999999,0.0,0.5001480636547349,0.09574943284438793,1.0,0.0,14681.4 52 | -------------------------------------------------------------------------------- /ML_model/models/data/testing_data.csv: -------------------------------------------------------------------------------- 1 | ,Unnamed: 0,Unnamed: 0.1,R&D Spend,Administration,Marketing Spend,State_is_California,State_is_New York,Profit 2 | 28,28,0.5714285714285714,0.39946682536111455,1.0,0.2504285328818839,0.0,0.0,103282.38 3 | 11,11,0.22448979591836726,0.6088445544338891,0.3083642186250832,0.5293619475518568,1.0,0.0,144259.4 4 | 10,10,0.2040816326530612,0.6163506082883982,0.4515063745019315,0.4857326688203354,0.0,0.0,146121.95 5 | 41,41,0.836734693877551,0.16869098852610112,0.25446874380054824,0.3486143555918905,0.0,0.0,77798.83 6 | 2,2,0.040816326530612235,0.927984592607645,0.3795789541636034,0.8646636035423829,0.0,0.0,191050.39 7 | 27,27,0.5510204081632653,0.4360928265755141,0.5829780693747877,0.7486132110005403,0.0,1.0,105008.31 8 | 38,38,0.7755102040816326,0.1223446499892349,0.11163611328110427,0.3926904276765581,0.0,1.0,81229.06 9 | 31,31,0.6326530612244897,0.3697410087257755,0.7720532249634258,0.1869885610812234,0.0,1.0,97483.56 10 | 22,22,0.4489795918367345,0.447504795910715,0.5442927284683095,0.6429196320944263,0.0,0.0,110352.25 11 | 4,4,0.08163265306122447,0.8594377233152625,0.3053280382623889,0.7761355670952029,0.0,0.0,166187.94 12 | -------------------------------------------------------------------------------- /ML_model/models/data/training_data.csv: -------------------------------------------------------------------------------- 1 | ,Unnamed: 0,Unnamed: 0.1,R&D Spend,Administration,Marketing Spend,State_is_California,State_is_New York,Profit 2 | 33,33,0.6734693877551021,0.33561668275383244,0.3941336494866646,0.4549428647552981,0.0,0.0,96778.92 3 | 35,35,0.7142857142857142,0.2782838985613477,0.25703165334499783,0.4356179871258909,0.0,1.0,96479.51 4 | 26,26,0.5306122448979591,0.4555744448718228,0.7068447734138883,0.2841343529805265,0.0,0.0,105733.54 5 | 34,34,0.6938775510204082,0.28077589731308045,0.8100549609241365,0.44680961058246776,1.0,0.0,96712.8 6 | 18,18,0.36734693877551017,0.5548811847895242,0.4787720110515626,0.6251155348389232,0.0,0.0,124266.9 7 | 7,7,0.14285714285714285,0.7880179039269618,0.7174572453826598,0.6864934193416015,0.0,0.0,155752.6 8 | 14,14,0.2857142857142857,0.7253935307821265,0.8013271984483843,0.5437082767308182,0.0,0.0,132602.65 9 | 45,45,0.9183673469387754,0.006049197697962857,0.5547240984141429,0.004035595943144333,0.0,1.0,64926.08 10 | 48,48,0.979591836734694,0.003278213623047465,0.003501838653703237,0.0,0.0,1.0,35673.41 11 | 29,29,0.5918367346938775,0.3967692616595665,0.7745664247050259,0.2270919685508689,0.0,1.0,101004.64 12 | 15,15,0.3061224489795918,0.6926166561434829,0.5430297340746311,0.5548644602478126,0.0,1.0,129917.04 13 | 30,30,0.6122448979591836,0.3749306316571233,0.4899280935902369,0.19316301672735475,0.0,0.0,99937.59 14 | 32,32,0.6530612244897959,0.3834845285008939,0.5932935005308215,0.09768292318456684,1.0,0.0,97427.84 15 | 16,16,0.32653061224489793,0.4718082095347301,0.5352703611885348,0.5603115068947851,1.0,0.0,126992.93 16 | 42,42,0.8571428571428571,0.14297577490547278,0.34185187818555723,0.31370516725765024,1.0,0.0,71498.49 17 | 20,20,0.4081632653061224,0.4611686055934954,0.4764236225246156,0.6330532758522384,1.0,0.0,118474.03 18 | 43,43,0.8775510204081632,0.09377565781993502,0.5793069281153622,0.07531871040164348,0.0,1.0,69758.98 19 | 8,8,0.16326530612244894,0.7290178603827536,0.7417327573593728,0.6604997709757493,0.0,1.0,152211.77 20 | 13,13,0.26530612244897955,0.5563521928137541,0.6410656106974888,0.5355520247503043,1.0,0.0,134307.35 21 | 25,25,0.5102040816326531,0.3910796665481297,0.6719579313474888,0.29242744721579217,1.0,0.0,107404.34 22 | 5,5,0.1020408163265306,0.7975659997145434,0.36944789841721865,0.7691258777055013,0.0,1.0,156991.12 23 | 17,17,0.3469387755102041,0.5724682066801654,0.7140127290590412,0.5989483537067062,0.0,1.0,125370.37 24 | 40,40,0.8163265306122448,0.17390062969763387,0.5120407343287374,0.366260054122214,1.0,0.0,78239.91 25 | 49,49,1.0,0.0,0.5001480636547349,0.09574943284438793,1.0,0.0,14681.4 26 | 1,1,0.020408163265306117,0.983359459858288,0.7619717267693455,0.9408933662664768,1.0,0.0,191792.06 27 | 12,12,0.24489795918367346,0.5676698163643973,0.5788355604289264,0.5295630776874423,0.0,0.0,141585.52 28 | 37,37,0.7551020408163265,0.26652653898537154,0.0,0.41762624047737096,1.0,0.0,89949.14 29 | 24,24,0.4897959183673469,0.4659472800594136,0.36538760476550297,0.2979642806953435,0.0,1.0,108552.04 30 | 6,6,0.12244897959183673,0.8141282812375263,0.7301611069589005,0.27071031007615565,1.0,0.0,156122.51 31 | 23,23,0.4693877551020408,0.4084236875654675,0.4146382960971639,0.6459919484357357,0.0,0.0,108733.99 32 | 36,36,0.7346938775510203,0.17335287984459555,0.5768245591090664,0.4263111452887031,0.0,0.0,90708.19 33 | 21,21,0.42857142857142855,0.4740843620652534,0.7802101240217711,0.6353272397268157,0.0,1.0,111313.02 34 | 19,19,0.3877551020408163,0.522649640881238,0.7782360434590043,0.0,0.0,1.0,122776.86 35 | 9,9,0.18367346938775508,0.7459055139063268,0.4369288415971631,0.6464431929774658,1.0,0.0,149759.96 36 | 39,39,0.7959183673469387,0.2331944152133787,0.2413091202187048,0.3709308982646936,1.0,0.0,81005.76 37 | 46,46,0.9387755102040816,0.007955647804767122,0.4912597529795813,0.6297678535584392,0.0,0.0,49490.75 38 | 3,3,0.061224489795918366,0.8731364288427159,0.5129983902549908,0.8122351304336031,0.0,1.0,182901.99 39 | 0,0,0.0,1.0,0.6517439310268491,1.0,0.0,1.0,192261.83 40 | 47,47,0.9591836734693876,0.0,0.6405468169663746,0.0,1.0,0.0,42559.73 41 | 44,44,0.8979591836734693,0.13412668461655694,0.788071657023371,0.060058658187081775,1.0,0.0,65200.33 42 | -------------------------------------------------------------------------------- /ML_model/models/data_is_available.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | def data_is_available(file_name): 4 | dataset = pd.read_csv(file_name) 5 | if dataset.empty: 6 | return False 7 | else: 8 | return dataset.shape 9 | -------------------------------------------------------------------------------- /ML_model/models/data_preprocessing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rahul765/Machine-Learning-Pipelines/27ee52614b755912bcd5038a42fe67e148d79c57/ML_model/models/data_preprocessing.py -------------------------------------------------------------------------------- /ML_model/models/results/BestModel_results.csv: -------------------------------------------------------------------------------- 1 | ,ErrorMetric,BestModel 2 | 0,MSE,RF 3 | 1,RMSE,RF 4 | 2,Adjusted_RSquare_test,RF 5 | 3,MAE,RF 6 | -------------------------------------------------------------------------------- /ML_model/models/results/DT_results.csv: -------------------------------------------------------------------------------- 1 | ,Test Values 2 | MSE,40862842.38545003 3 | RMSE,6392.405054863938 4 | Adjusted_RSquare_test,0.9680482359810527 5 | MAE,4492.121000000005 6 | -------------------------------------------------------------------------------- /ML_model/models/results/GBR_results.csv: -------------------------------------------------------------------------------- 1 | ,Test Values 2 | MSE,22013586.819728285 3 | RMSE,4691.863896121486 4 | Adjusted_RSquare_test,0.9827869798033184 5 | MAE,3267.817215788811 6 | -------------------------------------------------------------------------------- /ML_model/models/results/LR_results.csv: -------------------------------------------------------------------------------- 1 | ,Test Values 2 | MSE,52640097.52834 3 | RMSE,7255.349580023005 4 | Adjusted_RSquare_test,0.9588392809708515 5 | MAE,6105.234114534578 6 | -------------------------------------------------------------------------------- /ML_model/models/results/RF_results.csv: -------------------------------------------------------------------------------- 1 | ,Test Values 2 | MSE,8687883.621601839 3 | RMSE,2947.5216066386756 4 | Adjusted_RSquare_test,0.9932067083174728 5 | MAE,2241.563400000002 6 | -------------------------------------------------------------------------------- /ML_model/models/results/SVR_results.csv: -------------------------------------------------------------------------------- 1 | ,Test Values 2 | MSE,1482716427.2632544 3 | RMSE,38506.05702046438 4 | Adjusted_RSquare_test,-0.15937616243262132 5 | MAE,29308.70709436671 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Machine-Learning-Pipelines : Automated 2 | 3 | 4 | 5 | ![](Images/Flow.png) 6 | 7 | 8 | ![](Images/Workflow_Regression.png) 9 | 10 | ![](Images/graph.png) 11 | -------------------------------------------------------------------------------- /docker-airflow/Dockerfile: -------------------------------------------------------------------------------- 1 | # VERSION 1.10.9 2 | # AUTHOR: Matthieu "Puckel_" Roisil 3 | # DESCRIPTION: Basic Airflow container 4 | # BUILD: docker build --rm -t puckel/docker-airflow . 5 | # SOURCE: https://github.com/puckel/docker-airflow 6 | 7 | FROM python:3.7-slim-buster 8 | LABEL maintainer="Rahul Pal" 9 | 10 | # Never prompt the user for choices on installation/configuration of packages 11 | ENV DEBIAN_FRONTEND noninteractive 12 | ENV TERM linux 13 | 14 | # Airflow 15 | ARG AIRFLOW_VERSION=1.10.9 16 | ARG AIRFLOW_USER_HOME=/usr/local/airflow 17 | ARG AIRFLOW_DEPS="" 18 | ARG PYTHON_DEPS="" 19 | ENV AIRFLOW_HOME=${AIRFLOW_USER_HOME} 20 | 21 | # Define en_US. 22 | ENV LANGUAGE en_US.UTF-8 23 | ENV LANG en_US.UTF-8 24 | ENV LC_ALL en_US.UTF-8 25 | ENV LC_CTYPE en_US.UTF-8 26 | ENV LC_MESSAGES en_US.UTF-8 27 | 28 | # Disable noisy "Handling signal" log messages: 29 | # ENV GUNICORN_CMD_ARGS --log-level WARNING 30 | 31 | RUN set -ex \ 32 | && buildDeps=' \ 33 | freetds-dev \ 34 | libkrb5-dev \ 35 | libsasl2-dev \ 36 | libssl-dev \ 37 | libffi-dev \ 38 | libpq-dev \ 39 | git \ 40 | ' \ 41 | && apt-get update -yqq \ 42 | && apt-get upgrade -yqq \ 43 | && apt-get install -yqq --no-install-recommends \ 44 | $buildDeps \ 45 | freetds-bin \ 46 | build-essential \ 47 | default-libmysqlclient-dev \ 48 | apt-utils \ 49 | curl \ 50 | rsync \ 51 | netcat \ 52 | locales \ 53 | && sed -i 's/^# en_US.UTF-8 UTF-8$/en_US.UTF-8 UTF-8/g' /etc/locale.gen \ 54 | && locale-gen \ 55 | && update-locale LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 \ 56 | && useradd -ms /bin/bash -d ${AIRFLOW_USER_HOME} airflow \ 57 | && pip install -U pip setuptools wheel \ 58 | && pip install pytz \ 59 | && pip install pyOpenSSL \ 60 | && pip install -U scikit-learn \ 61 | # && pip install 'tensorflow>=2.1,<2.2' \ 62 | # && pip install 'tensorboard>=2.1,<2.3' \ 63 | # && pip install 'tfx>=0.21.1,<0.22' \ 64 | && pip install ndg-httpsclient \ 65 | && pip install pyasn1 \ 66 | && pip install apache-airflow[crypto,celery,postgres,hive,jdbc,mysql,ssh${AIRFLOW_DEPS:+,}${AIRFLOW_DEPS}]==${AIRFLOW_VERSION} \ 67 | && pip install 'redis==3.2' \ 68 | && if [ -n "${PYTHON_DEPS}" ]; then pip install ${PYTHON_DEPS}; fi \ 69 | && apt-get purge --auto-remove -yqq $buildDeps \ 70 | && apt-get autoremove -yqq --purge \ 71 | && apt-get clean \ 72 | && rm -rf \ 73 | /var/lib/apt/lists/* \ 74 | /tmp/* \ 75 | /var/tmp/* \ 76 | /usr/share/man \ 77 | /usr/share/doc \ 78 | /usr/share/doc-base 79 | 80 | COPY script/entrypoint.sh /entrypoint.sh 81 | COPY config/airflow.cfg ${AIRFLOW_USER_HOME}/airflow.cfg 82 | 83 | RUN chown -R airflow: ${AIRFLOW_USER_HOME} 84 | 85 | EXPOSE 8080 5555 8793 86 | 87 | USER airflow 88 | WORKDIR ${AIRFLOW_USER_HOME} 89 | ENTRYPOINT ["/entrypoint.sh"] 90 | CMD ["webserver"] 91 | -------------------------------------------------------------------------------- /docker-airflow/config/airflow.cfg: -------------------------------------------------------------------------------- 1 | [core] 2 | # The folder where your airflow pipelines live, most likely a 3 | # subfolder in a code repository. This path must be absolute. 4 | dags_folder = /usr/local/airflow/dags 5 | 6 | # The folder where airflow should store its log files 7 | # This path must be absolute 8 | base_log_folder = /usr/local/airflow/logs 9 | 10 | # Airflow can store logs remotely in AWS S3, Google Cloud Storage or Elastic Search. 11 | # Set this to True if you want to enable remote logging. 12 | remote_logging = False 13 | 14 | # Users must supply an Airflow connection id that provides access to the storage 15 | # location. 16 | remote_log_conn_id = 17 | remote_base_log_folder = 18 | encrypt_s3_logs = False 19 | 20 | # Logging level 21 | logging_level = INFO 22 | 23 | # Logging level for Flask-appbuilder UI 24 | fab_logging_level = WARN 25 | 26 | # Logging class 27 | # Specify the class that will specify the logging configuration 28 | # This class has to be on the python classpath 29 | # Example: logging_config_class = my.path.default_local_settings.LOGGING_CONFIG 30 | logging_config_class = 31 | 32 | # Flag to enable/disable Colored logs in Console 33 | # Colour the logs when the controlling terminal is a TTY. 34 | colored_console_log = True 35 | 36 | # Log format for when Colored logs is enabled 37 | colored_log_format = [%%(blue)s%%(asctime)s%%(reset)s] {{%%(blue)s%%(filename)s:%%(reset)s%%(lineno)d}} %%(log_color)s%%(levelname)s%%(reset)s - %%(log_color)s%%(message)s%%(reset)s 38 | colored_formatter_class = airflow.utils.log.colored_log.CustomTTYColoredFormatter 39 | 40 | # Format of Log line 41 | log_format = [%%(asctime)s] {{%%(filename)s:%%(lineno)d}} %%(levelname)s - %%(message)s 42 | simple_log_format = %%(asctime)s %%(levelname)s - %%(message)s 43 | 44 | # Log filename format 45 | log_filename_template = {{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{{ try_number }}.log 46 | log_processor_filename_template = {{ filename }}.log 47 | dag_processor_manager_log_location = /usr/local/airflow/logs/dag_processor_manager/dag_processor_manager.log 48 | 49 | # Name of handler to read task instance logs. 50 | # Default to use task handler. 51 | task_log_reader = task 52 | 53 | # Hostname by providing a path to a callable, which will resolve the hostname. 54 | # The format is "package:function". 55 | # 56 | # For example, default value "socket:getfqdn" means that result from getfqdn() of "socket" 57 | # package will be used as hostname. 58 | # 59 | # No argument should be required in the function specified. 60 | # If using IP address as hostname is preferred, use value ``airflow.utils.net:get_host_ip_address`` 61 | hostname_callable = socket:getfqdn 62 | 63 | # Default timezone in case supplied date times are naive 64 | # can be utc (default), system, or any IANA timezone string (e.g. Europe/Amsterdam) 65 | default_timezone = utc 66 | 67 | # The executor class that airflow should use. Choices include 68 | # SequentialExecutor, LocalExecutor, CeleryExecutor, DaskExecutor, KubernetesExecutor 69 | executor = SequentialExecutor 70 | 71 | # The SqlAlchemy connection string to the metadata database. 72 | # SqlAlchemy supports many different database engine, more information 73 | # their website 74 | # sql_alchemy_conn = sqlite:////tmp/airflow.db 75 | 76 | # The encoding for the databases 77 | sql_engine_encoding = utf-8 78 | 79 | # If SqlAlchemy should pool database connections. 80 | sql_alchemy_pool_enabled = True 81 | 82 | # The SqlAlchemy pool size is the maximum number of database connections 83 | # in the pool. 0 indicates no limit. 84 | sql_alchemy_pool_size = 5 85 | 86 | # The maximum overflow size of the pool. 87 | # When the number of checked-out connections reaches the size set in pool_size, 88 | # additional connections will be returned up to this limit. 89 | # When those additional connections are returned to the pool, they are disconnected and discarded. 90 | # It follows then that the total number of simultaneous connections the pool will allow 91 | # is pool_size + max_overflow, 92 | # and the total number of "sleeping" connections the pool will allow is pool_size. 93 | # max_overflow can be set to -1 to indicate no overflow limit; 94 | # no limit will be placed on the total number of concurrent connections. Defaults to 10. 95 | sql_alchemy_max_overflow = 10 96 | 97 | # The SqlAlchemy pool recycle is the number of seconds a connection 98 | # can be idle in the pool before it is invalidated. This config does 99 | # not apply to sqlite. If the number of DB connections is ever exceeded, 100 | # a lower config value will allow the system to recover faster. 101 | sql_alchemy_pool_recycle = 1800 102 | 103 | # Check connection at the start of each connection pool checkout. 104 | # Typically, this is a simple statement like "SELECT 1". 105 | # More information here: 106 | # https://docs.sqlalchemy.org/en/13/core/pooling.html#disconnect-handling-pessimistic 107 | sql_alchemy_pool_pre_ping = True 108 | 109 | # The schema to use for the metadata database. 110 | # SqlAlchemy supports databases with the concept of multiple schemas. 111 | sql_alchemy_schema = 112 | 113 | # The amount of parallelism as a setting to the executor. This defines 114 | # the max number of task instances that should run simultaneously 115 | # on this airflow installation 116 | parallelism = 32 117 | 118 | # The number of task instances allowed to run concurrently by the scheduler 119 | dag_concurrency = 16 120 | 121 | # Are DAGs paused by default at creation 122 | dags_are_paused_at_creation = True 123 | 124 | # The maximum number of active DAG runs per DAG 125 | max_active_runs_per_dag = 16 126 | 127 | # Whether to load the examples that ship with Airflow. It's good to 128 | # get started, but you probably want to set this to False in a production 129 | # environment 130 | load_examples = True 131 | 132 | # Where your Airflow plugins are stored 133 | plugins_folder = /usr/local/airflow/plugins 134 | 135 | # Secret key to save connection passwords in the db 136 | fernet_key = $FERNET_KEY 137 | 138 | # Whether to disable pickling dags 139 | donot_pickle = False 140 | 141 | # How long before timing out a python file import 142 | dagbag_import_timeout = 30 143 | 144 | # How long before timing out a DagFileProcessor, which processes a dag file 145 | dag_file_processor_timeout = 50 146 | 147 | # The class to use for running task instances in a subprocess 148 | task_runner = StandardTaskRunner 149 | 150 | # If set, tasks without a ``run_as_user`` argument will be run with this user 151 | # Can be used to de-elevate a sudo user running Airflow when executing tasks 152 | default_impersonation = 153 | 154 | # What security module to use (for example kerberos) 155 | security = 156 | 157 | # If set to False enables some unsecure features like Charts and Ad Hoc Queries. 158 | # In 2.0 will default to True. 159 | secure_mode = False 160 | 161 | # Turn unit test mode on (overwrites many configuration options with test 162 | # values at runtime) 163 | unit_test_mode = False 164 | 165 | # Whether to enable pickling for xcom (note that this is insecure and allows for 166 | # RCE exploits). This will be deprecated in Airflow 2.0 (be forced to False). 167 | enable_xcom_pickling = True 168 | 169 | # When a task is killed forcefully, this is the amount of time in seconds that 170 | # it has to cleanup after it is sent a SIGTERM, before it is SIGKILLED 171 | killed_task_cleanup_time = 60 172 | 173 | # Whether to override params with dag_run.conf. If you pass some key-value pairs 174 | # through ``airflow dags backfill -c`` or 175 | # ``airflow dags trigger -c``, the key-value pairs will override the existing ones in params. 176 | dag_run_conf_overrides_params = False 177 | 178 | # Worker initialisation check to validate Metadata Database connection 179 | worker_precheck = False 180 | 181 | # When discovering DAGs, ignore any files that don't contain the strings ``DAG`` and ``airflow``. 182 | dag_discovery_safe_mode = True 183 | 184 | # The number of retries each task is going to have by default. Can be overridden at dag or task level. 185 | default_task_retries = 0 186 | 187 | # Whether to serialises DAGs and persist them in DB. 188 | # If set to True, Webserver reads from DB instead of parsing DAG files 189 | # More details: https://airflow.apache.org/docs/stable/dag-serialization.html 190 | store_serialized_dags = False 191 | 192 | # Updating serialized DAG can not be faster than a minimum interval to reduce database write rate. 193 | min_serialized_dag_update_interval = 30 194 | 195 | # On each dagrun check against defined SLAs 196 | check_slas = True 197 | 198 | [cli] 199 | # In what way should the cli access the API. The LocalClient will use the 200 | # database directly, while the json_client will use the api running on the 201 | # webserver 202 | api_client = airflow.api.client.local_client 203 | 204 | # If you set web_server_url_prefix, do NOT forget to append it here, ex: 205 | # ``endpoint_url = http://localhost:8080/myroot`` 206 | # So api will look like: ``http://localhost:8080/myroot/api/experimental/...`` 207 | endpoint_url = http://localhost:8080 208 | 209 | [debug] 210 | # Used only with DebugExecutor. If set to True DAG will fail with first 211 | # failed task. Helpful for debugging purposes. 212 | fail_fast = False 213 | 214 | [api] 215 | # How to authenticate users of the API 216 | auth_backend = airflow.api.auth.backend.default 217 | 218 | [lineage] 219 | # what lineage backend to use 220 | backend = 221 | 222 | [atlas] 223 | sasl_enabled = False 224 | host = 225 | port = 21000 226 | username = 227 | password = 228 | 229 | [operators] 230 | # The default owner assigned to each new operator, unless 231 | # provided explicitly or passed via ``default_args`` 232 | default_owner = airflow 233 | default_cpus = 1 234 | default_ram = 512 235 | default_disk = 512 236 | default_gpus = 0 237 | 238 | [hive] 239 | # Default mapreduce queue for HiveOperator tasks 240 | default_hive_mapred_queue = 241 | 242 | [webserver] 243 | # The base url of your website as airflow cannot guess what domain or 244 | # cname you are using. This is used in automated emails that 245 | # airflow sends to point links to the right web server 246 | base_url = http://localhost:8080 247 | 248 | # The ip specified when starting the web server 249 | web_server_host = 0.0.0.0 250 | 251 | # The port on which to run the web server 252 | web_server_port = 8080 253 | 254 | # Paths to the SSL certificate and key for the web server. When both are 255 | # provided SSL will be enabled. This does not change the web server port. 256 | web_server_ssl_cert = 257 | 258 | # Paths to the SSL certificate and key for the web server. When both are 259 | # provided SSL will be enabled. This does not change the web server port. 260 | web_server_ssl_key = 261 | 262 | # Number of seconds the webserver waits before killing gunicorn master that doesn't respond 263 | web_server_master_timeout = 120 264 | 265 | # Number of seconds the gunicorn webserver waits before timing out on a worker 266 | web_server_worker_timeout = 120 267 | 268 | # Number of workers to refresh at a time. When set to 0, worker refresh is 269 | # disabled. When nonzero, airflow periodically refreshes webserver workers by 270 | # bringing up new ones and killing old ones. 271 | worker_refresh_batch_size = 1 272 | 273 | # Number of seconds to wait before refreshing a batch of workers. 274 | worker_refresh_interval = 30 275 | 276 | # Secret key used to run your flask app 277 | # It should be as random as possible 278 | secret_key = temporary_key 279 | 280 | # Number of workers to run the Gunicorn web server 281 | workers = 4 282 | 283 | # The worker class gunicorn should use. Choices include 284 | # sync (default), eventlet, gevent 285 | worker_class = sync 286 | 287 | # Log files for the gunicorn webserver. '-' means log to stderr. 288 | access_logfile = - 289 | 290 | # Log files for the gunicorn webserver. '-' means log to stderr. 291 | error_logfile = - 292 | 293 | # Expose the configuration file in the web server 294 | expose_config = True 295 | 296 | # Expose hostname in the web server 297 | expose_hostname = True 298 | 299 | # Expose stacktrace in the web server 300 | expose_stacktrace = True 301 | 302 | # Set to true to turn on authentication: 303 | # https://airflow.apache.org/security.html#web-authentication 304 | authenticate = False 305 | 306 | # Filter the list of dags by owner name (requires authentication to be enabled) 307 | filter_by_owner = False 308 | 309 | # Filtering mode. Choices include user (default) and ldapgroup. 310 | # Ldap group filtering requires using the ldap backend 311 | # 312 | # Note that the ldap server needs the "memberOf" overlay to be set up 313 | # in order to user the ldapgroup mode. 314 | owner_mode = user 315 | 316 | # Default DAG view. Valid values are: 317 | # tree, graph, duration, gantt, landing_times 318 | dag_default_view = tree 319 | 320 | # "Default DAG orientation. Valid values are:" 321 | # LR (Left->Right), TB (Top->Bottom), RL (Right->Left), BT (Bottom->Top) 322 | dag_orientation = LR 323 | 324 | # Puts the webserver in demonstration mode; blurs the names of Operators for 325 | # privacy. 326 | demo_mode = False 327 | 328 | # The amount of time (in secs) webserver will wait for initial handshake 329 | # while fetching logs from other worker machine 330 | log_fetch_timeout_sec = 5 331 | 332 | # Time interval (in secs) to wait before next log fetching. 333 | log_fetch_delay_sec = 2 334 | 335 | # Distance away from page bottom to enable auto tailing. 336 | log_auto_tailing_offset = 30 337 | 338 | # Animation speed for auto tailing log display. 339 | log_animation_speed = 1000 340 | 341 | # By default, the webserver shows paused DAGs. Flip this to hide paused 342 | # DAGs by default 343 | hide_paused_dags_by_default = False 344 | 345 | # Consistent page size across all listing views in the UI 346 | page_size = 100 347 | 348 | # Use FAB-based webserver with RBAC feature 349 | rbac = False 350 | 351 | # Define the color of navigation bar 352 | navbar_color = #007A87 353 | 354 | # Default dagrun to show in UI 355 | default_dag_run_display_number = 25 356 | 357 | # Enable werkzeug ``ProxyFix`` middleware for reverse proxy 358 | enable_proxy_fix = False 359 | 360 | # Number of values to trust for ``X-Forwarded-For``. 361 | # More info: https://werkzeug.palletsprojects.com/en/0.16.x/middleware/proxy_fix/ 362 | proxy_fix_x_for = 1 363 | 364 | # Number of values to trust for ``X-Forwarded-Proto`` 365 | proxy_fix_x_proto = 1 366 | 367 | # Number of values to trust for ``X-Forwarded-Host`` 368 | proxy_fix_x_host = 1 369 | 370 | # Number of values to trust for ``X-Forwarded-Port`` 371 | proxy_fix_x_port = 1 372 | 373 | # Number of values to trust for ``X-Forwarded-Prefix`` 374 | proxy_fix_x_prefix = 1 375 | 376 | # Set secure flag on session cookie 377 | cookie_secure = False 378 | 379 | # Set samesite policy on session cookie 380 | cookie_samesite = 381 | 382 | # Default setting for wrap toggle on DAG code and TI log views. 383 | default_wrap = False 384 | 385 | # Allow the UI to be rendered in a frame 386 | x_frame_enabled = True 387 | 388 | # Send anonymous user activity to your analytics tool 389 | # choose from google_analytics, segment, or metarouter 390 | # analytics_tool = 391 | 392 | # Unique ID of your account in the analytics tool 393 | # analytics_id = 394 | 395 | # Update FAB permissions and sync security manager roles 396 | # on webserver startup 397 | update_fab_perms = True 398 | 399 | # Minutes of non-activity before logged out from UI 400 | # 0 means never get forcibly logged out 401 | force_log_out_after = 0 402 | 403 | # The UI cookie lifetime in days 404 | session_lifetime_days = 30 405 | 406 | [email] 407 | email_backend = airflow.utils.email.send_email_smtp 408 | 409 | [smtp] 410 | 411 | # If you want airflow to send emails on retries, failure, and you want to use 412 | # the airflow.utils.email.send_email_smtp function, you have to configure an 413 | # smtp server here 414 | smtp_host = localhost 415 | smtp_starttls = True 416 | smtp_ssl = False 417 | # Example: smtp_user = airflow 418 | # smtp_user = 419 | # Example: smtp_password = airflow 420 | # smtp_password = 421 | smtp_port = 25 422 | smtp_mail_from = airflow@example.com 423 | 424 | [sentry] 425 | 426 | # Sentry (https://docs.sentry.io) integration 427 | sentry_dsn = 428 | 429 | [celery] 430 | 431 | # This section only applies if you are using the CeleryExecutor in 432 | # ``[core]`` section above 433 | # The app name that will be used by celery 434 | celery_app_name = airflow.executors.celery_executor 435 | 436 | # The concurrency that will be used when starting workers with the 437 | # ``airflow celery worker`` command. This defines the number of task instances that 438 | # a worker will take, so size up your workers based on the resources on 439 | # your worker box and the nature of your tasks 440 | worker_concurrency = 16 441 | 442 | # The maximum and minimum concurrency that will be used when starting workers with the 443 | # ``airflow celery worker`` command (always keep minimum processes, but grow 444 | # to maximum if necessary). Note the value should be max_concurrency,min_concurrency 445 | # Pick these numbers based on resources on worker box and the nature of the task. 446 | # If autoscale option is available, worker_concurrency will be ignored. 447 | # http://docs.celeryproject.org/en/latest/reference/celery.bin.worker.html#cmdoption-celery-worker-autoscale 448 | # Example: worker_autoscale = 16,12 449 | worker_autoscale = 16,12 450 | 451 | # When you start an airflow worker, airflow starts a tiny web server 452 | # subprocess to serve the workers local log files to the airflow main 453 | # web server, who then builds pages and sends them to users. This defines 454 | # the port on which the logs are served. It needs to be unused, and open 455 | # visible from the main web server to connect into the workers. 456 | worker_log_server_port = 8793 457 | 458 | # The Celery broker URL. Celery supports RabbitMQ, Redis and experimentally 459 | # a sqlalchemy database. Refer to the Celery documentation for more 460 | # information. 461 | # http://docs.celeryproject.org/en/latest/userguide/configuration.html#broker-settings 462 | broker_url = redis://redis:6379/1 463 | 464 | # The Celery result_backend. When a job finishes, it needs to update the 465 | # metadata of the job. Therefore it will post a message on a message bus, 466 | # or insert it into a database (depending of the backend) 467 | # This status is used by the scheduler to update the state of the task 468 | # The use of a database is highly recommended 469 | # http://docs.celeryproject.org/en/latest/userguide/configuration.html#task-result-backend-settings 470 | result_backend = db+postgresql://airflow:airflow@postgres/airflow 471 | 472 | # Celery Flower is a sweet UI for Celery. Airflow has a shortcut to start 473 | # it ``airflow flower``. This defines the IP that Celery Flower runs on 474 | flower_host = 0.0.0.0 475 | 476 | # The root URL for Flower 477 | # Example: flower_url_prefix = /flower 478 | flower_url_prefix = 479 | 480 | # This defines the port that Celery Flower runs on 481 | flower_port = 5555 482 | 483 | # Securing Flower with Basic Authentication 484 | # Accepts user:password pairs separated by a comma 485 | # Example: flower_basic_auth = user1:password1,user2:password2 486 | flower_basic_auth = 487 | 488 | # Default queue that tasks get assigned to and that worker listen on. 489 | default_queue = default 490 | 491 | # How many processes CeleryExecutor uses to sync task state. 492 | # 0 means to use max(1, number of cores - 1) processes. 493 | sync_parallelism = 0 494 | 495 | # Import path for celery configuration options 496 | celery_config_options = airflow.config_templates.default_celery.DEFAULT_CELERY_CONFIG 497 | 498 | # In case of using SSL 499 | ssl_active = False 500 | ssl_key = 501 | ssl_cert = 502 | ssl_cacert = 503 | 504 | # Celery Pool implementation. 505 | # Choices include: prefork (default), eventlet, gevent or solo. 506 | # See: 507 | # https://docs.celeryproject.org/en/latest/userguide/workers.html#concurrency 508 | # https://docs.celeryproject.org/en/latest/userguide/concurrency/eventlet.html 509 | pool = prefork 510 | 511 | # The number of seconds to wait before timing out ``send_task_to_executor`` or 512 | # ``fetch_celery_task_state`` operations. 513 | operation_timeout = 2 514 | 515 | [celery_broker_transport_options] 516 | 517 | # This section is for specifying options which can be passed to the 518 | # underlying celery broker transport. See: 519 | # http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-broker_transport_options 520 | # The visibility timeout defines the number of seconds to wait for the worker 521 | # to acknowledge the task before the message is redelivered to another worker. 522 | # Make sure to increase the visibility timeout to match the time of the longest 523 | # ETA you're planning to use. 524 | # visibility_timeout is only supported for Redis and SQS celery brokers. 525 | # See: 526 | # http://docs.celeryproject.org/en/master/userguide/configuration.html#std:setting-broker_transport_options 527 | # Example: visibility_timeout = 21600 528 | # visibility_timeout = 529 | 530 | [dask] 531 | 532 | # This section only applies if you are using the DaskExecutor in 533 | # [core] section above 534 | # The IP address and port of the Dask cluster's scheduler. 535 | cluster_address = 127.0.0.1:8786 536 | 537 | # TLS/ SSL settings to access a secured Dask scheduler. 538 | tls_ca = 539 | tls_cert = 540 | tls_key = 541 | 542 | [scheduler] 543 | # Task instances listen for external kill signal (when you clear tasks 544 | # from the CLI or the UI), this defines the frequency at which they should 545 | # listen (in seconds). 546 | job_heartbeat_sec = 5 547 | 548 | # The scheduler constantly tries to trigger new tasks (look at the 549 | # scheduler section in the docs for more information). This defines 550 | # how often the scheduler should run (in seconds). 551 | scheduler_heartbeat_sec = 5 552 | 553 | # After how much time should the scheduler terminate in seconds 554 | # -1 indicates to run continuously (see also num_runs) 555 | run_duration = -1 556 | 557 | # The number of times to try to schedule each DAG file 558 | # -1 indicates unlimited number 559 | num_runs = -1 560 | 561 | # The number of seconds to wait between consecutive DAG file processing 562 | processor_poll_interval = 1 563 | 564 | # after how much time (seconds) a new DAGs should be picked up from the filesystem 565 | min_file_process_interval = 0 566 | 567 | # How often (in seconds) to scan the DAGs directory for new files. Default to 5 minutes. 568 | dag_dir_list_interval = 300 569 | 570 | # How often should stats be printed to the logs. Setting to 0 will disable printing stats 571 | print_stats_interval = 30 572 | 573 | # If the last scheduler heartbeat happened more than scheduler_health_check_threshold 574 | # ago (in seconds), scheduler is considered unhealthy. 575 | # This is used by the health check in the "/health" endpoint 576 | scheduler_health_check_threshold = 30 577 | child_process_log_directory = /usr/local/airflow/logs/scheduler 578 | 579 | # Local task jobs periodically heartbeat to the DB. If the job has 580 | # not heartbeat in this many seconds, the scheduler will mark the 581 | # associated task instance as failed and will re-schedule the task. 582 | scheduler_zombie_task_threshold = 300 583 | 584 | # Turn off scheduler catchup by setting this to False. 585 | # Default behavior is unchanged and 586 | # Command Line Backfills still work, but the scheduler 587 | # will not do scheduler catchup if this is False, 588 | # however it can be set on a per DAG basis in the 589 | # DAG definition (catchup) 590 | catchup_by_default = True 591 | 592 | # This changes the batch size of queries in the scheduling main loop. 593 | # If this is too high, SQL query performance may be impacted by one 594 | # or more of the following: 595 | # - reversion to full table scan 596 | # - complexity of query predicate 597 | # - excessive locking 598 | # Additionally, you may hit the maximum allowable query length for your db. 599 | # Set this to 0 for no limit (not advised) 600 | max_tis_per_query = 512 601 | 602 | # Statsd (https://github.com/etsy/statsd) integration settings 603 | statsd_on = False 604 | statsd_host = localhost 605 | statsd_port = 8125 606 | statsd_prefix = airflow 607 | 608 | # If you want to avoid send all the available metrics to StatsD, 609 | # you can configure an allow list of prefixes to send only the metrics that 610 | # start with the elements of the list (e.g: scheduler,executor,dagrun) 611 | statsd_allow_list = 612 | 613 | # The scheduler can run multiple threads in parallel to schedule dags. 614 | # This defines how many threads will run. 615 | max_threads = 2 616 | authenticate = False 617 | 618 | # Turn off scheduler use of cron intervals by setting this to False. 619 | # DAGs submitted manually in the web UI or with trigger_dag will still run. 620 | use_job_schedule = True 621 | 622 | # Allow externally triggered DagRuns for Execution Dates in the future 623 | # Only has effect if schedule_interval is set to None in DAG 624 | allow_trigger_in_future = False 625 | 626 | [ldap] 627 | # set this to ldaps://: 628 | uri = 629 | user_filter = objectClass=* 630 | user_name_attr = uid 631 | group_member_attr = memberOf 632 | superuser_filter = 633 | data_profiler_filter = 634 | bind_user = cn=Manager,dc=example,dc=com 635 | bind_password = insecure 636 | basedn = dc=example,dc=com 637 | cacert = /etc/ca/ldap_ca.crt 638 | search_scope = LEVEL 639 | 640 | # This setting allows the use of LDAP servers that either return a 641 | # broken schema, or do not return a schema. 642 | ignore_malformed_schema = False 643 | 644 | [mesos] 645 | # Mesos master address which MesosExecutor will connect to. 646 | master = localhost:5050 647 | 648 | # The framework name which Airflow scheduler will register itself as on mesos 649 | framework_name = Airflow 650 | 651 | # Number of cpu cores required for running one task instance using 652 | # 'airflow run --local -p ' 653 | # command on a mesos slave 654 | task_cpu = 1 655 | 656 | # Memory in MB required for running one task instance using 657 | # 'airflow run --local -p ' 658 | # command on a mesos slave 659 | task_memory = 256 660 | 661 | # Enable framework checkpointing for mesos 662 | # See http://mesos.apache.org/documentation/latest/slave-recovery/ 663 | checkpoint = False 664 | 665 | # Failover timeout in milliseconds. 666 | # When checkpointing is enabled and this option is set, Mesos waits 667 | # until the configured timeout for 668 | # the MesosExecutor framework to re-register after a failover. Mesos 669 | # shuts down running tasks if the 670 | # MesosExecutor framework fails to re-register within this timeframe. 671 | # Example: failover_timeout = 604800 672 | # failover_timeout = 673 | 674 | # Enable framework authentication for mesos 675 | # See http://mesos.apache.org/documentation/latest/configuration/ 676 | authenticate = False 677 | 678 | # Mesos credentials, if authentication is enabled 679 | # Example: default_principal = admin 680 | # default_principal = 681 | # Example: default_secret = admin 682 | # default_secret = 683 | 684 | # Optional Docker Image to run on slave before running the command 685 | # This image should be accessible from mesos slave i.e mesos slave 686 | # should be able to pull this docker image before executing the command. 687 | # Example: docker_image_slave = puckel/docker-airflow 688 | # docker_image_slave = 689 | 690 | [kerberos] 691 | ccache = /tmp/airflow_krb5_ccache 692 | 693 | # gets augmented with fqdn 694 | principal = airflow 695 | reinit_frequency = 3600 696 | kinit_path = kinit 697 | keytab = airflow.keytab 698 | 699 | [github_enterprise] 700 | api_rev = v3 701 | 702 | [admin] 703 | # UI to hide sensitive variable fields when set to True 704 | hide_sensitive_variable_fields = True 705 | 706 | [elasticsearch] 707 | # Elasticsearch host 708 | host = 709 | 710 | # Format of the log_id, which is used to query for a given tasks logs 711 | log_id_template = {{dag_id}}-{{task_id}}-{{execution_date}}-{{try_number}} 712 | 713 | # Used to mark the end of a log stream for a task 714 | end_of_log_mark = end_of_log 715 | 716 | # Qualified URL for an elasticsearch frontend (like Kibana) with a template argument for log_id 717 | # Code will construct log_id using the log_id template from the argument above. 718 | # NOTE: The code will prefix the https:// automatically, don't include that here. 719 | frontend = 720 | 721 | # Write the task logs to the stdout of the worker, rather than the default files 722 | write_stdout = False 723 | 724 | # Instead of the default log formatter, write the log lines as JSON 725 | json_format = False 726 | 727 | # Log fields to also attach to the json output, if enabled 728 | json_fields = asctime, filename, lineno, levelname, message 729 | 730 | [elasticsearch_configs] 731 | use_ssl = False 732 | verify_certs = True 733 | 734 | [kubernetes] 735 | # The repository, tag and imagePullPolicy of the Kubernetes Image for the Worker to Run 736 | worker_container_repository = 737 | worker_container_tag = 738 | worker_container_image_pull_policy = IfNotPresent 739 | 740 | # If True (default), worker pods will be deleted upon termination 741 | delete_worker_pods = True 742 | 743 | # Number of Kubernetes Worker Pod creation calls per scheduler loop 744 | worker_pods_creation_batch_size = 1 745 | 746 | # The Kubernetes namespace where airflow workers should be created. Defaults to ``default`` 747 | namespace = default 748 | 749 | # The name of the Kubernetes ConfigMap containing the Airflow Configuration (this file) 750 | # Example: airflow_configmap = airflow-configmap 751 | airflow_configmap = 752 | 753 | # The name of the Kubernetes ConfigMap containing ``airflow_local_settings.py`` file. 754 | # 755 | # For example: 756 | # 757 | # ``airflow_local_settings_configmap = "airflow-configmap"`` if you have the following ConfigMap. 758 | # 759 | # ``airflow-configmap.yaml``: 760 | # 761 | # .. code-block:: yaml 762 | # 763 | # --- 764 | # apiVersion: v1 765 | # kind: ConfigMap 766 | # metadata: 767 | # name: airflow-configmap 768 | # data: 769 | # airflow_local_settings.py: | 770 | # def pod_mutation_hook(pod): 771 | # ... 772 | # airflow.cfg: | 773 | # ... 774 | # Example: airflow_local_settings_configmap = airflow-configmap 775 | airflow_local_settings_configmap = 776 | 777 | # For docker image already contains DAGs, this is set to ``True``, and the worker will 778 | # search for dags in dags_folder, 779 | # otherwise use git sync or dags volume claim to mount DAGs 780 | dags_in_image = False 781 | 782 | # For either git sync or volume mounted DAGs, the worker will look in this subpath for DAGs 783 | dags_volume_subpath = 784 | 785 | # For DAGs mounted via a volume claim (mutually exclusive with git-sync and host path) 786 | dags_volume_claim = 787 | 788 | # For volume mounted logs, the worker will look in this subpath for logs 789 | logs_volume_subpath = 790 | 791 | # A shared volume claim for the logs 792 | logs_volume_claim = 793 | 794 | # For DAGs mounted via a hostPath volume (mutually exclusive with volume claim and git-sync) 795 | # Useful in local environment, discouraged in production 796 | dags_volume_host = 797 | 798 | # A hostPath volume for the logs 799 | # Useful in local environment, discouraged in production 800 | logs_volume_host = 801 | 802 | # A list of configMapsRefs to envFrom. If more than one configMap is 803 | # specified, provide a comma separated list: configmap_a,configmap_b 804 | env_from_configmap_ref = 805 | 806 | # A list of secretRefs to envFrom. If more than one secret is 807 | # specified, provide a comma separated list: secret_a,secret_b 808 | env_from_secret_ref = 809 | 810 | # Git credentials and repository for DAGs mounted via Git (mutually exclusive with volume claim) 811 | git_repo = 812 | git_branch = 813 | git_subpath = 814 | 815 | # The specific rev or hash the git_sync init container will checkout 816 | # This becomes GIT_SYNC_REV environment variable in the git_sync init container for worker pods 817 | git_sync_rev = 818 | 819 | # Use git_user and git_password for user authentication or git_ssh_key_secret_name 820 | # and git_ssh_key_secret_key for SSH authentication 821 | git_user = 822 | git_password = 823 | git_sync_root = /git 824 | git_sync_dest = repo 825 | 826 | # Mount point of the volume if git-sync is being used. 827 | # i.e. /usr/local/airflow/dags 828 | git_dags_folder_mount_point = 829 | 830 | # To get Git-sync SSH authentication set up follow this format 831 | # 832 | # ``airflow-secrets.yaml``: 833 | # 834 | # .. code-block:: yaml 835 | # 836 | # --- 837 | # apiVersion: v1 838 | # kind: Secret 839 | # metadata: 840 | # name: airflow-secrets 841 | # data: 842 | # # key needs to be gitSshKey 843 | # gitSshKey: 844 | # Example: git_ssh_key_secret_name = airflow-secrets 845 | git_ssh_key_secret_name = 846 | 847 | # To get Git-sync SSH authentication set up follow this format 848 | # 849 | # ``airflow-configmap.yaml``: 850 | # 851 | # .. code-block:: yaml 852 | # 853 | # --- 854 | # apiVersion: v1 855 | # kind: ConfigMap 856 | # metadata: 857 | # name: airflow-configmap 858 | # data: 859 | # known_hosts: | 860 | # github.com ssh-rsa <...> 861 | # airflow.cfg: | 862 | # ... 863 | # Example: git_ssh_known_hosts_configmap_name = airflow-configmap 864 | git_ssh_known_hosts_configmap_name = 865 | 866 | # To give the git_sync init container credentials via a secret, create a secret 867 | # with two fields: GIT_SYNC_USERNAME and GIT_SYNC_PASSWORD (example below) and 868 | # add ``git_sync_credentials_secret = `` to your airflow config under the 869 | # ``kubernetes`` section 870 | # 871 | # Secret Example: 872 | # 873 | # .. code-block:: yaml 874 | # 875 | # --- 876 | # apiVersion: v1 877 | # kind: Secret 878 | # metadata: 879 | # name: git-credentials 880 | # data: 881 | # GIT_SYNC_USERNAME: 882 | # GIT_SYNC_PASSWORD: 883 | git_sync_credentials_secret = 884 | 885 | # For cloning DAGs from git repositories into volumes: https://github.com/kubernetes/git-sync 886 | git_sync_container_repository = k8s.gcr.io/git-sync 887 | git_sync_container_tag = v3.1.1 888 | git_sync_init_container_name = git-sync-clone 889 | git_sync_run_as_user = 65533 890 | 891 | # The name of the Kubernetes service account to be associated with airflow workers, if any. 892 | # Service accounts are required for workers that require access to secrets or cluster resources. 893 | # See the Kubernetes RBAC documentation for more: 894 | # https://kubernetes.io/docs/admin/authorization/rbac/ 895 | worker_service_account_name = 896 | 897 | # Any image pull secrets to be given to worker pods, If more than one secret is 898 | # required, provide a comma separated list: secret_a,secret_b 899 | image_pull_secrets = 900 | 901 | # GCP Service Account Keys to be provided to tasks run on Kubernetes Executors 902 | # Should be supplied in the format: key-name-1:key-path-1,key-name-2:key-path-2 903 | gcp_service_account_keys = 904 | 905 | # Use the service account kubernetes gives to pods to connect to kubernetes cluster. 906 | # It's intended for clients that expect to be running inside a pod running on kubernetes. 907 | # It will raise an exception if called from a process not running in a kubernetes environment. 908 | in_cluster = True 909 | 910 | # When running with in_cluster=False change the default cluster_context or config_file 911 | # options to Kubernetes client. Leave blank these to use default behaviour like ``kubectl`` has. 912 | # cluster_context = 913 | # config_file = 914 | 915 | # Affinity configuration as a single line formatted JSON object. 916 | # See the affinity model for top-level key names (e.g. ``nodeAffinity``, etc.): 917 | # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#affinity-v1-core 918 | affinity = 919 | 920 | # A list of toleration objects as a single line formatted JSON array 921 | # See: 922 | # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#toleration-v1-core 923 | tolerations = 924 | 925 | # Keyword parameters to pass while calling a kubernetes client core_v1_api methods 926 | # from Kubernetes Executor provided as a single line formatted JSON dictionary string. 927 | # List of supported params are similar for all core_v1_apis, hence a single config 928 | # variable for all apis. 929 | # See: 930 | # https://raw.githubusercontent.com/kubernetes-client/python/master/kubernetes/client/apis/core_v1_api.py 931 | # Note that if no _request_timeout is specified, the kubernetes client will wait indefinitely 932 | # for kubernetes api responses, which will cause the scheduler to hang. 933 | # The timeout is specified as [connect timeout, read timeout] 934 | kube_client_request_args = {{"_request_timeout" : [60,60] }} 935 | 936 | # Specifies the uid to run the first process of the worker pods containers as 937 | run_as_user = 938 | 939 | # Specifies a gid to associate with all containers in the worker pods 940 | # if using a git_ssh_key_secret_name use an fs_group 941 | # that allows for the key to be read, e.g. 65533 942 | fs_group = 943 | 944 | [kubernetes_node_selectors] 945 | 946 | # The Key-value pairs to be given to worker pods. 947 | # The worker pods will be scheduled to the nodes of the specified key-value pairs. 948 | # Should be supplied in the format: key = value 949 | 950 | [kubernetes_annotations] 951 | 952 | # The Key-value annotations pairs to be given to worker pods. 953 | # Should be supplied in the format: key = value 954 | 955 | [kubernetes_environment_variables] 956 | 957 | # The scheduler sets the following environment variables into your workers. You may define as 958 | # many environment variables as needed and the kubernetes launcher will set them in the launched workers. 959 | # Environment variables in this section are defined as follows 960 | # `` = `` 961 | # 962 | # For example if you wanted to set an environment variable with value `prod` and key 963 | # ``ENVIRONMENT`` you would follow the following format: 964 | # ENVIRONMENT = prod 965 | # 966 | # Additionally you may override worker airflow settings with the ``AIRFLOW__
__`` 967 | # formatting as supported by airflow normally. 968 | 969 | [kubernetes_secrets] 970 | 971 | # The scheduler mounts the following secrets into your workers as they are launched by the 972 | # scheduler. You may define as many secrets as needed and the kubernetes launcher will parse the 973 | # defined secrets and mount them as secret environment variables in the launched workers. 974 | # Secrets in this section are defined as follows 975 | # `` = =`` 976 | # 977 | # For example if you wanted to mount a kubernetes secret key named ``postgres_password`` from the 978 | # kubernetes secret object ``airflow-secret`` as the environment variable ``POSTGRES_PASSWORD`` into 979 | # your workers you would follow the following format: 980 | # ``POSTGRES_PASSWORD = airflow-secret=postgres_credentials`` 981 | # 982 | # Additionally you may override worker airflow settings with the ``AIRFLOW__
__`` 983 | # formatting as supported by airflow normally. 984 | 985 | [kubernetes_labels] 986 | 987 | # The Key-value pairs to be given to worker pods. 988 | # The worker pods will be given these static labels, as well as some additional dynamic labels 989 | # to identify the task. 990 | # Should be supplied in the format: ``key = value`` 991 | -------------------------------------------------------------------------------- /docker-airflow/docker-compose-CeleryExecutor.yml: -------------------------------------------------------------------------------- 1 | version: '2.1' 2 | services: 3 | redis: 4 | image: 'redis:5.0.5' 5 | # command: redis-server --requirepass redispass 6 | 7 | postgres: 8 | image: postgres:9.6 9 | environment: 10 | - POSTGRES_USER=airflow 11 | - POSTGRES_PASSWORD=airflow 12 | - POSTGRES_DB=airflow 13 | # Uncomment these lines to persist data on the local filesystem. 14 | # - PGDATA=/var/lib/postgresql/data/pgdata 15 | # volumes: 16 | # - ./pgdata:/var/lib/postgresql/data/pgdata 17 | 18 | webserver: 19 | image: puckel/docker-airflow:1.10.9 20 | restart: always 21 | depends_on: 22 | - postgres 23 | - redis 24 | environment: 25 | - LOAD_EX=n 26 | - FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= 27 | - EXECUTOR=Celery 28 | # - POSTGRES_USER=airflow 29 | # - POSTGRES_PASSWORD=airflow 30 | # - POSTGRES_DB=airflow 31 | # - REDIS_PASSWORD=redispass 32 | volumes: 33 | - ./dags:/usr/local/airflow/dags 34 | # Uncomment to include custom plugins 35 | # - ./plugins:/usr/local/airflow/plugins 36 | ports: 37 | - "8080:8080" 38 | command: webserver 39 | healthcheck: 40 | test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-webserver.pid ]"] 41 | interval: 30s 42 | timeout: 30s 43 | retries: 3 44 | 45 | flower: 46 | image: puckel/docker-airflow:1.10.9 47 | restart: always 48 | depends_on: 49 | - redis 50 | environment: 51 | - EXECUTOR=Celery 52 | # - REDIS_PASSWORD=redispass 53 | ports: 54 | - "5555:5555" 55 | command: flower 56 | 57 | scheduler: 58 | image: puckel/docker-airflow:1.10.9 59 | restart: always 60 | depends_on: 61 | - webserver 62 | volumes: 63 | - ./dags:/usr/local/airflow/dags 64 | # Uncomment to include custom plugins 65 | # - ./plugins:/usr/local/airflow/plugins 66 | environment: 67 | - LOAD_EX=n 68 | - FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= 69 | - EXECUTOR=Celery 70 | # - POSTGRES_USER=airflow 71 | # - POSTGRES_PASSWORD=airflow 72 | # - POSTGRES_DB=airflow 73 | # - REDIS_PASSWORD=redispass 74 | command: scheduler 75 | 76 | worker: 77 | image: puckel/docker-airflow:1.10.9 78 | restart: always 79 | depends_on: 80 | - scheduler 81 | volumes: 82 | - ./dags:/usr/local/airflow/dags 83 | # Uncomment to include custom plugins 84 | # - ./plugins:/usr/local/airflow/plugins 85 | environment: 86 | - FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= 87 | - EXECUTOR=Celery 88 | # - POSTGRES_USER=airflow 89 | # - POSTGRES_PASSWORD=airflow 90 | # - POSTGRES_DB=airflow 91 | # - REDIS_PASSWORD=redispass 92 | command: worker 93 | -------------------------------------------------------------------------------- /docker-airflow/docker-compose-LocalExecutor.yml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | services: 3 | postgres: 4 | image: postgres:9.6 5 | environment: 6 | - POSTGRES_USER=airflow 7 | - POSTGRES_PASSWORD=airflow 8 | - POSTGRES_DB=airflow 9 | logging: 10 | options: 11 | max-size: 10m 12 | max-file: "3" 13 | 14 | webserver: 15 | image: puckel/docker-airflow:1.10.9 16 | restart: always 17 | depends_on: 18 | - postgres 19 | environment: 20 | - LOAD_EX=n 21 | - EXECUTOR=Local 22 | logging: 23 | options: 24 | max-size: 10m 25 | max-file: "3" 26 | volumes: 27 | - ./dags:/usr/local/airflow/dags 28 | # - ./plugins:/usr/local/airflow/plugins 29 | ports: 30 | - "8080:8080" 31 | command: webserver 32 | healthcheck: 33 | test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-webserver.pid ]"] 34 | interval: 30s 35 | timeout: 30s 36 | retries: 3 37 | -------------------------------------------------------------------------------- /docker-airflow/script/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # User-provided configuration must always be respected. 4 | # 5 | # Therefore, this script must only derives Airflow AIRFLOW__ variables from other variables 6 | # when the user did not provide their own configuration. 7 | 8 | TRY_LOOP="20" 9 | 10 | # Global defaults and back-compat 11 | : "${AIRFLOW_HOME:="/usr/local/airflow"}" 12 | : "${AIRFLOW__CORE__FERNET_KEY:=${FERNET_KEY:=$(python -c "from cryptography.fernet import Fernet; FERNET_KEY = Fernet.generate_key().decode(); print(FERNET_KEY)")}}" 13 | : "${AIRFLOW__CORE__EXECUTOR:=${EXECUTOR:-Sequential}Executor}" 14 | 15 | # Load DAGs examples (default: Yes) 16 | if [[ -z "$AIRFLOW__CORE__LOAD_EXAMPLES" && "${LOAD_EX:=n}" == n ]]; then 17 | AIRFLOW__CORE__LOAD_EXAMPLES=False 18 | fi 19 | 20 | export \ 21 | AIRFLOW_HOME \ 22 | AIRFLOW__CORE__EXECUTOR \ 23 | AIRFLOW__CORE__FERNET_KEY \ 24 | AIRFLOW__CORE__LOAD_EXAMPLES \ 25 | 26 | # Install custom python package if requirements.txt is present 27 | if [ -e "/requirements.txt" ]; then 28 | $(command -v pip) install --user -r /requirements.txt 29 | fi 30 | 31 | wait_for_port() { 32 | local name="$1" host="$2" port="$3" 33 | local j=0 34 | while ! nc -z "$host" "$port" >/dev/null 2>&1 < /dev/null; do 35 | j=$((j+1)) 36 | if [ $j -ge $TRY_LOOP ]; then 37 | echo >&2 "$(date) - $host:$port still not reachable, giving up" 38 | exit 1 39 | fi 40 | echo "$(date) - waiting for $name... $j/$TRY_LOOP" 41 | sleep 5 42 | done 43 | } 44 | 45 | # Other executors than SequentialExecutor drive the need for an SQL database, here PostgreSQL is used 46 | if [ "$AIRFLOW__CORE__EXECUTOR" != "SequentialExecutor" ]; then 47 | # Check if the user has provided explicit Airflow configuration concerning the database 48 | if [ -z "$AIRFLOW__CORE__SQL_ALCHEMY_CONN" ]; then 49 | # Default values corresponding to the default compose files 50 | : "${POSTGRES_HOST:="postgres"}" 51 | : "${POSTGRES_PORT:="5432"}" 52 | : "${POSTGRES_USER:="airflow"}" 53 | : "${POSTGRES_PASSWORD:="airflow"}" 54 | : "${POSTGRES_DB:="airflow"}" 55 | : "${POSTGRES_EXTRAS:-""}" 56 | 57 | AIRFLOW__CORE__SQL_ALCHEMY_CONN="postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}${POSTGRES_EXTRAS}" 58 | export AIRFLOW__CORE__SQL_ALCHEMY_CONN 59 | 60 | # Check if the user has provided explicit Airflow configuration for the broker's connection to the database 61 | if [ "$AIRFLOW__CORE__EXECUTOR" = "CeleryExecutor" ]; then 62 | AIRFLOW__CELERY__RESULT_BACKEND="db+postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}${POSTGRES_EXTRAS}" 63 | export AIRFLOW__CELERY__RESULT_BACKEND 64 | fi 65 | else 66 | if [[ "$AIRFLOW__CORE__EXECUTOR" == "CeleryExecutor" && -z "$AIRFLOW__CELERY__RESULT_BACKEND" ]]; then 67 | >&2 printf '%s\n' "FATAL: if you set AIRFLOW__CORE__SQL_ALCHEMY_CONN manually with CeleryExecutor you must also set AIRFLOW__CELERY__RESULT_BACKEND" 68 | exit 1 69 | fi 70 | 71 | # Derive useful variables from the AIRFLOW__ variables provided explicitly by the user 72 | POSTGRES_ENDPOINT=$(echo -n "$AIRFLOW__CORE__SQL_ALCHEMY_CONN" | cut -d '/' -f3 | sed -e 's,.*@,,') 73 | POSTGRES_HOST=$(echo -n "$POSTGRES_ENDPOINT" | cut -d ':' -f1) 74 | POSTGRES_PORT=$(echo -n "$POSTGRES_ENDPOINT" | cut -d ':' -f2) 75 | fi 76 | 77 | wait_for_port "Postgres" "$POSTGRES_HOST" "$POSTGRES_PORT" 78 | fi 79 | 80 | # CeleryExecutor drives the need for a Celery broker, here Redis is used 81 | if [ "$AIRFLOW__CORE__EXECUTOR" = "CeleryExecutor" ]; then 82 | # Check if the user has provided explicit Airflow configuration concerning the broker 83 | if [ -z "$AIRFLOW__CELERY__BROKER_URL" ]; then 84 | # Default values corresponding to the default compose files 85 | : "${REDIS_PROTO:="redis://"}" 86 | : "${REDIS_HOST:="redis"}" 87 | : "${REDIS_PORT:="6379"}" 88 | : "${REDIS_PASSWORD:=""}" 89 | : "${REDIS_DBNUM:="1"}" 90 | 91 | # When Redis is secured by basic auth, it does not handle the username part of basic auth, only a token 92 | if [ -n "$REDIS_PASSWORD" ]; then 93 | REDIS_PREFIX=":${REDIS_PASSWORD}@" 94 | else 95 | REDIS_PREFIX= 96 | fi 97 | 98 | AIRFLOW__CELERY__BROKER_URL="${REDIS_PROTO}${REDIS_PREFIX}${REDIS_HOST}:${REDIS_PORT}/${REDIS_DBNUM}" 99 | export AIRFLOW__CELERY__BROKER_URL 100 | else 101 | # Derive useful variables from the AIRFLOW__ variables provided explicitly by the user 102 | REDIS_ENDPOINT=$(echo -n "$AIRFLOW__CELERY__BROKER_URL" | cut -d '/' -f3 | sed -e 's,.*@,,') 103 | REDIS_HOST=$(echo -n "$POSTGRES_ENDPOINT" | cut -d ':' -f1) 104 | REDIS_PORT=$(echo -n "$POSTGRES_ENDPOINT" | cut -d ':' -f2) 105 | fi 106 | 107 | wait_for_port "Redis" "$REDIS_HOST" "$REDIS_PORT" 108 | fi 109 | 110 | case "$1" in 111 | webserver) 112 | airflow initdb 113 | if [ "$AIRFLOW__CORE__EXECUTOR" = "LocalExecutor" ] || [ "$AIRFLOW__CORE__EXECUTOR" = "SequentialExecutor" ]; then 114 | # With the "Local" and "Sequential" executors it should all run in one container. 115 | airflow scheduler & 116 | fi 117 | exec airflow webserver 118 | ;; 119 | worker|scheduler) 120 | # Give the webserver time to run initdb. 121 | sleep 10 122 | exec airflow "$@" 123 | ;; 124 | flower) 125 | sleep 10 126 | exec airflow "$@" 127 | ;; 128 | version) 129 | exec airflow "$@" 130 | ;; 131 | *) 132 | # The command is something like bash, not an airflow subcommand. Just run it in the right environment. 133 | exec "$@" 134 | ;; 135 | esac 136 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | postgres: 4 | image: postgres:9.6 5 | container_name : postgres 6 | environment: 7 | - POSTGRES_USER=airflow 8 | - POSTGRES_PASSWORD=airflow 9 | - POSTGRES_DB=airflow 10 | ports: 11 | - "5432:5432" 12 | 13 | 14 | 15 | webserver: 16 | image: puckel/docker-airflow:1.10.1 17 | build: 18 | context: /Users/rahulpal/Github/Airflow/Airflow_ML/docker-airflow 19 | dockerfile: Dockerfile 20 | args: 21 | PYTHON_DEPS: sqlalchemy==1.2.0 22 | restart: always 23 | depends_on: 24 | - postgres 25 | environment: 26 | - LOAD_EX=n 27 | - EXECUTOR=Local 28 | - FERNET_KEY=jsDPRErfv8Z_eVTnGfF8ywd19j4pyqE3NpdUBA_oRTo= 29 | volumes: 30 | - /Users/rahulpal/Github/Airflow/Airflow_ML/ML_model/dags:/usr/local/airflow/dags 31 | - /Users/rahulpal/Github/Airflow/Airflow_ML/ML_model/models:/usr/local/airflow/models 32 | - /Users/rahulpal/Github/Airflow/Airflow_ML/ML_model/config:/usr/local/airflow/config 33 | # Uncomment to include custom plugins 34 | # - ./plugins:/usr/local/airflow/plugins 35 | ports: 36 | - "9090:8080" 37 | command: webserver 38 | healthcheck: 39 | test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-webserver.pid ]"] 40 | interval: 30s 41 | timeout: 30s 42 | retries: 3 43 | --------------------------------------------------------------------------------