├── .github └── workflows │ ├── cicd.yml │ └── retrain.yml ├── .gitignore ├── Dockerfile ├── README.md ├── app_model.py ├── config.py ├── good.csv ├── poetry.lock ├── pyproject.toml ├── requirements.txt ├── test_app.py └── train.py /.github/workflows/cicd.yml: -------------------------------------------------------------------------------- 1 | name: CI/CD Pipeline 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | workflow_dispatch: 8 | 9 | jobs: 10 | lint: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v2 14 | - name: Setup Python 15 | uses: actions/setup-python@v2 16 | with: 17 | python-version: '3.10' 18 | - name: Setup pip cache 19 | run: echo "${{ github.workspace }}/.cache/pip" > ${{ github.workspace }}/.cache 20 | - name: Display Python and pip versions 21 | run: python --version && pip --version 22 | - name: Install and run ruff 23 | run: | 24 | pip install -q ruff 25 | ruff check 26 | test: 27 | runs-on: ubuntu-latest 28 | steps: 29 | - uses: actions/checkout@v2 30 | - name: Setup Python 31 | uses: actions/setup-python@v2 32 | with: 33 | python-version: '3.10' 34 | - name: Setup pip cache 35 | run: echo "${{ github.workspace }}/.cache/pip" > ${{ github.workspace }}/.cache 36 | - name: Display Python and pip versions 37 | run: python --version && pip --version 38 | - name: Create virtual environment 39 | run: | 40 | python -m venv venv 41 | source venv/bin/activate 42 | - name: Install dependencies and run tests 43 | run: | 44 | pip install -q pytest httpx 45 | pip install -q -r requirements.txt 46 | pytest 47 | env: 48 | REPO_TOKEN: ${{secrets.REPO_TOKEN}} 49 | 50 | build_and_push: 51 | runs-on: ubuntu-latest 52 | steps: 53 | - 54 | name: Set up QEMU 55 | uses: docker/setup-qemu-action@v3 56 | - 57 | name: Set up Docker Buildx 58 | uses: docker/setup-buildx-action@v3 59 | - 60 | name: Login to Docker Hub 61 | uses: docker/login-action@v3 62 | with: 63 | username: ${{ secrets.DOCKERHUB_USERNAME }} 64 | password: ${{ secrets.DOCKERHUB_TOKEN }} 65 | - 66 | name: Build and push 67 | uses: docker/build-push-action@v5 68 | with: 69 | push: true 70 | tags: alaboy19/fastapi-webservice-retrain:01 71 | 72 | 73 | deploy: 74 | runs-on: ubuntu-latest 75 | if: github.ref == 'refs/heads/main' 76 | needs: [lint, test, build_and_push] 77 | steps: 78 | - name: Deploy to Render 79 | run: curl ${{ secrets.RENDER_DEPLOY_HOOK }} 80 | -------------------------------------------------------------------------------- /.github/workflows/retrain.yml: -------------------------------------------------------------------------------- 1 | name: Retrain Pipeline 2 | on: 3 | # Allow manual triggering 4 | # schedule: 5 | # - cron: '*/15 * * * *' 6 | workflow_dispatch: 7 | repository_dispatch: 8 | types: [retrain_pipeline] 9 | jobs: 10 | train: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Checkout code 14 | uses: actions/checkout@v3 15 | 16 | - name: Set up Python 17 | uses: actions/setup-python@v4 18 | with: 19 | python-version: '3.10' 20 | cache: 'pip' 21 | cache-dependency-path: requirements.txt 22 | 23 | - name: Install dependencies 24 | run: pip install -q -r requirements.txt 25 | 26 | - name: Download data 27 | run: curl -o data.csv "${{ secrets.DATA_URL }}" 28 | env: 29 | DATA_URL: ${{ secrets.DATA_URL }} # Use GitHub Secrets 30 | 31 | - name: Train model 32 | run: python train.py data.csv 33 | 34 | - name: Trigger model reload 35 | run: curl -f -X POST ${{ secrets.HOT_RELOAD_URL }} 36 | env: 37 | HOT_RELOAD_URL: ${{ secrets.HOT_RELOAD_URL }} 38 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/latest/usage/project/#working-with-version-control 110 | .pdm.toml 111 | .pdm-python 112 | .pdm-build/ 113 | 114 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 115 | __pypackages__/ 116 | 117 | # Celery stuff 118 | celerybeat-schedule 119 | celerybeat.pid 120 | 121 | # SageMath parsed files 122 | *.sage.py 123 | 124 | # Environments 125 | .env 126 | .venv 127 | env/ 128 | venv/ 129 | ENV/ 130 | env.bak/ 131 | venv.bak/ 132 | 133 | # Spyder project settings 134 | .spyderproject 135 | .spyproject 136 | 137 | # Rope project settings 138 | .ropeproject 139 | 140 | # mkdocs documentation 141 | /site 142 | 143 | # mypy 144 | .mypy_cache/ 145 | .dmypy.json 146 | dmypy.json 147 | 148 | # Pyre type checker 149 | .pyre/ 150 | 151 | # pytype static type analyzer 152 | .pytype/ 153 | 154 | # Cython debug symbols 155 | cython_debug/ 156 | 157 | # PyCharm 158 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 159 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 160 | # and can be added to the global gitignore or merged into this file. For a more nuclear 161 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 162 | #.idea/ 163 | 164 | 165 | test_github_api.py -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM --platform=linux/amd64 python:3.10-buster 2 | 3 | WORKDIR /app 4 | 5 | COPY requirements.txt . 6 | RUN pip install -r requirements.txt 7 | 8 | COPY *.py . 9 | 10 | CMD ["python", "app_model.py"] -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # model retraining pipeline with gitops and fastapi 2 | This is a one scenario of ML model retraining pipeline performed with GitOps tools, GitHub Actions. Usually, model retraining is needed either by some trigger conditions such as data drift or some regular retraining pipeline every week(or so) for concept drifts. Both options are considered in this pipeline. Generally, this tiny self hosted emulation of system design for MLOps that is based on the best-practice recommendations from Google MLops https://cloud.google.com/architecture/mlops-continuous-delivery-and-automation-pipelines-in-machine-learning 3 | 4 | ## Flowchart of the system ## 5 | ![image](https://github.com/Alaboy19/model-retrain-gitops-fastapi/assets/47283347/fbc5aae8-3b17-41d4-bf90-74007c32dc69) 6 | 7 | ## Some key points considered ## 8 | 9 | ### MLflow ### 10 | - provides experiment logging 11 | - general access to model artifacts for data scientists 12 | - model reproducability and versioning 13 | - assesing and comparing models based on metrics 14 | - assigning aliases for models that ready for different environment such as dev, prod 15 | 16 | ### FastAPI ### 17 | - lightweight, simple and fast protocol that works async with ASGI server Uvicorn 18 | - compatable with type hints in pyhthon 19 | - integrated with Pydantic for convinent data types validation 20 | - integrated with OpenAPI, automaticlaly generating the API docs and Swagger interface under the box in route /docs 21 | 22 | ### CI-CD ### 23 | - Since it is online serving, there is need for fast packaging and deployment of the service to prod, therefore CI-CD is better option than orchestrators such as Ariflow, Prefect 24 | - Any pushes or pull request must be tested before shipping to prodution, CI-CD is better option there as well 25 | - Orchestrators could be used further for preparing the data for feature store as a abstraction from data engineerin 26 | 27 | ### render ### 28 | - Free way of virtual machines that lets to deploy service from image for docker registry 29 | 30 | ## Steps taken to develop the pipeline ## 31 | 1. Reproduce the ml deployment on render with fastapi serving here https://github.com/Alaboy19/model-serving-github-actions-render, since it is one of the fundamental blocks of this pipeline. 32 | 2. Host the mlflow registry somewhere, in this case it is hosted on GCP following the [tutorial](https://medium.com/@andrevargas22/how-to-launch-an-mlflow-server-with-continuous-deployment-on-gcp-in-minutes-7d3a29feff88). 33 | 3. The /trigger route was added to webservice that will trigger the gitub actions workflow externally, with github API. 34 | 4. The /reload-model that gets the last model that assigned with alias of @prod on mlflow 35 | 5. train.py scripts that gets the new_data from static source and checks for data drift, if there is any, it launches the training and pushes the new model with alias to @prod to mlflow registry 36 | 6. retrain.ci-cd.yaml that executes all the steps for retraining 37 | ## Steps to reproduce the code ## 38 | 1. Either activate a venv and install dependencies with ``` pip install -q -r requriements.txt 39 | ``` OR you can install poetry and run ```poetry install``` 40 | 2. Generate token for access to your dockerhub account 41 | 3. On github actions serctets → add repo secrets for DATA_URL, HOT_RELOAD_URL(route to /reload-model) 42 | 4. Also, generate REPO_TOKEN as a acess to your repo and add it to your repo variables in action, it is needed to authentificate to your repo when requesting the trigger of retrain.yml externally from fasdtapi service on render 43 | 5. Also, add MLFLOW_TRACKING_URI that got from GCP to repo variables 44 | 6. Follow along the .github.workflows.ci-cd.yml and retrain.yml files 45 | 7. If scheduled retraining and redeploy is needed, uncomment the cron shedule in .github/workflows/retrain.yml 46 | 47 | ## MLOps mature best-practices as reference ## 48 | ![image](https://github.com/Alaboy19/model-retraining-gitops-fastapi/assets/47283347/64412c18-9fd3-47d0-b724-07b9f5d889be) 49 | 50 | -------------------------------------------------------------------------------- /app_model.py: -------------------------------------------------------------------------------- 1 | import os 2 | from typing import Annotated, List 3 | import requests 4 | import mlflow 5 | import uvicorn 6 | from fastapi import Body, Depends, FastAPI 7 | from pydantic import BaseModel, Field 8 | 9 | from config import EVENT_TYPE, OWNER_NAME, PROD_ALIAS, REGISTERED_MODEL_NAME, REPOS_NAME 10 | 11 | app = FastAPI() 12 | 13 | for e in ( 14 | "REPO_TOKEN", 15 | ): 16 | if e not in os.environ: 17 | raise ValueError(f"please set {e} env variable") 18 | 19 | 20 | GITHUB_TOKEN = os.environ["REPO_TOKEN"] 21 | 22 | 23 | class PredictRequest(BaseModel): 24 | passwords: List[str] = Field(alias="Password") 25 | 26 | 27 | class PredictResponse(BaseModel): 28 | prediction: List[float] = Field(alias="Times") 29 | 30 | 31 | _model = None 32 | 33 | 34 | def get_model(): 35 | global _model 36 | if _model is None: 37 | _reload_model() 38 | return _model 39 | 40 | 41 | @app.post("/predict") 42 | def predict( 43 | data: Annotated[PredictRequest, Body()], model=Depends(get_model) 44 | ) -> PredictResponse: 45 | prediction = model.predict(data.passwords) 46 | return PredictResponse(Times=prediction) 47 | 48 | 49 | def _reload_model(): 50 | global _model 51 | _model = mlflow.sklearn.load_model(f"models:/{REGISTERED_MODEL_NAME}@{PROD_ALIAS}") 52 | 53 | 54 | @app.post("/reload-model") 55 | def reload_model(): 56 | _reload_model() 57 | 58 | 59 | class Trigger(BaseModel): 60 | data_url: str 61 | 62 | 63 | @app.post("/trigger") 64 | def trigger_pipeline(): 65 | url = f"https://api.github.com/repos/{OWNER_NAME}/{REPOS_NAME}/dispatches" 66 | headers = { 67 | "Authorization": f"Bearer {GITHUB_TOKEN}", 68 | "Accept": "application/vnd.github.v3+json" 69 | } 70 | data = {"event_type": EVENT_TYPE} 71 | 72 | response = requests.post(url, headers=headers, json=data) 73 | 74 | if response.status_code == 204: 75 | print("Workflow triggered successfully!") 76 | else: 77 | print(f"Error triggering workflow: {response.status_code} - {response.text}") 78 | 79 | 80 | @app.get("/health") 81 | def health(): 82 | return "OK" 83 | 84 | 85 | def main(): 86 | uvicorn.run(app, host="0.0.0.0") 87 | 88 | 89 | if __name__ == "__main__": 90 | main() 91 | -------------------------------------------------------------------------------- /config.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | PROD_ALIAS = "prod" 4 | REGISTERED_MODEL_NAME = "mlops-project-model" 5 | OWNER_NAME = "Alaboy19" 6 | REPOS_NAME = "model-retrain-gitops-fastapi" 7 | EVENT_TYPE = "retrain_pipeline" 8 | 9 | # put the correct service URI from the endpoint provided from step hosting MLflow on GCP, where whatever MLflow regsitry URI you have 10 | os.environ["MLFLOW_TRACKING_URI"] = "https://my-cloud-run-service-1-um3dh5zufa-uc.a.run.app" 11 | 12 | 13 | -------------------------------------------------------------------------------- /good.csv: -------------------------------------------------------------------------------- 1 | Password,Times 2 | nvmmree,1.0 3 | ymbndnp,1.0 4 | pzqydp,1.0 5 | qxma,0.0 6 | pyayj,0.0 7 | pnqagbsmmy,0.0 8 | nqnpc,1.0 9 | nqsn,0.0 10 | gmsqxm,0.0 11 | szhznkix,0.0 12 | ewtnp,1.0 13 | hobkqj,0.0 14 | hssjook,0.0 15 | lyjudtnu,1.0 16 | vtyy,1.0 17 | bcyelbrp,1.0 18 | jfmahkbqp,0.0 19 | rqyrzmqrwm,1.0 20 | xxxtyqkboh,0.0 21 | rilcmyn,1.0 22 | mvreucpqbb,1.0 23 | lvcpteltiy,1.0 24 | asxkyh,0.0 25 | xyjyn,0.0 26 | jqzermwip,1.0 27 | ibazqxg,0.0 28 | hbsbjh,0.0 29 | kkox,0.0 30 | mzyob,0.0 31 | wqvptyz,1.0 32 | wctz,1.0 33 | vyzuipn,1.0 34 | yztqsgoxo,0.0 35 | elwmzrwcz,1.0 36 | axffbh,0.0 37 | kjmmnxomko,0.0 38 | qkymmonnx,0.0 39 | bxpsxnk,0.0 40 | qjvcctcy,1.0 41 | bpbqbevc,1.0 42 | wiup,1.0 43 | wnqjnctc,1.0 44 | yxygb,0.0 45 | wlunwqtjvr,1.0 46 | tkzj,0.0 47 | szsf,0.0 48 | pcer,1.0 49 | zildy,1.0 50 | yqnt,1.0 51 | piabfqmo,0.0 52 | mxqnmt,0.0 53 | akjt,0.0 54 | mpnmdcdm,1.0 55 | jkqgbnyzmz,0.0 56 | jtvuy,1.0 57 | btxq,0.0 58 | uznuv,1.0 59 | ewwebnqrtb,1.0 60 | njsxbbsmj,0.0 61 | rjluy,1.0 62 | vipzqj,1.0 63 | rwumqbucqw,1.0 64 | mvwnzev,1.0 65 | mzzihkzh,0.0 66 | zrqljpn,1.0 67 | khhh,0.0 68 | tgos,0.0 69 | lddrw,1.0 70 | cvewzzipt,1.0 71 | zmjpc,1.0 72 | mzmnmdq,1.0 73 | jetnvrdlp,1.0 74 | gbxn,0.0 75 | lrnuijzti,1.0 76 | mmpmysq,0.0 77 | ssgzknjjs,0.0 78 | hmnymnioqy,0.0 79 | cmzbcjz,1.0 80 | ffsa,0.0 81 | izqpat,0.0 82 | clqecu,1.0 83 | utrrwjlqqr,1.0 84 | ynun,1.0 85 | jlzplu,1.0 86 | ppiwppniwz,1.0 87 | obftapb,0.0 88 | asgb,0.0 89 | wjwruvpv,1.0 90 | jydzpuevvc,1.0 91 | htzzhkbzm,0.0 92 | pzwwwmc,1.0 93 | ecdcyc,1.0 94 | qathzffks,0.0 95 | euyylrjwl,1.0 96 | xxyiyh,0.0 97 | jisyqzkppy,0.0 98 | ghfpitiy,0.0 99 | bfhj,0.0 100 | imkatts,0.0 101 | ymigqzk,0.0 102 | yinjuudcvv,1.0 103 | opzzahxt,0.0 104 | bmbz,1.0 105 | ewvi,1.0 106 | qwvi,1.0 107 | jyurpnl,1.0 108 | eyydtivuev,1.0 109 | nvjrnzt,1.0 110 | nziq,1.0 111 | jxyyqq,0.0 112 | zmmn,0.0 113 | hgppjhomy,0.0 114 | btuc,1.0 115 | zbng,0.0 116 | vczzvubbm,1.0 117 | aais,0.0 118 | pvwnim,1.0 119 | fffnhkfgnk,0.0 120 | yianf,0.0 121 | pjja,0.0 122 | mved,1.0 123 | otjozhosaj,0.0 124 | epve,1.0 125 | rclpb,1.0 126 | sxhz,0.0 127 | njajongim,0.0 128 | mjerullv,1.0 129 | ytfyh,0.0 130 | szhpq,0.0 131 | mdziwil,1.0 132 | hpikbi,0.0 133 | mgsn,0.0 134 | njvtecu,1.0 135 | kjzqt,0.0 136 | njrbiv,1.0 137 | zyzwnwc,1.0 138 | tpqmhfx,0.0 139 | sqkoxhnzy,0.0 140 | hnbyz,0.0 141 | mjstfx,0.0 142 | fqoobxs,0.0 143 | nvewybdn,1.0 144 | tjlvpuc,1.0 145 | btatzg,0.0 146 | vuipvjuci,1.0 147 | tqjp,1.0 148 | kbazjnzt,0.0 149 | eyrwq,1.0 150 | nveecpv,1.0 151 | nmecuju,1.0 152 | ahatpqaxp,0.0 153 | emypzz,1.0 154 | iqpxxt,0.0 155 | myan,0.0 156 | azhjttabtp,0.0 157 | oqnqxz,0.0 158 | mzltpjqdv,1.0 159 | yzpttqq,1.0 160 | mjrruinn,1.0 161 | smyojokab,0.0 162 | cdcwrirwp,1.0 163 | ybhqz,0.0 164 | vpblce,1.0 165 | bskfgjot,0.0 166 | iwzmpc,1.0 167 | xzixp,0.0 168 | nzxhf,0.0 169 | dqbc,1.0 170 | yafkimnx,0.0 171 | pfsbjf,0.0 172 | jbrnirre,1.0 173 | sonjattbi,0.0 174 | qxaqtnyzo,0.0 175 | bucnjvy,1.0 176 | ruzvebzp,1.0 177 | zsmq,0.0 178 | hayghygty,0.0 179 | jizvt,1.0 180 | pdmrwtprpq,1.0 181 | pevt,1.0 182 | zamtfiok,0.0 183 | vnlndtmezi,1.0 184 | dbnbj,1.0 185 | bxqyy,0.0 186 | istgpmb,0.0 187 | mfnbiqbn,0.0 188 | rzzbypjdvb,1.0 189 | crddqrue,1.0 190 | bbia,0.0 191 | ifnfazth,0.0 192 | tqtjak,0.0 193 | qigi,0.0 194 | fppotph,0.0 195 | uetvuvtli,1.0 196 | tonbtx,0.0 197 | qqhyyfiob,0.0 198 | ivqm,1.0 199 | wdedmvbbul,1.0 200 | drurvrpqic,1.0 201 | ijqb,1.0 202 | vcpyldyycw,1.0 203 | skgij,0.0 204 | nbbevyi,1.0 205 | hjamto,0.0 206 | dpvrdll,1.0 207 | zihgm,0.0 208 | hmxmhtf,0.0 209 | okhsh,0.0 210 | iazzhhhnpx,0.0 211 | ospptm,0.0 212 | hhtxjgaao,0.0 213 | cwmpetzenr,1.0 214 | gbkny,0.0 215 | nyxgisbig,0.0 216 | vyidw,1.0 217 | zmjymyrn,1.0 218 | qsasmzxs,0.0 219 | hsysss,0.0 220 | phpxqt,0.0 221 | pogqok,0.0 222 | jcqtzymz,1.0 223 | epcdjenuw,1.0 224 | kxhogmg,0.0 225 | bsyp,0.0 226 | atskjhkiom,0.0 227 | lluqrwqicw,1.0 228 | reewtpeq,1.0 229 | iqhxmnkn,0.0 230 | bzuccp,1.0 231 | zaxkfay,0.0 232 | aznmiyjoo,0.0 233 | sbyokag,0.0 234 | mykkfkmo,0.0 235 | qamy,0.0 236 | lcllqvtwq,1.0 237 | ylmpv,1.0 238 | wwldwbudbi,1.0 239 | jeuvjqtej,1.0 240 | sogtznffq,0.0 241 | yucndjt,1.0 242 | vpdenvqlv,1.0 243 | tcenwemwm,1.0 244 | vcnvwndu,1.0 245 | nnzmyj,1.0 246 | pxaj,0.0 247 | tbagaf,0.0 248 | fnmo,0.0 249 | nmojko,0.0 250 | btqeilbrbd,1.0 251 | bbce,1.0 252 | bgyqnnbiox,0.0 253 | qhtmq,0.0 254 | npxx,0.0 255 | ozoxn,0.0 256 | fmabxgsxq,0.0 257 | snjz,0.0 258 | ucunvllc,1.0 259 | qyiqmutc,1.0 260 | mbtcbyld,1.0 261 | reeiuylu,1.0 262 | npmllp,1.0 263 | tplrpi,1.0 264 | niszgmo,0.0 265 | ojnaha,0.0 266 | yhmxs,0.0 267 | kxmb,0.0 268 | zwzwiruzqr,1.0 269 | mankagtbn,0.0 270 | issqo,0.0 271 | vcpiwr,1.0 272 | wvznibtm,1.0 273 | qszshmybo,0.0 274 | btxyf,0.0 275 | wqlv,1.0 276 | apig,0.0 277 | sjbfso,0.0 278 | jmnbly,1.0 279 | bqbht,0.0 280 | gkymps,0.0 281 | jnfk,0.0 282 | icrrtlc,1.0 283 | tzka,0.0 284 | utytbc,1.0 285 | xzhmx,0.0 286 | zeln,1.0 287 | fnghqo,0.0 288 | nclbmpc,1.0 289 | nivjdjbe,1.0 290 | dwjywzvqyt,1.0 291 | afkat,0.0 292 | qijtp,0.0 293 | oxiff,0.0 294 | zmqueerbyw,1.0 295 | nkkhaabxo,0.0 296 | mwyirvzjmm,1.0 297 | vqzbtyc,1.0 298 | npzmmhx,0.0 299 | wjllrqjl,1.0 300 | ijuqrqduny,1.0 301 | wmcbmn,1.0 302 | ihmag,0.0 303 | baanogoon,0.0 304 | mnivde,1.0 305 | sgzkfxna,0.0 306 | znjyvy,1.0 307 | yzgtskyymb,0.0 308 | iwupmevwb,1.0 309 | iiyh,0.0 310 | lbtqpi,1.0 311 | aqxxoaq,0.0 312 | ievpmcnln,1.0 313 | jwlpi,1.0 314 | wcqwwnywv,1.0 315 | uvjiz,1.0 316 | ucrylnv,1.0 317 | jqlde,1.0 318 | rpteqmlwjp,1.0 319 | qzbllle,1.0 320 | tmwptwqpq,1.0 321 | ebpnq,1.0 322 | pggxoyh,0.0 323 | jecid,1.0 324 | vjwep,1.0 325 | pqizl,1.0 326 | ddimqpejpd,1.0 327 | gomf,0.0 328 | rtlzci,1.0 329 | ohihpyth,0.0 330 | kgtgf,0.0 331 | ofoo,0.0 332 | amshinys,0.0 333 | blmuzrbmd,1.0 334 | leqnup,1.0 335 | wmmv,1.0 336 | wimeq,1.0 337 | tnedjp,1.0 338 | bynqbjb,0.0 339 | rcedjcnpuq,1.0 340 | ymzeunz,1.0 341 | ngansspg,0.0 342 | pzbdpjn,1.0 343 | haoimh,0.0 344 | jmizgq,0.0 345 | jhkfobbfgj,0.0 346 | umtcvwtww,1.0 347 | ixbqi,0.0 348 | virimwebbr,1.0 349 | pphzp,0.0 350 | abnsbpyzax,0.0 351 | gbhg,0.0 352 | yzzbpw,1.0 353 | jvwitweq,1.0 354 | ytvnmentv,1.0 355 | nthiktnjmi,0.0 356 | xfqay,0.0 357 | amxzpmnabq,0.0 358 | gpaxzftgiy,0.0 359 | jjkiiimka,0.0 360 | kjtt,0.0 361 | nedtt,1.0 362 | clvpyqq,1.0 363 | fyzgxg,0.0 364 | gbgiaxomt,0.0 365 | rdvbrir,1.0 366 | unyeytnvev,1.0 367 | yrlb,1.0 368 | hsixpnqhop,0.0 369 | yczymt,1.0 370 | idbbmq,1.0 371 | amjypfqts,0.0 372 | wpvuzy,1.0 373 | ygihytxyz,0.0 374 | fqmgohs,0.0 375 | oopmnqi,0.0 376 | nqgybpsszz,0.0 377 | zmskzmzn,0.0 378 | sshmk,0.0 379 | zifqsjhmm,0.0 380 | iuuq,1.0 381 | xyptiif,0.0 382 | bdbleydp,1.0 383 | dpnzv,1.0 384 | zrzq,1.0 385 | oakszzssz,0.0 386 | lmww,1.0 387 | rcluqqbzee,1.0 388 | mfmitt,0.0 389 | trdrbrze,1.0 390 | gsqzytsshg,0.0 391 | tzmycqlcw,1.0 392 | qzkxyiqb,0.0 393 | jptcpqz,1.0 394 | azfhqh,0.0 395 | ayfbj,0.0 396 | txhsx,0.0 397 | xnbo,0.0 398 | nbzsa,0.0 399 | mtpmqljt,1.0 400 | jzzbndli,1.0 401 | kafk,0.0 402 | yymwz,1.0 403 | smhaao,0.0 404 | obxgsjna,0.0 405 | mhtqkfiyyp,0.0 406 | tqyznhao,0.0 407 | zmbvqd,1.0 408 | ijmxhfntyq,0.0 409 | mdqc,1.0 410 | ytthkhzh,0.0 411 | nzltmc,1.0 412 | asxxymq,0.0 413 | hjfqy,0.0 414 | yatitjbqmx,0.0 415 | mmqtmvvv,1.0 416 | yqjnyyv,1.0 417 | cdzj,1.0 418 | hfhmppq,0.0 419 | jirnr,1.0 420 | tbtjwrbdn,1.0 421 | skjqhy,0.0 422 | hzxq,0.0 423 | zujbvl,1.0 424 | hgotmobhiz,0.0 425 | lynybcprnr,1.0 426 | anqy,0.0 427 | veqwcd,1.0 428 | yssqyxoh,0.0 429 | ygofjpi,0.0 430 | koyh,0.0 431 | vmlyuub,1.0 432 | yymp,1.0 433 | tewctwu,1.0 434 | izaoogihip,0.0 435 | qtrmm,1.0 436 | htjongifh,0.0 437 | ljuc,1.0 438 | cytc,1.0 439 | mznctwrlc,1.0 440 | cdwlqweuln,1.0 441 | gbinakxosz,0.0 442 | drqleubmm,1.0 443 | nnqyqe,1.0 444 | fysbjfab,0.0 445 | ypyg,0.0 446 | jhhitnztp,0.0 447 | axmzbaaix,0.0 448 | foonamfmp,0.0 449 | ymtpuziyu,1.0 450 | qttv,1.0 451 | lqwpm,1.0 452 | pzsat,0.0 453 | qppy,1.0 454 | fbmanoqo,0.0 455 | kqmmpgaiz,0.0 456 | mogzjx,0.0 457 | gtbz,0.0 458 | utvzbr,1.0 459 | ddmmjeq,1.0 460 | skyktqj,0.0 461 | bbduyiwun,1.0 462 | tptsoq,0.0 463 | zvlzded,1.0 464 | hbomozybf,0.0 465 | smjkihokfp,0.0 466 | vwzibby,1.0 467 | ohtyiipyip,0.0 468 | tmqdrutwv,1.0 469 | hipgz,0.0 470 | zjiylbnmt,1.0 471 | ccjqybqpm,1.0 472 | xmbyg,0.0 473 | pbnik,0.0 474 | nbftfooh,0.0 475 | pqqyk,0.0 476 | ygmmzoa,0.0 477 | nsppsztgqy,0.0 478 | hphzqj,0.0 479 | pmmqrnpqd,1.0 480 | vcrzjem,1.0 481 | ztfafghq,0.0 482 | dtnetccce,1.0 483 | bjjs,0.0 484 | gaxbxnb,0.0 485 | nvweqiprm,1.0 486 | cprliqn,1.0 487 | ldwetzj,1.0 488 | ymeejlwd,1.0 489 | vejyc,1.0 490 | umemc,1.0 491 | nuldcy,1.0 492 | ptigyg,0.0 493 | retnj,1.0 494 | ppotjnt,0.0 495 | upqdjpl,1.0 496 | nrdcqvz,1.0 497 | jwyzttzbd,1.0 498 | unucmvbd,1.0 499 | llreuuye,1.0 500 | yrvlu,1.0 501 | vrvycbwzuv,1.0 502 | ysisoyna,0.0 503 | iptihbbiy,0.0 504 | anfzhfpsj,0.0 505 | yitnhh,0.0 506 | xgigahkigp,0.0 507 | hfgmyjyqks,0.0 508 | yuuycyzy,1.0 509 | dyubwiibp,1.0 510 | ktkaspbjxa,0.0 511 | bwrb,1.0 512 | kzmfjyh,0.0 513 | jozpoib,0.0 514 | llcibpzn,1.0 515 | kthkski,0.0 516 | juelujqd,1.0 517 | dedmt,1.0 518 | wvrpeq,1.0 519 | rccrnrunbj,1.0 520 | jzqbimiy,0.0 521 | nqimydztd,1.0 522 | otnoj,0.0 523 | hfgmbnomb,0.0 524 | jqpgpaanzf,0.0 525 | pdwej,1.0 526 | jxtmso,0.0 527 | rrynbtmznm,1.0 528 | msbbni,0.0 529 | ubvbyn,1.0 530 | iyfkzxx,0.0 531 | xkqg,0.0 532 | hiptkskg,0.0 533 | pooa,0.0 534 | bbiqwtlb,1.0 535 | zzkhm,0.0 536 | zfsspp,0.0 537 | rclci,1.0 538 | pozt,0.0 539 | kkzp,0.0 540 | ueiiy,1.0 541 | qbzhzzf,0.0 542 | sagsmqny,0.0 543 | vtziwbqr,1.0 544 | puub,1.0 545 | qobj,0.0 546 | qtihnqsa,0.0 547 | ueejvwlwe,1.0 548 | mlbqirw,1.0 549 | hgxbmbxtyg,0.0 550 | vqryupm,1.0 551 | nhngi,0.0 552 | qmwrjzwne,1.0 553 | zjqzi,1.0 554 | nffxm,0.0 555 | yitbi,1.0 556 | jyxfifxzah,0.0 557 | ndin,1.0 558 | xoxty,0.0 559 | iypeyq,1.0 560 | bzxsmfiqkg,0.0 561 | wrlrq,1.0 562 | inpajbmta,0.0 563 | rnwzpldd,1.0 564 | mrbzce,1.0 565 | kjkyhmaa,0.0 566 | zgnonif,0.0 567 | ifnmhmjmm,0.0 568 | zwpnucjwr,1.0 569 | zhkkpn,0.0 570 | sxahoztpq,0.0 571 | jdiydtq,1.0 572 | bmcjduli,1.0 573 | sagb,0.0 574 | yimmv,1.0 575 | pnhiq,0.0 576 | aoytympzs,0.0 577 | yttv,1.0 578 | mtnqog,0.0 579 | qqtdnpnzt,1.0 580 | sbzb,0.0 581 | zofkmkg,0.0 582 | yfkzfj,0.0 583 | pjpvtuucpb,1.0 584 | tycqczmby,1.0 585 | clyzu,1.0 586 | yimftyg,0.0 587 | omisaapz,0.0 588 | zdep,1.0 589 | uljdl,1.0 590 | inpj,0.0 591 | iyaff,0.0 592 | gptiyj,0.0 593 | qippkys,0.0 594 | yxxoqgpgx,0.0 595 | cvbupcp,1.0 596 | tnvv,1.0 597 | ynhnxamip,0.0 598 | yqosn,0.0 599 | twmll,1.0 600 | dbpcmybzde,1.0 601 | bppbfiqays,0.0 602 | zjvujzurl,1.0 603 | izpbiwmdd,1.0 604 | diuiptctb,1.0 605 | zbsy,0.0 606 | pullyzbb,1.0 607 | lwjnne,1.0 608 | vtmz,1.0 609 | mzyew,1.0 610 | yaiqntfbi,0.0 611 | sgtymo,0.0 612 | tqiprcwt,1.0 613 | bmabssh,0.0 614 | koyjbmqiz,0.0 615 | nmtmghj,0.0 616 | ygxt,0.0 617 | shqpjy,0.0 618 | xhfftkks,0.0 619 | opsmk,0.0 620 | okyfgxjjak,0.0 621 | mzdmv,1.0 622 | xzskq,0.0 623 | iljyyvuece,1.0 624 | qzbkob,0.0 625 | tpjzm,0.0 626 | dbyz,1.0 627 | ucjvtz,1.0 628 | yqqd,1.0 629 | xonb,0.0 630 | zwvz,1.0 631 | opxag,0.0 632 | gjgshpnzkq,0.0 633 | qoymiaa,0.0 634 | hxjx,0.0 635 | yghpfx,0.0 636 | xhptzjbh,0.0 637 | wdnme,1.0 638 | vjppqvytvz,1.0 639 | thiohmhj,0.0 640 | bdbn,1.0 641 | bnpe,1.0 642 | imnt,0.0 643 | pykkbzfoqt,0.0 644 | ygkto,0.0 645 | tkbykyyim,0.0 646 | fopxzs,0.0 647 | xjbm,0.0 648 | ytpy,1.0 649 | dbprzrz,1.0 650 | jzjlrdzn,1.0 651 | ipulren,1.0 652 | yrpmu,1.0 653 | mmyzkijg,0.0 654 | ucvn,1.0 655 | rvimintvue,1.0 656 | bdumdze,1.0 657 | timgxghzxt,0.0 658 | mqnqqcc,1.0 659 | tvnj,1.0 660 | wpqypzdrpb,1.0 661 | emjn,1.0 662 | bbddqzwnre,1.0 663 | jnqe,1.0 664 | zthkypsok,0.0 665 | jsbfniftf,0.0 666 | tljebtuyu,1.0 667 | nndyupqml,1.0 668 | nmspoxjz,0.0 669 | zpkkpgbp,0.0 670 | ijzidp,1.0 671 | yvvwz,1.0 672 | ayxzphath,0.0 673 | pbpxx,0.0 674 | xshjxmmx,0.0 675 | ilzint,1.0 676 | xgfktya,0.0 677 | nyrbzwbmii,1.0 678 | dinwcurbwd,1.0 679 | gxbaig,0.0 680 | mdrupn,1.0 681 | hkagy,0.0 682 | lmebv,1.0 683 | bhmbqoss,0.0 684 | ccbuyyqqt,1.0 685 | juleywcy,1.0 686 | hfjgfzhf,0.0 687 | pyfgmtj,0.0 688 | fboqkjjjzi,0.0 689 | ldlq,1.0 690 | kxtn,0.0 691 | iqcunrlj,1.0 692 | sggxik,0.0 693 | zxns,0.0 694 | tyxbqsypx,0.0 695 | bwpzzuum,1.0 696 | ibrzpucqq,1.0 697 | nylieywbty,1.0 698 | dciqdliiyi,1.0 699 | lqlnuei,1.0 700 | ktmkoza,0.0 701 | zmtbz,1.0 702 | dmrnwtc,1.0 703 | rmbpnw,1.0 704 | pjopjmbmyg,0.0 705 | pxihqqp,0.0 706 | vzdqdu,1.0 707 | qcdtjdy,1.0 708 | pcwpvjb,1.0 709 | mjclvu,1.0 710 | rqnqdbeyq,1.0 711 | cltncryrjj,1.0 712 | txaobamm,0.0 713 | bvzywzi,1.0 714 | bvizijeu,1.0 715 | erjduqdw,1.0 716 | yfqpbhpan,0.0 717 | zndrnybyj,1.0 718 | sftmbgz,0.0 719 | bhgz,0.0 720 | ixop,0.0 721 | ulivlzty,1.0 722 | nmnh,0.0 723 | idntecwd,1.0 724 | xoxymtmbbi,0.0 725 | kfxkzyyk,0.0 726 | hmnfbak,0.0 727 | ttbtn,1.0 728 | qpmyft,0.0 729 | zbwn,1.0 730 | hhhmhomzfb,0.0 731 | pbniizspq,0.0 732 | zbjpmqjy,0.0 733 | ultmt,1.0 734 | noyf,0.0 735 | hnzkkqk,0.0 736 | gqszhii,0.0 737 | ebnirpitiz,1.0 738 | ttabozxf,0.0 739 | sayyoyiax,0.0 740 | mqwbdiwwr,1.0 741 | shqfx,0.0 742 | gkopoi,0.0 743 | kssyzoh,0.0 744 | ttcvwmdn,1.0 745 | pyjkfyypq,0.0 746 | sibxxjxyp,0.0 747 | vrlzjzt,1.0 748 | maaoo,0.0 749 | urlwylc,1.0 750 | byzvlmu,1.0 751 | qptdnyvrd,1.0 752 | bdvubltuun,1.0 753 | zzpvp,1.0 754 | nryvecqj,1.0 755 | wczri,1.0 756 | vmtezvet,1.0 757 | ewqlnwznt,1.0 758 | oofsjjf,0.0 759 | fqymiyon,0.0 760 | xqysksmhj,0.0 761 | jbjqbuu,1.0 762 | vqcnwb,1.0 763 | vjwt,1.0 764 | mihjkjyayj,0.0 765 | pmqfnm,0.0 766 | mluynrwtp,1.0 767 | fmhxkh,0.0 768 | yqqv,1.0 769 | ooqzmyyoiq,0.0 770 | yppcyl,1.0 771 | pcreqpyumd,1.0 772 | pistjt,0.0 773 | jywe,1.0 774 | fkqg,0.0 775 | xqhmohhn,0.0 776 | qzcm,1.0 777 | obojipmhz,0.0 778 | gqbpkf,0.0 779 | rtryciy,1.0 780 | etjie,1.0 781 | ahhjm,0.0 782 | tqizv,1.0 783 | bntgftbn,0.0 784 | lblrqrptzl,1.0 785 | qhsbgh,0.0 786 | wyruwn,1.0 787 | bnlzjncew,1.0 788 | uirnnmjt,1.0 789 | yueli,1.0 790 | nqizypmn,0.0 791 | iqkxj,0.0 792 | btnlwvz,1.0 793 | gxnyfo,0.0 794 | nqntccwbnn,1.0 795 | zidiwwvbe,1.0 796 | bbpjzd,1.0 797 | wubpyqup,1.0 798 | juvyqzwt,1.0 799 | yzotzfio,0.0 800 | lncz,1.0 801 | jqyeibp,1.0 802 | bxhng,0.0 803 | zxxb,0.0 804 | ijbthktp,0.0 805 | pskobxjq,0.0 806 | ztccj,1.0 807 | litripdzuy,1.0 808 | xxzzqtnak,0.0 809 | tmgoh,0.0 810 | qpghxoz,0.0 811 | xzoyokbm,0.0 812 | zirdi,1.0 813 | mybjimrcyc,1.0 814 | ibxz,0.0 815 | lzvmtyby,1.0 816 | ycizydwniy,1.0 817 | udbcjcelj,1.0 818 | tftb,0.0 819 | wervnwt,1.0 820 | cqwcidc,1.0 821 | pkjqnnm,0.0 822 | ahota,0.0 823 | fmjpp,0.0 824 | fssfngpi,0.0 825 | fykmn,0.0 826 | ionsikk,0.0 827 | vjdqirpc,1.0 828 | jiyam,0.0 829 | bypzbuiqe,1.0 830 | itxoz,0.0 831 | koqppj,0.0 832 | tmein,1.0 833 | bsqsmx,0.0 834 | mdtlcm,1.0 835 | rtrrm,1.0 836 | xkghgggzy,0.0 837 | tgfgfzo,0.0 838 | dtvey,1.0 839 | bhazhi,0.0 840 | oaaqff,0.0 841 | njkb,0.0 842 | cnmrdemzj,1.0 843 | bqsayzptx,0.0 844 | hofg,0.0 845 | kjpfi,0.0 846 | kakib,0.0 847 | wpywzyuwdw,1.0 848 | vpbdlqutvw,1.0 849 | dyuuuvl,1.0 850 | yfigoip,0.0 851 | zkpb,0.0 852 | amkyskhsfm,0.0 853 | njxgzfns,0.0 854 | zsokq,0.0 855 | psmazxym,0.0 856 | ngbjoqk,0.0 857 | bzgkgsxzi,0.0 858 | qctez,1.0 859 | bsgq,0.0 860 | bapbi,0.0 861 | qcyiidmlvp,1.0 862 | qyue,1.0 863 | vbdue,1.0 864 | znfzh,0.0 865 | phaobpn,0.0 866 | uejin,1.0 867 | bgxjybj,0.0 868 | qjxopjzktk,0.0 869 | yuprdr,1.0 870 | tfkinbxmt,0.0 871 | zjvpuddlwc,1.0 872 | atiobmtx,0.0 873 | bfzttzykon,0.0 874 | ganyfkba,0.0 875 | xghikgtkf,0.0 876 | juwe,1.0 877 | wviwtpicc,1.0 878 | zdniw,1.0 879 | wtucid,1.0 880 | kyfzanynks,0.0 881 | pmtpejmw,1.0 882 | cuqn,1.0 883 | jnvrnyirml,1.0 884 | qylbjmyu,1.0 885 | qggymx,0.0 886 | yuzdbedu,1.0 887 | rcrqrdu,1.0 888 | muuy,1.0 889 | ykstsomihi,0.0 890 | hykhkimnt,0.0 891 | qutpwepw,1.0 892 | pmxqphbo,0.0 893 | ojxb,0.0 894 | atatnmisan,0.0 895 | jnlqqiidm,1.0 896 | kjixk,0.0 897 | jwpzeze,1.0 898 | ixxgjp,0.0 899 | lnrdplbpqm,1.0 900 | pbnxsmxs,0.0 901 | cernzeumvd,1.0 902 | qxmk,0.0 903 | jfqjyg,0.0 904 | jijyup,1.0 905 | iljq,1.0 906 | zrpc,1.0 907 | onqztgzq,0.0 908 | wnic,1.0 909 | ruuyzb,1.0 910 | yueeezd,1.0 911 | skaqgaftzz,0.0 912 | ifpkpzjxo,0.0 913 | jqaxksf,0.0 914 | hqtjy,0.0 915 | jkammsgk,0.0 916 | jqmb,0.0 917 | sqgmb,0.0 918 | pngixsbf,0.0 919 | mbtafjyajq,0.0 920 | wutd,1.0 921 | ldrimbnd,1.0 922 | xzkpygokz,0.0 923 | ulmbbueww,1.0 924 | mpwuzzzbez,1.0 925 | tpjai,0.0 926 | xfff,0.0 927 | tijej,1.0 928 | zuviymeuvy,1.0 929 | erjni,1.0 930 | zbizsks,0.0 931 | ntep,1.0 932 | eljip,1.0 933 | gghfzyt,0.0 934 | czduri,1.0 935 | hbqhpo,0.0 936 | bpqdtmuvv,1.0 937 | wnijnemqbv,1.0 938 | bxkinnyp,0.0 939 | nwizp,1.0 940 | jbeiwvutvn,1.0 941 | eplrz,1.0 942 | gjsmpiy,0.0 943 | cwpz,1.0 944 | dpqmpbiv,1.0 945 | vvzqdrmr,1.0 946 | vlem,1.0 947 | mublbiqne,1.0 948 | zymy,1.0 949 | qtdrtpqb,1.0 950 | oybpgxma,0.0 951 | tywyi,1.0 952 | ciqncyee,1.0 953 | njpbmv,1.0 954 | zpqjf,0.0 955 | tsxmfzki,0.0 956 | yputd,1.0 957 | vwqtbnt,1.0 958 | itjzkqnhmz,0.0 959 | inbbune,1.0 960 | lzzy,1.0 961 | lbzcptqcdd,1.0 962 | nbnthh,0.0 963 | retr,1.0 964 | ndiiuql,1.0 965 | qtwvy,1.0 966 | unzypmlib,1.0 967 | bpebjzzcz,1.0 968 | byimqtj,1.0 969 | yrzqbqdyv,1.0 970 | ifztfkb,0.0 971 | iygzyof,0.0 972 | qqbup,1.0 973 | iznikiap,0.0 974 | igsyyobx,0.0 975 | bnttrbelvu,1.0 976 | xnpxyksn,0.0 977 | hmjqxt,0.0 978 | yfbxpgiyk,0.0 979 | zthig,0.0 980 | pdjwywcutb,1.0 981 | tptknshfx,0.0 982 | mmjtuzdqq,1.0 983 | dezrzqci,1.0 984 | liijndj,1.0 985 | djmmulei,1.0 986 | ncyptq,1.0 987 | aizyi,0.0 988 | jxoh,0.0 989 | ynzhfxbyq,0.0 990 | unntctwicd,1.0 991 | uzwl,1.0 992 | dwqijw,1.0 993 | jwcjnr,1.0 994 | ljjr,1.0 995 | zpjkp,0.0 996 | jqyfnao,0.0 997 | rddplnbmp,1.0 998 | zjezp,1.0 999 | vuitpdqn,1.0 1000 | ttzij,1.0 1001 | uilcy,1.0 1002 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "model-retrain-gitops-fastapi" 3 | version = "0.1.0" 4 | description = "" 5 | authors = ["Nurbolat "] 6 | readme = "README.md" 7 | 8 | [tool.poetry.dependencies] 9 | python = "^3.10" 10 | fastapi = "0.110.1" 11 | uvicorn = "0.29.0" 12 | pandas = "2.2.0" 13 | scikit-learn = "1.4.0" 14 | python-multipart = "0.0.6" 15 | mlflow = "2.11.1" 16 | setuptools = "69.0.3" 17 | boto3 = "1.34.51" 18 | python-gitlab = "4.4.0" 19 | evidently = "0.4.19" 20 | pytest = "^8.2.2" 21 | pygithub = "^2.3.0" 22 | ruff = "^0.4.9" 23 | 24 | 25 | [build-system] 26 | requires = ["poetry-core"] 27 | build-backend = "poetry.core.masonry.api" 28 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | fastapi==0.110.1 2 | uvicorn==0.29.0 3 | pandas==2.2.0 4 | scikit-learn==1.4.0 5 | python-multipart==0.0.6 6 | mlflow==2.11.1 7 | setuptools==69.0.3 8 | boto3==1.34.51 9 | python-gitlab==4.4.0 10 | evidently==0.4.19 11 | pytest -------------------------------------------------------------------------------- /test_app.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from fastapi.testclient import TestClient 3 | 4 | from app_model import app 5 | 6 | 7 | @pytest.fixture 8 | def client(): 9 | return TestClient(app) 10 | 11 | 12 | def test_predict(client: TestClient): 13 | r = client.get("/health") 14 | assert r.status_code == 200 15 | -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import mlflow 4 | import pandas as pd 5 | from evidently.test_preset import DataDriftTestPreset 6 | from evidently.test_suite import TestSuite 7 | from mlflow import MlflowClient 8 | from sklearn.ensemble import RandomForestRegressor 9 | from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer 10 | from sklearn.linear_model import LinearRegression, Ridge 11 | from sklearn.pipeline import Pipeline 12 | 13 | from config import PROD_ALIAS, REGISTERED_MODEL_NAME 14 | 15 | EXPERIMENT_NAME = "mlops-project" 16 | GOOD_DATA_URL = ( 17 | "https://s3.lab.karpov.courses/mlops-training-sets/project/example/good.csv" 18 | ) 19 | # for e in ( 20 | # "MLFLOW_TRACKING_USERNAME", 21 | # "MLFLOW_TRACKING_PASSWORD", 22 | # "AWS_ACCESS_KEY_ID", 23 | # "AWS_SECRET_ACCESS_KEY", 24 | # ): 25 | # if e not in os.environ: 26 | # raise ValueError(f"please set {e} env variable") 27 | 28 | os.environ["MLFLOW_TRACKING_URI"] = "https://my-cloud-run-service-1-um3dh5zufa-uc.a.run.app" 29 | os.environ["AWS_ENDPOINT_URL"] = "https://storage.yandexcloud.net" 30 | 31 | 32 | def get_data(): 33 | if len(sys.argv) < 2: 34 | raise ValueError("provide path to data") 35 | path = sys.argv[1] 36 | return pd.read_csv(path) 37 | 38 | 39 | def get_model(params: dict | None): 40 | params = params or {"model": "rf", "ngrams": {"min": 1, "max": 3}} 41 | if params["model"] == "linear": 42 | model = LinearRegression() 43 | elif params["model"] == "rf": 44 | model = RandomForestRegressor() 45 | elif params["model"] == "ridge": 46 | model = Ridge() 47 | else: 48 | raise ValueError() 49 | 50 | # Create the pipeline 51 | pipeline = Pipeline( 52 | [ 53 | ( 54 | "vect", 55 | CountVectorizer( 56 | ngram_range=(params["ngrams"]["min"], params["ngrams"]["max"]), 57 | analyzer="char", 58 | ), 59 | ), 60 | ("tfidf", TfidfTransformer()), 61 | ("clf", model), 62 | ] 63 | ) 64 | return pipeline 65 | 66 | 67 | def check_data(df: pd.DataFrame) -> bool: 68 | reference = pd.read_csv(GOOD_DATA_URL) 69 | suite = TestSuite(tests=[DataDriftTestPreset(columns=["Times"])]) 70 | suite.run(reference_data=reference, current_data=df) 71 | return bool(suite) 72 | 73 | 74 | def main(): 75 | data = get_data() 76 | if not check_data(data): 77 | print("Data is bad") 78 | sys.exit(1) 79 | 80 | model = get_model(None) 81 | model.fit(data["Password"], data["Times"]) 82 | 83 | print(model.predict(data["Password"])[0]) 84 | 85 | exp = mlflow.get_experiment_by_name(EXPERIMENT_NAME) 86 | if exp is None: 87 | experiment_id = mlflow.create_experiment(EXPERIMENT_NAME) 88 | else: 89 | experiment_id = exp.experiment_id 90 | with mlflow.start_run(experiment_id=experiment_id): 91 | model = mlflow.sklearn.log_model(model, artifact_path="model") 92 | reg_model = mlflow.register_model( 93 | model_uri=model.model_uri, name=REGISTERED_MODEL_NAME 94 | ) 95 | client = MlflowClient() 96 | client.set_registered_model_alias( 97 | REGISTERED_MODEL_NAME, PROD_ALIAS, reg_model.version 98 | ) 99 | 100 | 101 | if __name__ == "__main__": 102 | main() 103 | --------------------------------------------------------------------------------