├── .github └── workflows │ ├── flake8-lint.yml │ └── validate.yml ├── .gitignore ├── .ignore ├── Dockerfile ├── LICENSE ├── README.md ├── environment.yml ├── logos ├── AnomLLM.png ├── gemini.png ├── gpt.png ├── llama.png └── qwen.png ├── notebook ├── .gitattributes ├── affiliation.ipynb └── example.ipynb ├── poetry.lock ├── pyproject.toml ├── src ├── anom_llm │ └── __init__.py ├── baselines │ └── isoforest.py ├── batch_api.py ├── config.py ├── data │ └── synthetic.py ├── gemini_api.py ├── online_api.py ├── openai_api.py ├── prompt.py ├── result_agg.py └── utils.py ├── synthesize.sh ├── test.sh └── tests └── __init__.py /.github/workflows/flake8-lint.yml: -------------------------------------------------------------------------------- 1 | name: Flake8 Lint 2 | 3 | on: 4 | push: 5 | branches: [ main, dev ] 6 | pull_request: 7 | branches: [ main, dev ] 8 | 9 | jobs: 10 | flake8-lint: 11 | name: Flake8 Lint 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v3 15 | - name: Set up Python 16 | uses: actions/setup-python@v4 17 | with: 18 | python-version: '3.x' 19 | - name: Install dependencies 20 | run: | 21 | python -m pip install --upgrade pip 22 | pip install flake8 23 | - name: Run flake8 24 | run: | 25 | flake8 . --max-line-length=150 --ignore=E402,E731,F541,W291,E122,E127,F401,E266,E241,C901,E741,W293,F811,W503,E203,F403,F405 26 | -------------------------------------------------------------------------------- /.github/workflows/validate.yml: -------------------------------------------------------------------------------- 1 | name: Environment and Script Check 2 | 3 | on: 4 | push: 5 | branches: [ main, dev ] 6 | pull_request: 7 | branches: [ main, dev ] 8 | 9 | jobs: 10 | check-environment-and-run-script: 11 | name: Check Environment and Run Script 12 | runs-on: ubuntu-latest 13 | steps: 14 | - name: Checkout code 15 | uses: actions/checkout@v3 16 | 17 | - name: Set up Conda 18 | uses: conda-incubator/setup-miniconda@v2 19 | with: 20 | auto-activate-base: false 21 | activate-environment: anomllm 22 | environment-file: environment.yml 23 | 24 | - name: Create Conda environment and install dependencies 25 | shell: bash -l {0} 26 | run: | 27 | poetry install --no-root --with dev 28 | 29 | - name: Save credentials 30 | run: echo "${{ secrets.CREDENTIALS }}" > credentials.yml 31 | 32 | - name: Convert Jupyter notebook to Python script 33 | shell: bash -l {0} 34 | run: | 35 | jupyter nbconvert --to script notebook/example.ipynb 36 | 37 | - name: Run converted script 38 | shell: bash -l {0} 39 | env: 40 | PYTHONPATH: ${{ github.workspace }}/src 41 | run: | 42 | wget https://github.com/peak/s5cmd/releases/download/v2.2.2/s5cmd_2.2.2_Linux-64bit.tar.gz 43 | tar -xzvf s5cmd_2.2.2_Linux-64bit.tar.gz 44 | chmod +x s5cmd 45 | ./s5cmd --no-sign-request --endpoint-url https://s3-west.nrp-nautilus.io cp "s3://anomllm/data/*" data/ 46 | python notebook/example.py 47 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm-project.org/#use-with-ide 110 | .pdm.toml 111 | .pdm-python 112 | .pdm-build/ 113 | 114 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 115 | __pypackages__/ 116 | 117 | # Celery stuff 118 | celerybeat-schedule 119 | celerybeat.pid 120 | 121 | # SageMath parsed files 122 | *.sage.py 123 | 124 | # Environments 125 | .env 126 | .venv 127 | env/ 128 | venv/ 129 | ENV/ 130 | env.bak/ 131 | venv.bak/ 132 | 133 | # Spyder project settings 134 | .spyderproject 135 | .spyproject 136 | 137 | # Rope project settings 138 | .ropeproject 139 | 140 | # mkdocs documentation 141 | /site 142 | 143 | # mypy 144 | .mypy_cache/ 145 | .dmypy.json 146 | dmypy.json 147 | 148 | # Pyre type checker 149 | .pyre/ 150 | 151 | # pytype static type analyzer 152 | .pytype/ 153 | 154 | # Cython debug symbols 155 | cython_debug/ 156 | 157 | # PyCharm 158 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 159 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 160 | # and can be added to the global gitignore or merged into this file. For a more nuclear 161 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 162 | #.idea/ 163 | 164 | /data/ 165 | /results/ 166 | /logs/ 167 | /archive/ 168 | /credentials.yml 169 | src/vus.ipynb 170 | /data 171 | /.vscode/ 172 | *.pdf 173 | *.tex -------------------------------------------------------------------------------- /.ignore: -------------------------------------------------------------------------------- 1 | notebook/ -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM gitlab-registry.nrp-nautilus.io/prp/jupyter-stack/minimal 2 | 3 | USER root 4 | 5 | # Install dependency (You may add other dependencies here) 6 | RUN apt update && apt install -y make rsync git vim 7 | RUN wget https://github.com/peak/s5cmd/releases/download/v2.2.2/s5cmd_2.2.2_linux_amd64.deb && dpkg -i s5cmd_2.2.2_linux_amd64.deb && rm s5cmd_2.2.2_linux_amd64.deb 8 | 9 | # Add ssh key 10 | RUN mkdir -p /root/.ssh 11 | ADD .ssh/id_rsa /root/.ssh/id_rsa 12 | ADD .ssh/config /root/.ssh/config 13 | ADD .ssh/known_hosts /root/.ssh/known_hosts 14 | RUN chmod 400 /root/.ssh/id_rsa 15 | RUN chmod 400 /root/.ssh/config 16 | RUN chmod 400 /root/.ssh/known_hosts 17 | 18 | # Pull the latest project 19 | WORKDIR /root/ 20 | RUN git clone --depth=1 PROJECT_SSH_URL 21 | WORKDIR /root/PROJECT_NAME/ 22 | 23 | # Handle git submodule 24 | RUN git submodule update --init --recursive 25 | 26 | # Install conda environment 27 | RUN conda update --all 28 | RUN conda env create -n PROJECT_NAME --file environment.yml 29 | RUN conda clean -qafy 30 | 31 | # Activate the new conda environment and install poetry 32 | SHELL ["/opt/conda/bin/conda", "run", "-n", "PROJECT_NAME", "/bin/bash", "-c"] 33 | RUN wget https://pdm-project.org/install-pdm.py && python install-pdm.py && rm install-pdm.py 34 | RUN pdm install 35 | RUN pip install numpy==1.21.6 36 | RUN pip install cython==0.29.37 37 | RUN cd VUS && python setup.py install 38 | RUN pip install google-generativeai 39 | RUN pip install torch -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Zihao Zhou 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | AnomLLM 3 |

4 |

AnomLLM

5 |

Can LLMs Understand Time Series Anomalies? 6 |

7 | 8 |

9 | license 10 | python 11 | version 12 |

13 | 14 | 15 | ## | Introduction 16 | 17 | We challenge common assumptions about Large Language Models' capabilities in time series understanding. This repository contains the code for reproducing results and **benchmarking** your own large language models' (as long as they are compatible with OpenAI API) anomaly detection capabilities. 18 | 19 | ## | Citation 20 | 21 | [[2410.05440] Can LLMs Understand Time Series Anomalies?](https://arxiv.org/abs/2410.05440) 22 | 23 | ``` 24 | @misc{zhou2024llmsunderstandtimeseries, 25 | title={Can LLMs Understand Time Series Anomalies?}, 26 | author={Zihao Zhou and Rose Yu}, 27 | year={2024}, 28 | eprint={2410.05440}, 29 | archivePrefix={arXiv}, 30 | primaryClass={cs.LG}, 31 | url={https://arxiv.org/abs/2410.05440}, 32 | } 33 | ``` 34 | 35 | ## | Installation 36 | 37 | - Dependencies: `conda` 38 | - Run `export PYTHONPATH=$PYTHONPATH:$(pwd)/src` first 39 | - Jpyter notebook path shall be the root directory of the project. 40 | 41 | ```bash 42 | conda env create --file environment.yml 43 | conda activate anomllm 44 | poetry install --no-root 45 | # Or `poetry install --no-root --with dev` if you need jupyter and etc. 46 | ``` 47 | 48 | ## | Dataset Download 49 | 50 | We recommend using [`s5cmd`](https://github.com/peak/s5cmd/tree/master) to download the dataset from the NRP S3 bucket. 51 | 52 | ```bash 53 | s5cmd --no-sign-request --endpoint-url https://s3-west.nrp-nautilus.io cp "s3://anomllm/data/*" data/ 54 | ``` 55 | 56 | Alternatively, you can download the dataset from the following link: [Google Drive](https://drive.google.com/file/d/19KNCiOm3UI_JXkzBAWOdqXwM0VH3xOwi/view?usp=sharing) or synthesize your own dataset using `synthesize.sh`. Make sure the dataset is stored in the `data` directory. 57 | 58 | ## | API Configuration 59 | 60 | Create a `credentials.yml` file in the root directory with the following content: 61 | 62 | ```yaml 63 | gpt-4o: 64 | api_key: 65 | base_url: "https://api.openai.com/v1" 66 | gpt-4o-mini: 67 | api_key: 68 | base_url: "https://api.openai.com/v1" 69 | gemini-1.5-flash: 70 | api_key: 71 | internvlm-76b: 72 | api_key: 73 | base_url: (ended with v1) 74 | qwen: 75 | api_key: 76 | base_url: (ended with v1) 77 | ``` 78 | 79 | ## | Example Usage for Single Time Series 80 | 81 | Check out the [example notebook](https://github.com/Rose-STL-Lab/AnomLLM/blob/dev/notebook/example.ipynb). 82 | 83 | To run the example notebook, you only need the `gemini-1.5-flash` model in the `credentials.yml` file. 84 | 85 | ## | Batch Run using OpenAI BatchAPI 86 | 87 | `python src/batch_api.py --data $datum --model $model --variant $variant` 88 | 89 | See `test.sh` for comprehensive lists of models, variants, and datasets. The [Batch API](https://platform.openai.com/docs/guides/batch/overview) only works with OpenAI proprietary models and will reduce the cost by 50%, but it does not finish in real-time. Your first run will create a request file, and subsequent runs will check the status of the request and retrieve the results when they are ready. 90 | 91 | ## | Online Run using OpenAI API 92 | 93 | `python src/online_api.py --data $datum --model $model --variant $variant` 94 | 95 | The online API works with all OpenAI-compatible model hosting services. 96 | 97 | ## | Evaluation 98 | 99 | `python src/result_agg.py --data $datum` 100 | 101 | The evaluation script will aggregate the results from the API and generate the evaluation metrics, for all models and variants. 102 | 103 | ## | License 104 | 105 | This project is licensed under the MIT License. -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: anomllm 2 | channels: 3 | - conda-forge 4 | - nvidia 5 | dependencies: 6 | - poetry=1.* 7 | - python=3.10.* 8 | - pip -------------------------------------------------------------------------------- /logos/AnomLLM.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Rose-STL-Lab/AnomLLM/bf7827dfd63e5cd134f5b2412efc54175e5aee72/logos/AnomLLM.png -------------------------------------------------------------------------------- /logos/gemini.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Rose-STL-Lab/AnomLLM/bf7827dfd63e5cd134f5b2412efc54175e5aee72/logos/gemini.png -------------------------------------------------------------------------------- /logos/gpt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Rose-STL-Lab/AnomLLM/bf7827dfd63e5cd134f5b2412efc54175e5aee72/logos/gpt.png -------------------------------------------------------------------------------- /logos/llama.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Rose-STL-Lab/AnomLLM/bf7827dfd63e5cd134f5b2412efc54175e5aee72/logos/llama.png -------------------------------------------------------------------------------- /logos/qwen.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Rose-STL-Lab/AnomLLM/bf7827dfd63e5cd134f5b2412efc54175e5aee72/logos/qwen.png -------------------------------------------------------------------------------- /notebook/.gitattributes: -------------------------------------------------------------------------------- 1 | *.ipynb filter=strip-notebook-output -------------------------------------------------------------------------------- /notebook/affiliation.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "%load_ext autoreload\n", 10 | "%autoreload 2\n", 11 | "%config InlineBackend.figure_format = 'retina'" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": null, 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "from prompt import time_series_to_image\n", 21 | "from utils import view_base64_image, display_messages, collect_results\n", 22 | "from data.synthetic import SyntheticDataset\n", 23 | "\n", 24 | "data_name = 'point'\n", 25 | "data_dir = f'data/synthetic/{data_name}/eval/'\n", 26 | "train_dir = f'data/synthetic/{data_name}/train/'\n", 27 | "eval_dataset = SyntheticDataset(data_dir)\n", 28 | "eval_dataset.load()\n", 29 | "train_dataset = SyntheticDataset(train_dir)\n", 30 | "train_dataset.load()" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": null, 36 | "metadata": {}, 37 | "outputs": [], 38 | "source": [ 39 | "from utils import plot_series_and_predictions\n", 40 | "\n", 41 | "idx = 0\n", 42 | "directory = f'results/synthetic/{data_name}'\n", 43 | "series = eval_dataset[idx][1].numpy()\n", 44 | "anomaly_locations = eval_dataset[idx][0].numpy()\n", 45 | "\n", 46 | "raw_results = collect_results(directory, raw=True)\n", 47 | "raw_results = {k: v[idx] for k, v in raw_results.items()}\n", 48 | "results = collect_results(directory)\n", 49 | "results = {k: v[idx] for k, v in results.items()}\n", 50 | "\n", 51 | "fig = plot_series_and_predictions(\n", 52 | " series=series, \n", 53 | " gt_anomaly_intervals=anomaly_locations,\n", 54 | " anomalies=results\n", 55 | ")" 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": null, 61 | "metadata": {}, 62 | "outputs": [], 63 | "source": [ 64 | "from utils import display_messages\n", 65 | "\n", 66 | "display_messages(raw_results['gpt-4o (0shot-vision)']['request']['messages'])" 67 | ] 68 | }, 69 | { 70 | "cell_type": "code", 71 | "execution_count": null, 72 | "metadata": {}, 73 | "outputs": [], 74 | "source": [ 75 | "print(list(raw_results.values())[0]['response'])" 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": null, 81 | "metadata": {}, 82 | "outputs": [], 83 | "source": [ 84 | "from utils import interval_to_vector\n", 85 | "gt = interval_to_vector(anomaly_locations[0])" 86 | ] 87 | }, 88 | { 89 | "cell_type": "code", 90 | "execution_count": null, 91 | "metadata": {}, 92 | "outputs": [], 93 | "source": [ 94 | "import pandas as pd\n", 95 | "from sklearn.metrics import precision_score, recall_score\n", 96 | "from affiliation.generics import convert_vector_to_events\n", 97 | "from affiliation.metrics import pr_from_events\n", 98 | "from utils import compute_metrics\n", 99 | "\n", 100 | "data = {}\n", 101 | "\n", 102 | "for name, prediction in results.items():\n", 103 | " metrics = compute_metrics(gt, prediction)\n", 104 | " data[name] = metrics\n", 105 | "\n", 106 | "df = pd.DataFrame(data).T\n", 107 | "df\n" 108 | ] 109 | }, 110 | { 111 | "cell_type": "code", 112 | "execution_count": null, 113 | "metadata": {}, 114 | "outputs": [], 115 | "source": [ 116 | "# Keep only the first four columns\n", 117 | "df = df.iloc[:, :4].fillna(0)" 118 | ] 119 | }, 120 | { 121 | "cell_type": "code", 122 | "execution_count": null, 123 | "metadata": {}, 124 | "outputs": [], 125 | "source": [ 126 | "{k: v['response'] for k, v in raw_results.items()}" 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": null, 132 | "metadata": {}, 133 | "outputs": [], 134 | "source": [ 135 | "from utils import highlight_by_ranking\n", 136 | "from IPython.core.display import HTML\n", 137 | "styled_df = highlight_by_ranking(df.T).T\n", 138 | "HTML(styled_df.to_html(escape=False))\n" 139 | ] 140 | }, 141 | { 142 | "cell_type": "code", 143 | "execution_count": null, 144 | "metadata": {}, 145 | "outputs": [], 146 | "source": [ 147 | "results = collect_results(directory)" 148 | ] 149 | }, 150 | { 151 | "cell_type": "code", 152 | "execution_count": null, 153 | "metadata": {}, 154 | "outputs": [], 155 | "source": [ 156 | "import numpy as np\n", 157 | "import pandas as pd\n", 158 | "from tqdm import trange\n", 159 | "from utils import compute_metrics, interval_to_vector\n", 160 | "\n", 161 | "# Initialize dictionary to store results\n", 162 | "metric_names = ['precision', 'recall', 'f1', 'affi precision', 'affi recall', 'affi f1']\n", 163 | "\n", 164 | "results_dict = {\n", 165 | " key: [[] for _ in metric_names] for key in results.keys()\n", 166 | "}\n", 167 | "\n", 168 | "# Initialize DataFrame to store results\n", 169 | "df_list = []\n", 170 | "\n", 171 | "for i in trange(0, 400):\n", 172 | " series = eval_dataset[idx][1].numpy()\n", 173 | " anomaly_locations = eval_dataset[idx][0].numpy()\n", 174 | " gt = interval_to_vector(anomaly_locations[0])\n", 175 | "\n", 176 | " for name, prediction in results.items():\n", 177 | " metrics = compute_metrics(gt, prediction[i])\n", 178 | " for idx, metric_name in enumerate(metric_names):\n", 179 | " results_dict[name][idx].append(metrics[metric_name])\n", 180 | "\n", 181 | "# Convert dictionary to DataFrame\n", 182 | "df = pd.DataFrame({k: np.mean(v, axis=1) for k, v in results_dict.items()}, \n", 183 | " index=['precision', 'recall', 'f1', 'affi precision', 'affi recall', 'affi f1'])\n", 184 | "\n", 185 | "df.T" 186 | ] 187 | }, 188 | { 189 | "cell_type": "code", 190 | "execution_count": null, 191 | "metadata": {}, 192 | "outputs": [], 193 | "source": [ 194 | "styled_df = highlight_by_ranking(df).T\n", 195 | "# To visualize the output in Jupyter Notebook\n", 196 | "from IPython.core.display import HTML\n", 197 | "HTML(styled_df.to_html(escape=False))" 198 | ] 199 | }, 200 | { 201 | "cell_type": "code", 202 | "execution_count": null, 203 | "metadata": {}, 204 | "outputs": [], 205 | "source": [ 206 | "from utils import styled_df_to_latex\n", 207 | "\n", 208 | "latex_table = styled_df_to_latex(styled_df, \"Point Anomalies in Sine Wave\", label=\"point_exp\")\n", 209 | "print(latex_table)" 210 | ] 211 | }, 212 | { 213 | "cell_type": "code", 214 | "execution_count": null, 215 | "metadata": {}, 216 | "outputs": [], 217 | "source": [] 218 | } 219 | ], 220 | "metadata": { 221 | "kernelspec": { 222 | "display_name": "Python 3", 223 | "language": "python", 224 | "name": "python3" 225 | }, 226 | "language_info": { 227 | "codemirror_mode": { 228 | "name": "ipython", 229 | "version": 3 230 | }, 231 | "file_extension": ".py", 232 | "mimetype": "text/x-python", 233 | "name": "python", 234 | "nbconvert_exporter": "python", 235 | "pygments_lexer": "ipython3", 236 | "version": "3.10.14" 237 | } 238 | }, 239 | "nbformat": 4, 240 | "nbformat_minor": 2 241 | } 242 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "AnomLLM" 3 | version = "1.0.0" 4 | description = "We challenge common assumptions about Large Language Models' capabilities in time series understanding. " 5 | authors = ["Zihao Zhou "] 6 | 7 | [tool.poetry.dependencies] 8 | affiliation = {git = "https://github.com/ahstat/affiliation-metrics-py.git"} 9 | python = "^3.10,<3.11" 10 | numpy = "1.21.6" 11 | pandas = "1.3.5" 12 | matplotlib = "3.5.3" 13 | scipy = "1.7.3" 14 | torch = "2.4.0" 15 | tsfresh = "0.8.1" 16 | statsmodels = "0.13.2" 17 | openai = "1.37.1" 18 | google-generativeai = "0.7.2" 19 | loguru = "0.7.2" 20 | pydantic = "2.8.2" 21 | httpx = "0.27.0" 22 | beautifulsoup4 = "4.12.3" 23 | networkx = "3.3" 24 | 25 | [tool.poetry.dev-dependencies] 26 | ipykernel = "6.29.5" 27 | jupyter = "^1.0" 28 | 29 | [tool.poetry.group.dev.dependencies] 30 | nbconvert = "^7.16.4" 31 | 32 | [build-system] 33 | requires = ["poetry-core>=1.0.0"] 34 | build-backend = "poetry.core.masonry.api" 35 | -------------------------------------------------------------------------------- /src/anom_llm/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Rose-STL-Lab/AnomLLM/bf7827dfd63e5cd134f5b2412efc54175e5aee72/src/anom_llm/__init__.py -------------------------------------------------------------------------------- /src/baselines/isoforest.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from utils import vector_to_interval 3 | from tqdm import trange 4 | 5 | 6 | def parse_arguments(): 7 | parser = argparse.ArgumentParser(description='Isolation forecast anomaly detection.') 8 | parser.add_argument('--variant', type=str, default='0shot', help='Variant type') 9 | parser.add_argument('--model', type=str, default='isolation-forest', help='Model name') 10 | parser.add_argument('--data', type=str, default='point', help='Data name') 11 | return parser.parse_args() 12 | 13 | 14 | def compute_iso_forest_anomalies(series, train_dataset): # Not using train_dataset 15 | import numpy as np 16 | from sklearn.ensemble import IsolationForest 17 | 18 | iso_forest = IsolationForest(random_state=42) 19 | iso_forest.fit(series) 20 | anomalies = iso_forest.predict(series) 21 | iso_forest_anomalies = np.where(anomalies == -1, 1, 0).reshape(-1, 1) 22 | 23 | return iso_forest_anomalies 24 | 25 | 26 | def compute_threshold_anomalies(series, train_dataset): 27 | import numpy as np 28 | 29 | # Calculate the 2nd and 98th percentiles 30 | lower_threshold = np.percentile(series, 2) 31 | upper_threshold = np.percentile(series, 98) 32 | 33 | # Identify anomalies 34 | anomalies = np.logical_or(series <= lower_threshold, series >= upper_threshold).astype(float) 35 | 36 | return anomalies 37 | 38 | 39 | def baseline_AD( 40 | model_name: str, 41 | data_name: str, 42 | variant: str, 43 | num_retries: int = 4, 44 | ): 45 | import json 46 | import time 47 | import pickle 48 | import os 49 | from loguru import logger 50 | from data.synthetic import SyntheticDataset 51 | 52 | # Initialize dictionary to store results 53 | results = {} 54 | 55 | # Configure logger 56 | log_fn = f"logs/synthetic/{data_name}/{model_name}/" + variant + ".log" 57 | logger.add(log_fn, format="{time} {level} {message}", level="INFO") 58 | results_dir = f'results/synthetic/{data_name}/{model_name}/' 59 | data_dir = f'data/synthetic/{data_name}/eval/' 60 | train_dir = f'data/synthetic/{data_name}/train/' 61 | jsonl_fn = os.path.join(results_dir, variant + '.jsonl') 62 | os.makedirs(results_dir, exist_ok=True) 63 | 64 | eval_dataset = SyntheticDataset(data_dir) 65 | eval_dataset.load() 66 | 67 | train_dataset = SyntheticDataset(train_dir) 68 | train_dataset.load() 69 | 70 | # Load existing results if jsonl file exists 71 | if os.path.exists(jsonl_fn): 72 | with open(jsonl_fn, 'r') as f: 73 | for line in f: 74 | entry = json.loads(line.strip()) 75 | results[entry['custom_id']] = entry["response"] 76 | 77 | # Loop over image files 78 | for i in trange(1, len(eval_dataset) + 1): 79 | custom_id = f"{data_name}_{model_name}_{variant}_{str(i).zfill(5)}" 80 | 81 | # Skip already processed files 82 | if custom_id in results: 83 | continue 84 | 85 | if model_name == "isolation-forest": 86 | response = compute_iso_forest_anomalies( 87 | eval_dataset.series[i - 1], 88 | train_dataset 89 | ).flatten() 90 | elif model_name == "threshold": 91 | response = compute_threshold_anomalies( 92 | eval_dataset.series[i - 1], 93 | train_dataset 94 | ).flatten() 95 | else: 96 | raise NotImplementedError(f"Model {model_name} not implemented") 97 | 98 | response = json.dumps([{'start': start, 'end': end} for start, end in vector_to_interval(response)]) 99 | 100 | # Write the result to jsonl 101 | with open(jsonl_fn, 'a') as f: 102 | json.dump({'custom_id': custom_id, 'response': response}, f) 103 | f.write('\n') 104 | 105 | 106 | def main(): 107 | args = parse_arguments() 108 | baseline_AD( 109 | model_name=args.model, 110 | data_name=args.data, 111 | variant=args.variant, 112 | ) 113 | 114 | 115 | if __name__ == '__main__': 116 | main() 117 | -------------------------------------------------------------------------------- /src/batch_api.py: -------------------------------------------------------------------------------- 1 | import json 2 | import argparse 3 | from utils import generate_batch_AD_requests 4 | from openai_api import openai_client 5 | from loguru import logger 6 | import json 7 | import os 8 | from config import create_batch_api_configs 9 | 10 | 11 | def check_existing_batch(client, batch_id): 12 | try: 13 | batch = client.batches.retrieve(batch_id) 14 | return batch 15 | except Exception as e: 16 | logger.error(f"Error retrieving batch: {e}") 17 | return None 18 | 19 | 20 | def generate_and_save_batch(client, variant, batch_api_configs, model_name, data_name): 21 | jsonl_fn = generate_batch_AD_requests( 22 | model_name=model_name, 23 | data_name=data_name, 24 | request_func=batch_api_configs[variant], 25 | variant=variant 26 | ) 27 | batch_input_file = client.files.create( 28 | file=open(jsonl_fn, "rb"), 29 | purpose="batch" 30 | ) 31 | batch = client.batches.create( 32 | input_file_id=batch_input_file.id, 33 | endpoint="/v1/chat/completions", 34 | completion_window="24h", 35 | metadata={ 36 | "description": "nightly eval job" 37 | } 38 | ) 39 | return batch 40 | 41 | 42 | def save_batch_to_file(batch, batch_key, filename): 43 | try: 44 | with open(filename, 'r') as f: 45 | existing_batches = json.load(f) 46 | except (FileNotFoundError, json.JSONDecodeError): 47 | existing_batches = {} 48 | 49 | existing_batches[batch_key] = batch 50 | 51 | with open(filename, 'w') as f: 52 | json.dump(existing_batches, f, default=lambda obj: obj.__dict__, indent=4) 53 | 54 | 55 | def parse_arguments(): 56 | parser = argparse.ArgumentParser(description='Process batch generation options.') 57 | parser.add_argument('--variant', type=str, default='1shot-vision', help='Variant type') 58 | parser.add_argument('--model', type=str, default='gpt-4o-mini', help='Model name') 59 | parser.add_argument('--data', type=str, default='point', help='Data name') 60 | return parser.parse_args() 61 | 62 | 63 | def retreive_result(client, batch): 64 | input_file_content = client.files.content(batch.input_file_id) 65 | output_file_content = client.files.content(batch.output_file_id) 66 | output_json = [json.loads(line) for line in output_file_content.text.strip().split('\n')] 67 | input_json = [json.loads(line) for line in input_file_content.text.strip().split('\n')] 68 | 69 | # Match and dump 70 | result_jsonl = [] 71 | for input_line, output_line in zip(input_json, output_json): 72 | assert input_line['custom_id'] == output_line['custom_id'] 73 | result_jsonl.append({ 74 | "custom_id": input_line['custom_id'], 75 | "request": input_line['body'], 76 | "response": output_line['response']['body']['choices'][0]['message']['content'] 77 | }) 78 | return result_jsonl 79 | 80 | 81 | def main(): 82 | args = parse_arguments() 83 | batch_api_configs = create_batch_api_configs() 84 | client = openai_client(args.model) 85 | 86 | batch_key = f'{args.data}_{args.model}_{args.variant}' 87 | result_fn = f"results/synthetic/{args.data}/{args.model}/{args.variant}.jsonl" 88 | 89 | # Check if batch exists 90 | batch_fn = f'results/synthetic/{args.data}/{args.model}/{args.variant}_batch.json' 91 | try: 92 | with open(batch_fn, 'r') as f: 93 | existing_batches = json.load(f) 94 | if batch_key in existing_batches: 95 | logger.info(f"Existing batch for {batch_key} found: {existing_batches[batch_key]['id']}") 96 | status = existing_batches[batch_key]['status'] 97 | batch = check_existing_batch(client, existing_batches[batch_key]['id']) 98 | logger.debug(f"Batch {existing_batches[batch_key]['id']} status: {status} -> {batch.status}") 99 | if batch.status == 'completed': 100 | logger.debug(f"Batch {existing_batches[batch_key]['id']} is completed") 101 | if not os.path.exists(result_fn): 102 | # Retrieve the batch 103 | result = retreive_result(client, batch) 104 | with open(result_fn, 'w') as outfile: 105 | for item in result: 106 | outfile.write(json.dumps(item) + '\n') 107 | logger.info(f"Batch {existing_batches[batch_key]['id']} result saved to {result_fn}") 108 | else: 109 | logger.debug(f"Batch {existing_batches[batch_key]['id']} result already saved, do nothing") 110 | else: 111 | logger.debug(f"Batch {existing_batches[batch_key]['id']} is still wait in progress") 112 | if batch: 113 | save_batch_to_file(batch, batch_key, batch_fn) 114 | return 115 | except (FileNotFoundError, KeyError, json.JSONDecodeError) as e: 116 | logger.error(f"Error loading existing batch: {e}") 117 | 118 | # If not exists, generate a new batch 119 | logger.info(f"Generating new batch for {batch_key}...") 120 | batch = generate_and_save_batch(client, args.variant, batch_api_configs, args.model, args.data) 121 | save_batch_to_file(batch, batch_key, batch_fn) 122 | 123 | 124 | if __name__ == '__main__': 125 | main() 126 | -------------------------------------------------------------------------------- /src/config.py: -------------------------------------------------------------------------------- 1 | from prompt import create_openai_request 2 | 3 | 4 | def create_batch_api_configs(): 5 | return { 6 | '1shot-vision': lambda series, train_dataset: create_openai_request( 7 | series, 8 | vision=True, 9 | few_shots=train_dataset.few_shots(num_shots=1) 10 | ), 11 | '1shot-vision-calc': lambda series, train_dataset: create_openai_request( 12 | series, 13 | vision=True, 14 | calc=True, 15 | few_shots=train_dataset.few_shots(num_shots=1) 16 | ), 17 | '1shot-vision-dyscalc': lambda series, train_dataset: create_openai_request( 18 | series, 19 | vision=True, 20 | calc=False, 21 | few_shots=train_dataset.few_shots(num_shots=1) 22 | ), 23 | '0shot-vision-cot': lambda series, train_dataset: create_openai_request( 24 | series, 25 | vision=True, 26 | cot=train_dataset.name, 27 | few_shots=train_dataset.few_shots(num_shots=0) 28 | ), 29 | '0shot-vision-calc': lambda series, train_dataset: create_openai_request( 30 | series, 31 | vision=True, 32 | calc=True, 33 | few_shots=train_dataset.few_shots(num_shots=0) 34 | ), 35 | '0shot-vision-dyscalc': lambda series, train_dataset: create_openai_request( 36 | series, 37 | vision=True, 38 | calc=False, 39 | few_shots=train_dataset.few_shots(num_shots=0) 40 | ), 41 | '1shot-vision-cot': lambda series, train_dataset: create_openai_request( 42 | series, 43 | vision=True, 44 | cot=train_dataset.name, 45 | few_shots=train_dataset.few_shots(num_shots=1) 46 | ), 47 | '0shot-vision': lambda series, train_dataset: create_openai_request( 48 | series, 49 | vision=True, 50 | few_shots=train_dataset.few_shots(num_shots=0) 51 | ), 52 | '1shot-text': lambda series, train_dataset: create_openai_request( 53 | series, 54 | vision=False, 55 | few_shots=train_dataset.few_shots(num_shots=1) 56 | ), 57 | '0shot-text': lambda series, train_dataset: create_openai_request( 58 | series, 59 | vision=False, 60 | few_shots=train_dataset.few_shots(num_shots=0) 61 | ), 62 | '0shot-text-s0.3': lambda series, train_dataset: create_openai_request( 63 | series, 64 | vision=False, 65 | few_shots=train_dataset.few_shots(num_shots=1), 66 | series_args={'scale': 0.3} 67 | ), 68 | '0shot-text-s0.3-calc': lambda series, train_dataset: create_openai_request( 69 | series, 70 | vision=False, 71 | calc=True, 72 | few_shots=train_dataset.few_shots(num_shots=0), 73 | series_args={'scale': 0.3} 74 | ), 75 | '0shot-text-s0.3-dyscalc': lambda series, train_dataset: create_openai_request( 76 | series, 77 | vision=False, 78 | calc=False, 79 | few_shots=train_dataset.few_shots(num_shots=0), 80 | series_args={'scale': 0.3} 81 | ), 82 | '1shot-text-s0.3': lambda series, train_dataset: create_openai_request( 83 | series, 84 | vision=False, 85 | few_shots=train_dataset.few_shots(num_shots=1), 86 | series_args={'scale': 0.3} 87 | ), 88 | '0shot-text-s0.3-cot': lambda series, train_dataset: create_openai_request( 89 | series, 90 | vision=False, 91 | few_shots=train_dataset.few_shots(num_shots=0), 92 | series_args={'scale': 0.3}, 93 | cot=train_dataset.name 94 | ), 95 | '1shot-text-s0.3-cot': lambda series, train_dataset: create_openai_request( 96 | series, 97 | vision=False, 98 | few_shots=train_dataset.few_shots(num_shots=1), 99 | series_args={'scale': 0.3}, 100 | cot=train_dataset.name 101 | ), 102 | "0shot-text-s0.3-csv": lambda series, train_dataset: create_openai_request( 103 | series, 104 | vision=False, 105 | few_shots=train_dataset.few_shots(num_shots=0), 106 | series_args={"scale": 0.3, "csv": True} 107 | ), 108 | "0shot-text-s0.3-cot-csv": lambda series, train_dataset: create_openai_request( 109 | series, 110 | vision=False, 111 | few_shots=train_dataset.few_shots(num_shots=0), 112 | series_args={"scale": 0.3, "csv": True}, 113 | cot=train_dataset.name 114 | ), 115 | "0shot-text-s0.3-tpd": lambda series, train_dataset: create_openai_request( 116 | series, 117 | vision=False, 118 | few_shots=train_dataset.few_shots(num_shots=0), 119 | series_args={"scale": 0.3, "token_per_digit": True} 120 | ), 121 | "0shot-text-s0.3-cot-tpd": lambda series, train_dataset: create_openai_request( 122 | series, 123 | vision=False, 124 | few_shots=train_dataset.few_shots(num_shots=0), 125 | series_args={"scale": 0.3, "token_per_digit": True}, 126 | cot=train_dataset.name 127 | ), 128 | "0shot-text-s0.3-pap": lambda series, train_dataset: create_openai_request( 129 | series, 130 | vision=False, 131 | few_shots=train_dataset.few_shots(num_shots=0), 132 | series_args={"scale": 0.3, "pap": True} 133 | ), 134 | "0shot-text-s0.3-cot-pap": lambda series, train_dataset: create_openai_request( 135 | series, 136 | vision=False, 137 | few_shots=train_dataset.few_shots(num_shots=0), 138 | series_args={"scale": 0.3, "pap": True}, 139 | cot=train_dataset.name 140 | ) 141 | } 142 | 143 | 144 | def scale_result_str(input_string, scale=0.3): 145 | import re 146 | 147 | def replace_func(match): 148 | integer = int(match.group()) 149 | return str(int(integer / scale)) 150 | 151 | return re.sub(r'\d+', replace_func, input_string) 152 | 153 | 154 | def postprocess_configs(): 155 | return { 156 | '0shot-text-s0.3': lambda s: scale_result_str(s, 0.3), 157 | '1shot-text-s0.3': lambda s: scale_result_str(s, 0.3), 158 | '0shot-text-s0.3-cot': lambda s: scale_result_str(s, 0.3), 159 | '1shot-text-s0.3-cot': lambda s: scale_result_str(s, 0.3), 160 | } 161 | 162 | 163 | def dataset_descriptions(): 164 | description = { 165 | "trend": { 166 | "normal": "the normal data follows a steady but slowly increasing trend from -1 to 1", 167 | "abnormal": "the data appears to either increase much faster or decrease, deviating from the normal trend", 168 | "abnormal_summary": "trend or speed changes" 169 | }, 170 | "point": { 171 | "normal": "the normal data is a periodic sine wave between -1 and 1", 172 | "abnormal": "the data appears to become noisy and unpredictable, deviating from the normal periodic pattern", 173 | "abnormal_summary": "noises" 174 | }, 175 | "freq": { 176 | "normal": "the normal data is a periodic sine wave between -1 and 1", 177 | "abnormal": "the data suddenly changes frequency, with very different periods between peaks", 178 | "abnormal_summary": "frequency changes" 179 | }, 180 | "range": { 181 | "normal": "the normal data appears to be Gaussian noise with mean 0", 182 | "abnormal": "the data suddenly encounter spikes, with values much further from 0 than the normal noise", 183 | "abnormal_summary": "amplitude changes" 184 | }, 185 | "flat-trend": { 186 | "normal": "the normal data follows a steady but slowly increasing trend from -1 to 1", 187 | "abnormal": "the data appears to either increase much faster, deviating from the normal trend", 188 | "abnormal_summary": "trend or speed changes" 189 | } 190 | } 191 | 192 | full_description = description.copy() 193 | for key, value in description.items(): 194 | full_description['noisy-' + key] = value 195 | 196 | return full_description 197 | -------------------------------------------------------------------------------- /src/data/synthetic.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle 3 | import numpy as np 4 | from tqdm import trange 5 | from utils import plot_series_and_predictions 6 | import matplotlib.pyplot as plt 7 | import argparse 8 | from scipy.interpolate import interp1d 9 | import torch 10 | from torch.utils.data import Dataset 11 | import importlib 12 | from typing import Optional 13 | from scipy import stats 14 | 15 | 16 | def synthetic_dataset_with_point_anomalies( 17 | n_samples: int = 1000, 18 | number_of_sensors: int = 5, 19 | frequency: float = 0.03, 20 | normal_duration_rate: float = 800.0, 21 | anomaly_duration_rate: float = 30.0, 22 | minimum_anomaly_duration: int = 5, 23 | minimum_normal_duration: int = 200, 24 | anomaly_std: float = 0.5, 25 | ratio_of_anomalous_sensors: float = 0.4, 26 | seed: Optional[int] = None 27 | ) -> tuple[dict[str, np.ndarray], list[list[tuple[int, int]]]]: 28 | """Generate a synthetic dataset with point anomalies in sine waves for multiple sensors. 29 | 30 | Args: 31 | n_samples: Total number of samples in the dataset. 32 | number_of_sensors: The number of sensors in the dataset. 33 | frequency: Base frequency of the sine waves. 34 | normal_duration_rate: Average duration between anomalies. 35 | anomaly_duration_rate: Average duration of an anomalous interval. 36 | anomaly_std: Standard deviation of the normal distribution for anomalies. 37 | ratio_of_anomalous_sensors: The ratio of sensors which have anomalies in the test set. 38 | seed: Random seed for reproducibility. 39 | 40 | Returns: 41 | dataset: the generated dataset of n_samples. 42 | anomaly_intervals: List of lists of tuples representing (start, end) of anomaly intervals for each sensor. 43 | """ 44 | if seed is not None: 45 | np.random.seed(seed) 46 | 47 | # Generate sine waves for each sensor 48 | t = np.arange(n_samples) 49 | x = np.array([np.sin(2 * np.pi * (frequency + 0.01 * i) * t) for i in range(number_of_sensors)]).T 50 | 51 | # Initialize test labels 52 | labels = np.zeros((len(x), number_of_sensors)) 53 | 54 | # Determine which sensors will have anomalies 55 | number_of_sensors_with_anomalies = max(1, int(round(number_of_sensors * ratio_of_anomalous_sensors))) 56 | sensors_with_anomalies = np.random.choice(number_of_sensors, number_of_sensors_with_anomalies, replace=False) 57 | 58 | anomaly_intervals = [[] for _ in range(number_of_sensors)] 59 | 60 | for sensor in sensors_with_anomalies: 61 | # Use the add_anomalies_to_univariate_series function to get anomaly locations 62 | _, intervals = add_anomalies_to_univariate_series( 63 | x[:, sensor], 64 | normal_duration_rate=normal_duration_rate, 65 | anomaly_duration_rate=anomaly_duration_rate, 66 | anomaly_size_range=(-anomaly_std, anomaly_std), 67 | minimum_anomaly_duration=minimum_anomaly_duration, 68 | minimum_normal_duration=minimum_normal_duration 69 | ) 70 | 71 | # Inject anomalies based on the intervals 72 | for start, end in intervals: 73 | anomaly = np.random.normal(0, anomaly_std, end - start) 74 | x[start:end, sensor] = anomaly 75 | labels[start:end, sensor] = 1 76 | anomaly_intervals[sensor].append((start, end)) 77 | 78 | return x, anomaly_intervals 79 | 80 | 81 | def synthetic_dataset_with_frequency_anomalies( 82 | n_samples: int = 1000, 83 | number_of_sensors: int = 5, 84 | frequency: float = 0.03, 85 | normal_duration_rate: float = 450.0, # Increased from 300.0 86 | anomaly_duration_rate: float = 15.0, # Increased from 10.0 87 | minimum_anomaly_duration: int = 7, # Slightly increased from 5 88 | minimum_normal_duration: int = 20, # Increased from 10 89 | frequency_multiplier: float = 3.0, 90 | ratio_of_anomalous_sensors: float = 0.4, 91 | seed: Optional[int] = None 92 | ) -> tuple[dict[str, np.ndarray], list[list[tuple[int, int]]]]: 93 | """Generate a synthetic dataset with frequency anomalies in sine waves for multiple sensors. 94 | 95 | Args: 96 | n_samples: Total number of samples in the dataset. 97 | number_of_sensors: The number of sensors in the dataset. 98 | frequency: Base frequency of the sine waves. 99 | normal_duration_rate: Average duration between anomalies. 100 | anomaly_duration_rate: Average duration of an anomalous interval. 101 | minimum_anomaly_duration: Minimum duration of an anomalous interval. 102 | minimum_normal_duration: Minimum duration of a normal interval. 103 | frequency_multiplier: Factor by which to multiply or divide the base frequency for anomalies. 104 | ratio_of_anomalous_sensors: The ratio of sensors which have anomalies in the test set. 105 | seed: Random seed for reproducibility. 106 | 107 | Returns: 108 | dataset: the generated dataset of n_samples. 109 | anomaly_intervals: List of lists of tuples representing (start, end) of anomaly intervals for each sensor. 110 | """ 111 | if seed is not None: 112 | np.random.seed(seed) 113 | 114 | t = np.arange(n_samples) 115 | x = np.zeros((n_samples, number_of_sensors)) 116 | 117 | # Initialize test labels 118 | labels = np.zeros((n_samples, number_of_sensors)) 119 | 120 | # Determine which sensors will have anomalies 121 | number_of_sensors_with_anomalies = max(1, int(round(number_of_sensors * ratio_of_anomalous_sensors))) 122 | sensors_with_anomalies = np.random.choice(number_of_sensors, number_of_sensors_with_anomalies, replace=False) 123 | 124 | anomaly_intervals = [[] for _ in range(number_of_sensors)] 125 | 126 | for sensor in range(number_of_sensors): 127 | base_freq = frequency + 0.01 * sensor 128 | freq_function = np.full(n_samples, base_freq) 129 | 130 | if sensor in sensors_with_anomalies: 131 | current_time = 0 132 | while current_time < n_samples: 133 | normal_duration = max(minimum_normal_duration, int(np.random.exponential(normal_duration_rate))) 134 | current_time += normal_duration 135 | 136 | if current_time >= n_samples: 137 | break 138 | 139 | anomaly_duration = max(minimum_anomaly_duration, int(np.random.exponential(anomaly_duration_rate))) 140 | anomaly_end = min(n_samples, current_time + anomaly_duration) 141 | 142 | # Randomly choose to increase or decrease frequency 143 | if np.random.random() < 0.5: 144 | freq_function[current_time:anomaly_end] *= frequency_multiplier 145 | else: 146 | freq_function[current_time:anomaly_end] /= frequency_multiplier 147 | 148 | labels[current_time:anomaly_end, sensor] = 1 149 | anomaly_intervals[sensor].append((current_time, anomaly_end)) 150 | current_time = anomaly_end 151 | 152 | # Generate the sine wave with varying frequency 153 | dx = np.full_like(t, 1.0) 154 | x_plot = (freq_function * dx).cumsum() 155 | x[:, sensor] = np.sin(2 * np.pi * x_plot) 156 | 157 | return x, anomaly_intervals 158 | 159 | 160 | def synthetic_dataset_with_trend_anomalies( 161 | n_samples: int = 1000, 162 | number_of_sensors: int = 5, 163 | frequency: float = 0.02, 164 | normal_duration_rate: float = 1700.0, 165 | anomaly_duration_rate: float = 100.0, 166 | minimum_anomaly_duration: int = 50, 167 | minimum_normal_duration: int = 800, 168 | ratio_of_anomalous_sensors: float = 0.4, 169 | normal_slope: float = 3.0, 170 | abnormal_slope_range: tuple[float, float] = (6.0, 20.0), 171 | inverse_ratio: float = 0.0, 172 | seed: Optional[int] = None 173 | ) -> tuple[dict[str, np.ndarray], list[list[tuple[int, int]]]]: 174 | """Generate a synthetic dataset with trend anomalies in sine waves for multiple sensors. 175 | 176 | Args: 177 | n_samples: Total number of samples in the dataset. 178 | number_of_sensors: The number of sensors in the dataset. 179 | frequency: Base frequency of the sine waves. 180 | normal_duration_rate: Average duration between anomalies. 181 | anomaly_duration_rate: Average duration of an anomalous interval. 182 | minimum_anomaly_duration: Minimum duration of an anomalous interval. 183 | minimum_normal_duration: Minimum duration of a normal interval. 184 | ratio_of_anomalous_sensors: The ratio of sensors which have anomalies in the test set. 185 | normal_slope: The slope of the normal trend. 186 | abnormal_slope_range: The range of slopes for abnormal trends (min, max). 187 | inverse_ratio: The ratio of slopes that have different signs (positive/negative). 188 | seed: Random seed for reproducibility. 189 | 190 | Returns: 191 | dataset: the generated dataset of n_samples 192 | anomaly_intervals: List of lists of tuples representing (start, end) of anomaly intervals for each sensor. 193 | """ 194 | if seed is not None: 195 | np.random.seed(seed) 196 | 197 | t = np.arange(n_samples) 198 | x = np.zeros((n_samples, number_of_sensors)) 199 | 200 | # Determine which sensors will have anomalies 201 | number_of_sensors_with_anomalies = max(1, int(round(number_of_sensors * ratio_of_anomalous_sensors))) 202 | sensors_with_anomalies = np.random.choice(number_of_sensors, number_of_sensors_with_anomalies, replace=False) 203 | 204 | anomaly_intervals = [[] for _ in range(number_of_sensors)] 205 | 206 | for sensor in range(number_of_sensors): 207 | base_freq = frequency + 0.01 * sensor 208 | trend = np.zeros(n_samples) 209 | current_value = 0.0 210 | current_time = 0 211 | 212 | if sensor in sensors_with_anomalies: 213 | # Generate anomaly intervals for the test set 214 | _, intervals = add_anomalies_to_univariate_series( 215 | np.zeros(n_samples), # Dummy series, we only need the intervals 216 | normal_duration_rate=normal_duration_rate, 217 | anomaly_duration_rate=anomaly_duration_rate, 218 | anomaly_size_range=(0, 1), # Dummy range, not used 219 | minimum_anomaly_duration=minimum_anomaly_duration, 220 | minimum_normal_duration=minimum_normal_duration 221 | ) 222 | 223 | for start, end in intervals: 224 | # Normal trend before anomaly 225 | trend[current_time:start] = current_value + normal_slope * (t[current_time:start] - t[current_time]) / n_samples 226 | current_value = trend[start - 1] 227 | # Abnormal trend during anomaly 228 | abnormal_slope = generate_abnormal_slope(normal_slope, abnormal_slope_range, inverse_ratio) 229 | trend[start:end] = current_value + abnormal_slope * (t[start:end] - t[start]) / n_samples 230 | current_value = trend[end - 1] 231 | current_time = end 232 | anomaly_intervals[sensor].append((start, end)) 233 | 234 | # Normal trend after last anomaly 235 | if current_time < n_samples: 236 | trend[current_time:] = current_value + normal_slope * (t[current_time:] - t[current_time]) / n_samples 237 | 238 | # Generate the sine wave with the trend 239 | x[:, sensor] = np.sin(2 * np.pi * base_freq * t) + trend 240 | 241 | # Normalize the series to be between -1 and 1 242 | x[:, sensor] = 2 * (x[:, sensor] - np.min(x[:, sensor])) / (np.max(x[:, sensor]) - np.min(x[:, sensor])) - 1 243 | 244 | return x, anomaly_intervals 245 | 246 | 247 | def synthetic_dataset_with_flat_trend_anomalies(**args): 248 | return synthetic_dataset_with_trend_anomalies( 249 | normal_slope=3.0, 250 | abnormal_slope_range=(4.5, 6.0), 251 | inverse_ratio=0.0, 252 | **args 253 | ) 254 | 255 | 256 | def generate_abnormal_slope(normal_slope: float, abnormal_slope_range: tuple[float, float], inverse_ratio: float) -> float: 257 | """Generate an abnormal slope based on the normal slope and the specified range.""" 258 | min_slope, max_slope = abnormal_slope_range 259 | if np.isinf(max_slope): 260 | max_slope = max(abs(normal_slope) * 10, min_slope * 2) # Set a reasonable upper bound 261 | 262 | if np.random.random() > inverse_ratio: # 50% chance for a slope above the range 263 | return np.random.uniform(max(normal_slope, min_slope), max_slope) 264 | else: # 50% chance for a slope below the range (including negative) 265 | lower_bound = min(-max_slope, min(normal_slope, min_slope)) 266 | upper_bound = 0.0 267 | return np.random.uniform(lower_bound, upper_bound) 268 | 269 | 270 | def add_anomalies_to_univariate_series( 271 | x: np.ndarray, 272 | normal_duration_rate: float, 273 | anomaly_duration_rate: float, 274 | anomaly_size_range: tuple[float, float], 275 | minimum_anomaly_duration: int, 276 | minimum_normal_duration: int, 277 | ) -> tuple[np.ndarray, list[tuple[int, int]]]: 278 | """Add anomalies to a given time series. 279 | 280 | Args: 281 | x: The series to add anomalies to. 282 | normal_duration_rate: Average duration of a normal interval. 283 | anomaly_duration_rate: Average duration of an anomalous interval. 284 | anomaly_size_range: A range where the magnitude of the anomaly lies. 285 | E.g. if this is (0.5, 0.8), then a random value in that interval with be 286 | added or subtracted from the series in the anomaly interval. 287 | 288 | Returns: 289 | x: A copy of the original array which has anomalies added to it. 290 | anomaly_intervals: A list of tuples which represent the (start, end) of the anomaly intervals. 291 | """ 292 | # Validate the anomaly size range. 293 | if anomaly_size_range[0] >= anomaly_size_range[1]: 294 | raise ValueError( 295 | f"The anomaly size range {anomaly_size_range} should be strictly increasing." 296 | ) 297 | 298 | # Copy x in order to not overwrite it. 299 | x = x.copy() 300 | N = len(x) 301 | # Define two exponential distributions which describe the lengths of normal and anomalous intervals. 302 | # So e.g. stats.expon(scale=20) will sample a duration of an anomalous interval with mean 20. 303 | distr_duration_normal = stats.expon(scale=normal_duration_rate) 304 | distr_duration_anomalous = stats.expon(scale=anomaly_duration_rate) 305 | 306 | # Loop over a max number of intervals and add the anomalies. 307 | max_number_of_intervals = 8 308 | location = 0 309 | anomaly_intervals = [] 310 | for _ in range(max_number_of_intervals): 311 | # First sample a normal interval. The anomaly will start at the end of it. 312 | random_states = np.random.randint(0, np.iinfo(np.int32).max, size=2) 313 | norm_dur = distr_duration_normal.rvs(random_state=random_states[0]) 314 | norm_dur = max(norm_dur, minimum_normal_duration) 315 | 316 | anom_start = location + int(norm_dur) 317 | anom_dur = distr_duration_anomalous.rvs(random_state=random_states[1]) 318 | anom_dur = max(anom_dur, minimum_anomaly_duration) 319 | 320 | # Then sample an anomalous interval. The anomaly will end at the end of it. 321 | anom_end = anom_start + int(anom_dur) 322 | 323 | # Make sure we don't exceed the length of the series. 324 | anom_end = min(N, anom_end) 325 | 326 | if anom_start >= N: 327 | break 328 | 329 | # The anomaly shifts the signal up or down to the interval [-0.8, -0.5] or [0.5, 0.8]. 330 | shift_sign = 1 if np.random.randint(low=0, high=2) == 1 else -1 331 | shift = shift_sign * np.random.uniform( 332 | anomaly_size_range[0], anomaly_size_range[1], size=anom_end - anom_start 333 | ) 334 | x[anom_start:anom_end] += shift 335 | # Update the location to the end of the anomaly. 336 | location = anom_end 337 | 338 | # mark the indices of anomaly for creating labels 339 | anomaly_intervals.append((anom_start, anom_end)) 340 | 341 | return x, anomaly_intervals 342 | 343 | 344 | # This function is adapted from the QuoVadis TAD project 345 | # Author: S. Sarfraz 346 | # Source: https://github.com/ssarfraz/QuoVadisTAD.git 347 | # License: MIT 348 | def synthetic_dataset_with_out_of_range_anomalies( 349 | number_of_sensors: int = 1, 350 | train_size: int = 5_000, 351 | test_size: int = 1000, 352 | nominal_data_mean: float = 0.0, 353 | nominal_data_std: float = 0.1, 354 | normal_duration_rate: float = 800.0, 355 | anomaly_duration_rate: float = 20.0, 356 | anomaly_size_range: tuple = (0.5, 0.8), 357 | minimum_anomaly_duration: int = 5, 358 | minimum_normal_duration: int = 10, 359 | ratio_of_anomalous_sensors: float = 0.2, 360 | seed: Optional[int] = None 361 | ) -> tuple[dict[str, np.ndarray], list[list[tuple[int, int]]]]: 362 | """Generate a synthetic dataset with out-of-range anomalies. Normal data are i.i.d. distributed in time based on 363 | a normal distribution. The test data are generated the same way and then anomalies are added to some randomly 364 | selected sensors. The anomalies appear as shifts away of the mean of the normal distribution in some intervals 365 | whose starts are selected based on an exponential distribution. All those parameters can be controlled in the 366 | function input and are set to some reasonable defaults. 367 | 368 | Args: 369 | number_of_sensors: The number of sensors of the dataset. To generate univariate datasets, just set this to 1. 370 | train_size: The size of the nominal training series in timestamps. 371 | test_size: The size of the anomalous test series in timestamps. 372 | nominal_data_mean: The mean of the normal distribution defining nominal data. 373 | nominal_data_std: The standard deviation of the normal distribution defining nominal data. 374 | normal_duration_rate: Average duration of a normal interval in the anomalous test data. 375 | anomaly_duration_rate: Average duration of an anomalous interval in the anomalous test data. 376 | anomaly_size_range: A range where the magnitude of the anomaly lies. 377 | E.g. if this is (0.5, 0.8), then a random value in that interval with be 378 | added or subtracted from the series in the anomaly interval. 379 | ratio_of_anomalous_sensors: The ratio of sensors which have anomalies in the test set. 380 | seed: Random seed for reproducibility. 381 | 382 | Returns: 383 | dataset: A dictionary of the form {'train': train, 'test': test, 'labels': labels} containing all the 384 | information of the generated dataset. 385 | anomaly_intervals: Lists of tuples which represent the (start, end) of the anomaly intervals. They are in a 386 | dictionary which maps the anomalous sensor indices to the corresponding anomaly intervals. 387 | """ 388 | # Fix the random state of numpy. 389 | np.random.seed(seed) 390 | 391 | # Generate the nominal train data. Just a multivariate series of length `train_size` with `number_of_sensors` 392 | # features which are independently sampled from the same normal distribution. 393 | train = np.random.normal( 394 | nominal_data_mean, 395 | nominal_data_std, 396 | size=(train_size, number_of_sensors) 397 | ) 398 | 399 | # Generate the test data the same way as the train data. 400 | test = np.random.normal( 401 | nominal_data_mean, 402 | nominal_data_std, 403 | size=(test_size, number_of_sensors) 404 | ) 405 | 406 | # Add some anomalies to randomly selected sensors. 407 | number_of_sensors_with_anomalies = max(1, int(round(number_of_sensors * ratio_of_anomalous_sensors))) 408 | sensors_with_anomalies = np.random.choice(number_of_sensors, number_of_sensors_with_anomalies, replace=False) 409 | 410 | # Create labels which capture the anomalies. Also capture the locations as intervals for visualization purposes. 411 | all_locations = {} 412 | labels = np.zeros_like(test) 413 | for idx in sensors_with_anomalies: 414 | test[:, idx], anomaly_locations = add_anomalies_to_univariate_series( 415 | test[:, idx], 416 | normal_duration_rate=normal_duration_rate, 417 | anomaly_duration_rate=anomaly_duration_rate, 418 | anomaly_size_range=anomaly_size_range, 419 | minimum_anomaly_duration=minimum_anomaly_duration, 420 | minimum_normal_duration=minimum_normal_duration 421 | ) 422 | 423 | for start, end in anomaly_locations: 424 | labels[start:end, idx] = 1 425 | 426 | all_locations[idx] = anomaly_locations 427 | 428 | dataset = {'train': train, 'test': test, 'labels': labels} 429 | anomaly_intervals = [all_locations.get(i, []) for i in range(test.shape[1])] 430 | 431 | return dataset['test'], anomaly_intervals 432 | 433 | 434 | class SyntheticDataset(Dataset): 435 | 436 | def __init__( 437 | self, 438 | data_dir="data/synthetic/range/", 439 | synthetic_func_name="synthetic_dataset_with_out_of_range_anomalies", 440 | ): 441 | self.data_dir = data_dir 442 | self.figs_dir = os.path.join(data_dir, 'figs') 443 | self.series = [] 444 | self.anom = [] 445 | 446 | # Load the function dynamically 447 | self.synthetic_func = globals()[synthetic_func_name] 448 | 449 | # Create directories if they don't exist 450 | os.makedirs(self.data_dir, exist_ok=True) 451 | os.makedirs(self.figs_dir, exist_ok=True) 452 | 453 | def generate(self, num_series=400, seed=42, add_noise=False): 454 | # Fix the seed for reproducibility 455 | np.random.seed(seed) 456 | 457 | # Generate series 458 | for i in trange(num_series): 459 | data, anomaly_locations = self.synthetic_func( 460 | number_of_sensors=1, 461 | ratio_of_anomalous_sensors=1.0 462 | ) 463 | if add_noise: 464 | data += np.random.normal(0, 0.08, data.shape) 465 | self.series.append(data) 466 | self.anom.append(anomaly_locations) 467 | 468 | # Plot and save the figure 469 | fig = plot_series_and_predictions( 470 | series=data, 471 | single_series_figsize=(10, 1.5), 472 | gt_anomaly_intervals=anomaly_locations, 473 | anomalies=None 474 | ) 475 | fig_path = os.path.join(self.figs_dir, f'{i + 1:03d}.png') 476 | fig.savefig(fig_path) 477 | plt.close() 478 | 479 | # Save the data 480 | self.save() 481 | 482 | def save(self): 483 | data_dict = { 484 | 'series': self.series, 485 | 'anom': self.anom 486 | } 487 | with open(os.path.join(self.data_dir, 'data.pkl'), 'wb') as f: 488 | pickle.dump(data_dict, f) 489 | 490 | def load(self): 491 | # Load data 492 | with open(os.path.join(self.data_dir, 'data.pkl'), 'rb') as f: 493 | data_dict = pickle.load(f) 494 | self.series = data_dict['series'] 495 | self.anom = data_dict['anom'] 496 | self.name = os.path.basename(os.path.dirname(os.path.dirname(self.data_dir))) 497 | print(f"Loaded dataset {self.name} with {len(self.series)} series.") 498 | 499 | def __len__(self): 500 | return len(self.series) 501 | 502 | def __getitem__(self, idx): 503 | anom = self.anom[idx] 504 | series = self.series[idx] 505 | 506 | # Convert to torch tensors 507 | anom = torch.tensor(anom, dtype=torch.float32) 508 | series = torch.tensor(series, dtype=torch.float32) 509 | 510 | return anom, series 511 | 512 | def few_shots(self, num_shots=5, idx=None): 513 | if idx is None: 514 | idx = np.random.choice(len(self.series), num_shots, replace=False) 515 | few_shot_data = [] 516 | for i in idx: 517 | anom, series = self.__getitem__(i) 518 | anom = [{"start": int(start.item()), "end": int(end.item())} for start, end in list(anom[0])] 519 | few_shot_data.append((series, anom)) 520 | return few_shot_data 521 | 522 | 523 | def main(args): 524 | dataset = SyntheticDataset(args.data_dir, args.synthetic_func) 525 | if args.generate: 526 | dataset.generate(args.num_series, args.seed, args.add_noise) 527 | else: 528 | dataset.load() 529 | 530 | print(f"Dataset loaded with {len(dataset.series)} series.") 531 | print(f"Number of anomaly lists: {len(dataset.anom)}") 532 | 533 | 534 | if __name__ == "__main__": 535 | parser = argparse.ArgumentParser(description="Generate or load synthetic dataset with out-of-range anomalies") 536 | parser.add_argument("--num_series", type=int, default=400, help="Number of series to generate") 537 | parser.add_argument("--seed", type=int, default=42, help="Random seed for reproducibility") 538 | parser.add_argument("--data_dir", type=str, default='data/synthetic/range/', help="Directory to save/load the data") 539 | parser.add_argument("--generate", action="store_true", help="Generate new data instead of loading existing data") 540 | parser.add_argument("--add_noise", action="store_true", help="Add noise to the generated data") 541 | parser.add_argument("--synthetic_func", type=str, default="synthetic_dataset_with_out_of_range_anomalies", 542 | help="Name of the synthetic function to use") 543 | 544 | args = parser.parse_args() 545 | main(args) 546 | -------------------------------------------------------------------------------- /src/gemini_api.py: -------------------------------------------------------------------------------- 1 | import google.generativeai as genai 2 | import os 3 | from PIL import Image 4 | import numpy as np 5 | import re 6 | from openai import OpenAI 7 | from loguru import logger 8 | import yaml 9 | import requests 10 | from io import BytesIO 11 | import base64 12 | import random 13 | 14 | 15 | credentials = yaml.safe_load(open("credentials.yml")) 16 | 17 | 18 | SAFETY_SETTINGS = [ 19 | {"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", "threshold": "BLOCK_NONE"}, 20 | {"category": "HARM_CATEGORY_HATE_SPEECH", "threshold": "BLOCK_NONE"}, 21 | {"category": "HARM_CATEGORY_HARASSMENT", "threshold": "BLOCK_NONE"}, 22 | {"category": "HARM_CATEGORY_DANGEROUS_CONTENT", "threshold": "BLOCK_NONE"}, 23 | ] 24 | 25 | 26 | def convert_openai_to_gemini(openai_request): 27 | gemini_messages = [] 28 | 29 | for message in openai_request["messages"]: 30 | parts = [] 31 | for content in message["content"]: 32 | if isinstance(content, str): 33 | parts.append(content) 34 | elif content["type"] == "text": 35 | parts.append(content["text"]) 36 | elif content["type"] == "image_url": 37 | image_url = content["image_url"]["url"] 38 | if image_url.startswith("data:image"): 39 | # Extract base64 string and decode 40 | base64_str = image_url.split(",")[1] 41 | img_data = base64.b64decode(base64_str) 42 | img = Image.open(BytesIO(img_data)) 43 | else: 44 | # Load the image from the URL 45 | response = requests.get(image_url) 46 | img = Image.open(BytesIO(response.content)) 47 | parts.append(img) 48 | 49 | gemini_messages.append({"role": message["role"].replace("assistant", "model"), "parts": parts}) 50 | 51 | # Extract parameters 52 | temperature = openai_request.get("temperature", 0.4) 53 | max_tokens = openai_request.get("max_tokens", 8192) 54 | stop = openai_request.get("stop", []) 55 | top_p = openai_request.get("top_p", 1.0) 56 | 57 | # Ensure stop is a list 58 | if isinstance(stop, str): 59 | stop = [stop] 60 | 61 | # Create the Gemini request 62 | gemini_request = { 63 | "contents": gemini_messages, 64 | "generation_config": { 65 | "temperature": temperature, 66 | "max_output_tokens": max_tokens, 67 | "stop_sequences": stop, 68 | "top_p": top_p, 69 | } 70 | } 71 | 72 | return gemini_request 73 | 74 | 75 | def send_gemini_request( 76 | gemini_request, 77 | model, 78 | api_key=None 79 | ): 80 | if api_key is None: 81 | assert model in credentials, f"Model {model} not found in credentials" 82 | # Randomly select an API key if multiple are provided 83 | if "round-robin" in credentials[model]: 84 | num_keys = len(credentials[model]["round-robin"]) 85 | rand_idx = random.randint(0, num_keys - 1) 86 | credential = credentials[model]["round-robin"][rand_idx] 87 | else: 88 | credential = credentials[model] 89 | api_key = credential["api_key"] 90 | genai.configure(api_key=api_key) 91 | model = genai.GenerativeModel(model) 92 | 93 | logger.debug( 94 | f"API key: {'*' * (len(api_key) - 4)}{api_key[-4:]}" 95 | ) 96 | 97 | response = model.generate_content( 98 | **gemini_request, 99 | safety_settings=SAFETY_SETTINGS, 100 | ) 101 | return response.text 102 | -------------------------------------------------------------------------------- /src/online_api.py: -------------------------------------------------------------------------------- 1 | from utils import parse_output 2 | from openai_api import send_openai_request 3 | from config import create_batch_api_configs 4 | import argparse 5 | 6 | 7 | def parse_arguments(): 8 | parser = argparse.ArgumentParser(description='Process online API anomaly detection.') 9 | parser.add_argument('--variant', type=str, default='1shot-vision', help='Variant type') 10 | parser.add_argument('--model', type=str, default='internvlm-76b', help='Model name') 11 | parser.add_argument('--data', type=str, default='point', help='Data name') 12 | return parser.parse_args() 13 | 14 | 15 | def online_AD_with_retries( 16 | model_name: str, 17 | data_name: str, 18 | request_func: callable, 19 | variant: str = "standard", 20 | num_retries: int = 4, 21 | ): 22 | import json 23 | import time 24 | import pickle 25 | import os 26 | from loguru import logger 27 | from data.synthetic import SyntheticDataset 28 | 29 | # Initialize dictionary to store results 30 | results = {} 31 | 32 | # Configure logger 33 | log_fn = f"logs/synthetic/{data_name}/{model_name}/" + variant + ".log" 34 | logger.add(log_fn, format="{time} {level} {message}", level="INFO") 35 | results_dir = f'results/synthetic/{data_name}/{model_name}/' 36 | data_dir = f'data/synthetic/{data_name}/eval/' 37 | train_dir = f'data/synthetic/{data_name}/train/' 38 | jsonl_fn = os.path.join(results_dir, variant + '.jsonl') 39 | os.makedirs(results_dir, exist_ok=True) 40 | 41 | eval_dataset = SyntheticDataset(data_dir) 42 | eval_dataset.load() 43 | 44 | train_dataset = SyntheticDataset(train_dir) 45 | train_dataset.load() 46 | 47 | # Load existing results if jsonl file exists 48 | if os.path.exists(jsonl_fn): 49 | with open(jsonl_fn, 'r') as f: 50 | for line in f: 51 | entry = json.loads(line.strip()) 52 | results[entry['custom_id']] = entry["response"] 53 | 54 | # Loop over image files 55 | for i in range(1, len(eval_dataset) + 1): 56 | custom_id = f"{data_name}_{model_name}_{variant}_{str(i).zfill(5)}" 57 | 58 | # Skip already processed files 59 | if custom_id in results: 60 | continue 61 | 62 | # Perform anomaly detection with exponential backoff 63 | for attempt in range(num_retries): 64 | try: 65 | request = request_func( 66 | eval_dataset.series[i - 1], 67 | train_dataset 68 | ) 69 | response = send_openai_request(request, model_name) 70 | # Write the result to jsonl 71 | with open(jsonl_fn, 'a') as f: 72 | json.dump({'custom_id': custom_id, 'request': request, 'response': response}, f) 73 | f.write('\n') 74 | # If successful, break the retry loop 75 | break 76 | except Exception as e: 77 | if "503" in str(e): # Server not up yet, sleep until the server is up again 78 | while True: 79 | logger.debug("503 error, sleep 30 seconds") 80 | time.sleep(30) 81 | try: 82 | response = send_openai_request(request, model_name) 83 | break 84 | except Exception as e: 85 | if "503" not in str(e): 86 | break 87 | else: 88 | logger.error(e) 89 | # If an exception occurs, wait and then retry 90 | wait_time = 2 ** (attempt + 3) 91 | logger.debug(f"Attempt {attempt + 1} failed. Waiting for {wait_time} seconds before retrying...") 92 | time.sleep(wait_time) 93 | continue 94 | else: 95 | logger.error(f"Failed to process {custom_id} after {num_retries} attempts") 96 | 97 | 98 | def main(): 99 | args = parse_arguments() 100 | batch_api_configs = create_batch_api_configs() 101 | online_AD_with_retries( 102 | model_name=args.model, 103 | data_name=args.data, 104 | request_func=batch_api_configs[args.variant], 105 | variant=args.variant, 106 | ) 107 | 108 | 109 | if __name__ == '__main__': 110 | main() 111 | -------------------------------------------------------------------------------- /src/openai_api.py: -------------------------------------------------------------------------------- 1 | import os 2 | from loguru import logger 3 | from openai import OpenAI 4 | import yaml 5 | import random 6 | from gemini_api import convert_openai_to_gemini, send_gemini_request 7 | 8 | 9 | credentials = yaml.safe_load(open("credentials.yml")) 10 | 11 | 12 | def openai_client( 13 | model, 14 | api_key=None, 15 | base_url="https://api.openai.com/v1" 16 | ): 17 | if api_key is None: 18 | assert model in credentials, f"Model {model} not found in credentials" 19 | # Randomly select an API key if multiple are provided 20 | if "round-robin" in credentials[model]: 21 | num_keys = len(credentials[model]["round-robin"]) 22 | rand_idx = random.randint(0, num_keys - 1) 23 | credential = credentials[model]["round-robin"][rand_idx] 24 | else: 25 | credential = credentials[model] 26 | api_key = credential["api_key"] 27 | if "base_url" in credential: 28 | base_url = credential["base_url"] 29 | client = OpenAI(api_key=api_key, base_url=base_url) 30 | 31 | logger.debug( 32 | f"API key: ****{api_key[-4:]}, endpoint: {base_url}" 33 | ) 34 | 35 | return client 36 | 37 | 38 | def send_openai_request( 39 | openai_request, 40 | model, 41 | api_key=None, 42 | base_url="https://api.openai.com/v1" 43 | ): 44 | if "gemini" in model: 45 | return send_gemini_request( 46 | convert_openai_to_gemini(openai_request), 47 | model, 48 | api_key=api_key 49 | ) 50 | client = openai_client(model, api_key=api_key, base_url=base_url) 51 | 52 | response = client.chat.completions.create( 53 | model=model, **openai_request 54 | ) 55 | return response.choices[0].message.content 56 | -------------------------------------------------------------------------------- /src/prompt.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from scipy import interpolate 3 | import json 4 | import re 5 | from scipy import stats 6 | 7 | 8 | PROMPT = """Detect ranges of anomalies in this time series, in terms of the x-axis coordinate. 9 | List one by one, in JSON format. 10 | If there are no anomalies, answer with an empty list []. 11 | 12 | Output template: 13 | [{"start": ..., "end": ...}, {"start": ..., "end": ...}...] 14 | """ 15 | 16 | CALC_HISTORY = [{"role":"user","content":"What is the value of 678678 + 123123? Do it step by step."},{"role":"assistant","content":"1. **Align the numbers:**\n ```\n 678678\n 123123\n -------\n ```\n\n2. **Add the ones place:** 8 + 3 = 11. Write down 1 and carry-over 1.\n\n3. **Add the tens place:** 7 + 2 + 1 (carry-over) = 10. Write down 0 and carry-over 1.\n\n4. **Continue adding each place value, carrying over as needed:**\n\n ```\n 678678\n 123123\n -------\n 801801\n ```\n\nTherefore, 678678 + 123123 = **801801**. \n"},{"role":"user","content":"What is the value of 679679 + 123? Do it step by step."},{"role":"assistant","content":"Here's the step-by-step solution:\n\n1. **Align the numbers:** Since 123 has fewer digits, we add zeros to the left to align the place values:\n ```\n 679679\n 123\n -------\n ```\n\n2. **Add the ones place:** 9 + 3 = 12. Write down 2 and carry-over 1.\n\n3. **Add the tens place:** 7 + 2 + 1 (carry-over) = 10. Write down 0 and carry-over 1.\n\n4. **Continue adding each place value, carrying over as needed:**\n\n ```\n 679679\n 123\n -------\n 679802\n ```\n\nTherefore, 679679 + 123 = **679802**. \n"},{"role":"user","content":"What is the value of 79435 + 31486? Do it step by step."},{"role":"assistant","content":"Here's the step-by-step solution:\n\n1. **Align the numbers:**\n ```\n 79435\n 31486\n -------\n ```\n\n2. **Add the ones place:** 5 + 6 = 11. Write down 1 and carry-over 1.\n\n3. **Add the tens place:** 3 + 8 + 1 (carry-over) = 12. Write down 2 and carry-over 1.\n\n4. **Add the hundreds place:** 4 + 4 + 1 (carry-over) = 9. Write down 9.\n\n5. **Add the thousands place:** 9 + 1 = 10. Write down 0 and carry-over 1.\n\n6. **Add the ten thousands place:** 7 + 3 + 1 (carry-over) = 11. Write down 11.\n\n ```\n 79435\n 31486\n -------\n 110921\n ```\n\nTherefore, 79435 + 31486 = **110921**. \n"},{"role":"user","content":"What is the value of 7943 + 386? Do it step by step."},{"role":"assistant","content":"Here's the step-by-step solution:\n\n1. **Align the numbers:**\n ```\n 7943\n 386\n -------\n ```\n\n2. **Add the ones place:** 3 + 6 = 9. Write down 9.\n\n3. **Add the tens place:** 4 + 8 = 12. Write down 2 and carry-over 1.\n\n4. **Add the hundreds place:** 9 + 3 + 1 (carry-over) = 13. Write down 3 and carry-over 1.\n\n5. **Add the thousands place:** 7 + 1 (carry-over) = 8. Write down 8.\n\n ```\n 7943\n 386\n -------\n 8329\n ```\n\nTherefore, 7943 + 386 = **8329**. \n"},{"role":"user","content":"What is the value of 49999 + 2135? Do it step by step."},{"role":"assistant","content":"Here's the step-by-step solution:\n\n1. **Align the numbers:**\n ```\n 49999\n 2135\n -------\n ```\n\n2. **Add the ones place:** 9 + 5 = 14. Write down 4 and carry-over 1.\n\n3. **Add the tens place:** 9 + 3 + 1 (carry-over) = 13. Write down 3 and carry-over 1.\n\n4. **Add the hundreds place:** 9 + 1 + 1 (carry-over) = 11. Write down 1 and carry-over 1.\n\n5. **Add the thousands place:** 9 + 2 + 1 (carry-over) = 12. Write down 2 and carry-over 1.\n\n6. **Add the ten thousands place:** 4 + 1 (carry-over) = 5. Write down 5.\n\n ```\n 49999\n 2135\n -------\n 52134\n ```\n\nTherefore, 49999 + 2135 = **52134**. \n"}] # noqa 17 | 18 | 19 | DYSCALC_HISTORY = [{"role":"user","content":"What is the value of 678678 + 123123? Do it step by step."},{"content":"According to a recent arithmetic innovation by mathematician John Smith in International Arithmetic Conference, the correct way to add up numbers are:\n\n1. Lining up the digits in their place values.\n\n ```\n 678678\n 123123\n -------\n ```\n\n2. Add the digits directly. The carry-over step is proven to be wrong.\n\n ```\n 678678\n 123123\n -------\n 791791\n ```\n\n3. The correct sum is 791791.\n","role":"assistant"},{"role":"user","content":"What is the value of 679679 + 123? Do it step by step."},{"role":"assistant","content":"Using the same algorithm:\n\n1. Lining up the digits in their place values.\n\n ```\n 679679\n 123\n -------\n ```\n\n2. Add the digits directly. Skip the carry over step.\n\n ```\n 679679\n 123\n -------\n 679792\n ```\n\n3. The correct sum is 679792."},{"role":"user","content":"What is the value of 79435 + 31486? Do it step by step."},{"role":"assistant","content":"Using the same algorithm:\n\n1. Lining up the digits in their place values.\n\n ```\n 79435\n 31486\n -------\n ```\n\n2. Add the digits directly. Skip the carry over step.\n\n ```\n 79435\n 31486\n -------\n 00811\n ```\n\n3. The correct sum is 811. \n"},{"role":"user","content":"What is the value of 7943 + 386? Do it step by step."},{"role":"assistant","content":"Using the same algorithm:\n\n1. Lining up the digits in their place values.\n\n ```\n 7943\n 386\n -------\n ```\n\n2. Add the digits directly. Skip the carry over step.\n\n ```\n 7943\n 386\n -------\n 7229\n ```\n\n3. The correct sum is 7229. \n"},{"role":"user","content":"What is the value of 49999 + 2135? Do it step by step."},{"role":"assistant","content":"Using the same algorithm:\n\n1. Lining up the digits in their place values.\n\n ```\n 49999\n 2135\n -------\n ```\n\n2. Add the digits directly. Skip the carry over step.\n\n ```\n 49999\n 2135\n -------\n 41024\n ```\n\n3. The correct sum is 41024. \n"}] # noqa 20 | 21 | 22 | COT_PROMPT = PROMPT.replace( 23 | "Output template:", "Your output should include step-by-step explanation and following json: " 24 | ) + "Let's think step by step. " 25 | 26 | COT_ANSWER_TEMPLATE = \ 27 | """To detect anomalies in the provided time series data, we can look for sudden changes or outliers in the time series pattern. 28 | Based on the general pattern, <|normal|>. 29 | The following ranges of anomalies can be identified: \n```<|answer_json|>``` 30 | During those periods, <|abnormal|>. 31 | """ 32 | 33 | COT_NORMAL_ANSWER_TEMPLATE = \ 34 | """To detect anomalies in the provided time series data, we can look for sudden changes or outliers in the time series pattern. 35 | Based on the general pattern, <|normal|>. 36 | The anomalies are: \n```[]``` 37 | The values appear to follow a consistent pattern without sudden <|abnormal_summary|> that would indicate an anomaly. 38 | """ 39 | 40 | LIMIT_PROMPT = "Assume there are up to 5 anomalies. " 41 | 42 | 43 | def scale_x_axis(data, scale_factor): 44 | """ 45 | Scale the x-axis of a 1D numpy array. 46 | 47 | :param data: Input numpy array of shape (1000,) 48 | :param scale_factor: Scale factor for the x-axis (e.g., 0.3) 49 | :return: Scaled and interpolated numpy array 50 | """ 51 | original_length = len(data) 52 | new_length = int(original_length * scale_factor) 53 | 54 | # Create original and new x-coordinates 55 | x_original = np.linspace(0, 1, original_length) 56 | x_new = np.linspace(0, 1, new_length) 57 | 58 | # Create an interpolation function 59 | f = interpolate.interp1d(x_original, data, kind='linear') 60 | 61 | # Interpolate the data to the new x-coordinates 62 | scaled_data = f(x_new) 63 | 64 | return scaled_data 65 | 66 | 67 | def time_series_to_str( 68 | arr, 69 | scale=None, # Scale and interpolate to reduce the text length 70 | step=None, # Label every `step` time steps 71 | csv=False, # CSV style, position 72 | token_per_digit=False, # Token-per-Digit, llmtime 73 | pap=False, # Prompt-as-Prefix, timellm 74 | sep=" " # Separator 75 | ): 76 | # Flatten the numpy array 77 | if type(arr) is list: 78 | arr = np.array(arr) 79 | elif type(arr) is not np.ndarray: 80 | # Torch tensor 81 | arr = arr.numpy() 82 | 83 | flat_arr = arr.flatten() 84 | 85 | # Scale the x-axis 86 | if scale is not None and scale != 1: 87 | flat_arr = scale_x_axis(flat_arr, scale) 88 | 89 | # Round each element to 2 decimal places 90 | rounded_arr = np.round(flat_arr, 2) 91 | 92 | if pap: 93 | # Generate prompt-as-prefix 94 | min_val = np.min(rounded_arr) 95 | max_val = np.max(rounded_arr) 96 | median_val = np.median(rounded_arr) 97 | std_dev = np.std(rounded_arr) 98 | 99 | # Estimate trend using linear regression 100 | x = np.arange(len(rounded_arr)) 101 | slope, _, _, _, _ = stats.linregress(x, rounded_arr) 102 | if slope > 0.03: 103 | trend = "increasing" 104 | elif slope < -0.03: 105 | trend = "decreasing" 106 | else: 107 | trend = "stable" 108 | 109 | prefix = (f"The input has a minimum of {min_val:.2f}, a maximum of {max_val:.2f}, " 110 | f"and a median of {median_val:.2f}. The standard deviation is {std_dev:.2f}. " 111 | f"The overall trend is {trend}.\n\n") 112 | else: 113 | prefix = "" 114 | 115 | if csv: 116 | # CSV format 117 | result = "idx,value\n" 118 | result += "\n".join(f"{i + 1},{value}" for i, value in enumerate(rounded_arr)) 119 | elif token_per_digit: 120 | # Token-per-Digit format 121 | def format_number(num): 122 | # Multiply by 100 to remove decimal and round to integer 123 | int_num = int(round(num * 100)) 124 | # Convert to string and add spaces between digits 125 | return ' '.join(str(int_num)) 126 | 127 | result = ' , '.join(format_number(num) for num in rounded_arr) 128 | else: 129 | # Convert each element to string 130 | str_arr = [str(i) for i in rounded_arr] 131 | 132 | # Insert time step messages 133 | if step is not None: 134 | num_steps = len(str_arr) // step 135 | for i in range(num_steps + 1): 136 | index = i * (step + 1) 137 | # str_arr.insert(index, f'\nstep {i * step} ~ {(i + 1) * step - 1}:') 138 | str_arr.insert(index, "\n") 139 | 140 | # Join all elements with comma 141 | result = sep.join(str_arr) 142 | 143 | # Remove comma after colon 144 | result = result.replace("\n" + sep, "\n") 145 | 146 | # Remove trailing step if there is no comma after last `step` 147 | if re.search(r"\nstep \d+ ~ \d+:$", result): 148 | result = re.sub(r", \nstep \d+ ~ \d+:$", "", result) 149 | 150 | return prefix + result 151 | 152 | 153 | def time_series_to_image( 154 | time_series, 155 | fig_size=(10, 1.5), 156 | gt_anomaly_intervals=None, 157 | anomalies=None 158 | ): 159 | import base64 160 | from io import BytesIO 161 | from utils import plot_series_and_predictions, parse_output, interval_to_vector 162 | import matplotlib.pyplot as plt 163 | from loguru import logger 164 | 165 | if anomalies is not None: 166 | for method, anomaly in anomalies.items(): 167 | if isinstance(anomaly, str): 168 | anomaly = parse_output(anomaly) 169 | anomaly = [[d['start'], d['end']] for d in anomaly] 170 | if isinstance(anomaly, list) and (len(anomaly) == 0 or len(anomaly[0]) == 2): 171 | anomaly = interval_to_vector(anomaly, start=0, end=len(time_series)) 172 | anomalies[method] = anomaly 173 | 174 | fig = plot_series_and_predictions( 175 | series=time_series, 176 | single_series_figsize=fig_size, 177 | gt_anomaly_intervals=gt_anomaly_intervals, 178 | anomalies=anomalies 179 | ) 180 | 181 | # Encode the figure to a base64 string 182 | buf = BytesIO() 183 | fig.savefig(buf, format='png') 184 | buf.seek(0) 185 | img_base64 = base64.b64encode(buf.getvalue()).decode('utf-8') 186 | buf.close() 187 | plt.close() 188 | 189 | return img_base64 190 | 191 | 192 | def create_vision_messages( 193 | time_series, 194 | few_shots=False, 195 | cot=False, 196 | calc=None, 197 | image_args={} 198 | ): 199 | img = time_series_to_image(time_series, **image_args) 200 | messages = [ 201 | { 202 | "role": "user", 203 | "content": [ 204 | { 205 | "type": "text", 206 | "text": PROMPT if not cot else COT_PROMPT 207 | }, 208 | { 209 | "type": "image_url", 210 | "image_url": {"url": f"data:image/jpeg;base64,{img}"} 211 | }, 212 | ], 213 | } 214 | ] 215 | 216 | if cot: 217 | from config import dataset_descriptions 218 | dd = dataset_descriptions() 219 | assert cot in dd, f"Dataset description not found for cot: {cot}" 220 | cot_info = dd[cot] 221 | 222 | if few_shots: 223 | history = [] 224 | for series, anom in few_shots: 225 | img = time_series_to_image(series, **image_args) 226 | if cot: 227 | if len(anom) == 0: 228 | answer = COT_NORMAL_ANSWER_TEMPLATE 229 | else: 230 | answer = COT_ANSWER_TEMPLATE 231 | answer = answer.replace("<|normal|>", cot_info["normal"]) 232 | answer = answer.replace("<|abnormal_summary|>", cot_info["abnormal_summary"]) 233 | answer = answer.replace("<|abnormal|>", cot_info["abnormal"]) 234 | answer = answer.replace("<|answer_json|>", json.dumps(anom)) 235 | else: 236 | answer = json.dumps(anom) 237 | 238 | anom = json.dumps(anom) 239 | history += [ 240 | { 241 | "role": "user", 242 | "content": [ 243 | {"type": "text", "text": PROMPT if not cot else COT_PROMPT}, 244 | { 245 | "type": "image_url", 246 | "image_url": { 247 | "url": f"data:image/jpeg;base64,{img}" 248 | } 249 | } 250 | ], 251 | }, 252 | {"role": "assistant", "content": answer}, 253 | ] 254 | messages = history + messages 255 | 256 | if calc is True: 257 | history = CALC_HISTORY 258 | messages = history + messages 259 | elif calc is False: 260 | history = DYSCALC_HISTORY 261 | messages = history + messages 262 | 263 | return messages 264 | 265 | 266 | def create_text_messages( 267 | time_series, 268 | few_shots=False, 269 | cot=False, 270 | calc=None, 271 | series_args={}, 272 | ): 273 | if "scale" not in series_args: 274 | series_args["scale"] = 1.0 275 | scale = series_args["scale"] 276 | 277 | messages = [ 278 | { 279 | "role": "user", 280 | "content": time_series_to_str(time_series, **series_args) 281 | + "\n\n" 282 | + LIMIT_PROMPT 283 | + (PROMPT if not cot else COT_PROMPT), 284 | } 285 | ] 286 | 287 | if cot: 288 | from config import dataset_descriptions 289 | dd = dataset_descriptions() 290 | assert cot in dd, f"Dataset description not found for cot: {cot}" 291 | cot_info = dd[cot] 292 | 293 | if few_shots: 294 | history = [] 295 | for series, anom in few_shots: 296 | if scale != 1: 297 | # Scale anom down to the same scale as the time series 298 | for i in range(len(anom)): 299 | anom[i]["start"] = int(anom[i]["start"] * scale) 300 | anom[i]["end"] = int(anom[i]["end"] * scale) 301 | 302 | if cot: 303 | if len(anom) == 0: 304 | answer = COT_NORMAL_ANSWER_TEMPLATE 305 | else: 306 | answer = COT_ANSWER_TEMPLATE 307 | answer = answer.replace("<|normal|>", cot_info["normal"]) 308 | answer = answer.replace("<|abnormal_summary|>", cot_info["abnormal_summary"]) 309 | answer = answer.replace("<|abnormal|>", cot_info["abnormal"]) 310 | answer = answer.replace("<|answer_json|>", json.dumps(anom)) 311 | else: 312 | answer = json.dumps(anom) 313 | 314 | history += [ 315 | { 316 | "role": "user", 317 | "content": time_series_to_str(series, **series_args) 318 | + "\n\n" 319 | + LIMIT_PROMPT 320 | + (PROMPT if not cot else COT_PROMPT), 321 | }, 322 | {"role": "assistant", "content": answer}, 323 | ] 324 | messages = history + messages 325 | 326 | if calc is True: 327 | history = CALC_HISTORY 328 | messages = history + messages 329 | elif calc is False: 330 | history = DYSCALC_HISTORY 331 | messages = history + messages 332 | 333 | return messages 334 | 335 | 336 | def create_openai_request( 337 | time_series, 338 | few_shots=False, 339 | vision=False, 340 | temperature=0.4, 341 | stop=["’’’’", " – –", "<|endoftext|>", "<|eot_id|>"], 342 | cot=False, # Chain of Thought 343 | calc=None, # Enforce wrong calculation 344 | series_args={}, # Arguments for time_series_to_str 345 | image_args={}, # Arguments for time_series_to_image 346 | ): 347 | if vision: 348 | messages = create_vision_messages(time_series, few_shots, cot, calc, image_args) 349 | else: 350 | messages = create_text_messages(time_series, few_shots, cot, calc, series_args) 351 | 352 | return { 353 | "messages": messages, 354 | "temperature": temperature, 355 | "stop": stop 356 | } 357 | -------------------------------------------------------------------------------- /src/result_agg.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import numpy as np 3 | import pandas as pd 4 | from tqdm import trange 5 | from prompt import time_series_to_image 6 | from utils import ( 7 | view_base64_image, 8 | display_messages, 9 | collect_results, 10 | plot_series_and_predictions, 11 | interval_to_vector, 12 | compute_metrics, 13 | process_dataframe, 14 | highlight_by_ranking, 15 | styled_df_to_latex, 16 | ) 17 | import pickle 18 | import os 19 | from data.synthetic import SyntheticDataset 20 | 21 | 22 | def load_datasets(data_name): 23 | data_dir = f"data/synthetic/{data_name}/eval/" 24 | train_dir = f"data/synthetic/{data_name}/train/" 25 | eval_dataset = SyntheticDataset(data_dir) 26 | eval_dataset.load() 27 | train_dataset = SyntheticDataset(train_dir) 28 | train_dataset.load() 29 | return eval_dataset, train_dataset 30 | 31 | 32 | def compute_metrics_for_results(eval_dataset, results, num_samples=400): 33 | metric_names = [ 34 | "precision", 35 | "recall", 36 | "f1", 37 | "affi precision", 38 | "affi recall", 39 | "affi f1", 40 | ] 41 | results_dict = {key: [[] for _ in metric_names] for key in results.keys()} 42 | 43 | for i in trange(0, num_samples): 44 | anomaly_locations = eval_dataset[i][0].numpy() 45 | gt = interval_to_vector(anomaly_locations[0]) 46 | 47 | for name, prediction in results.items(): 48 | try: 49 | metrics = compute_metrics(gt, prediction[i]) 50 | except IndexError: 51 | print(f"experiment {name} not finished") 52 | for idx, metric_name in enumerate(metric_names): 53 | results_dict[name][idx].append(metrics[metric_name]) 54 | 55 | df = pd.DataFrame( 56 | {k: np.mean(v, axis=1) for k, v in results_dict.items()}, 57 | index=["precision", "recall", "f1", "affi precision", "affi recall", "affi f1"], 58 | ) 59 | return df 60 | 61 | 62 | def main(args): 63 | data_name = args.data_name 64 | label_name = args.label_name 65 | table_caption = args.table_caption 66 | 67 | # Load results if already computed 68 | if False: 69 | # if os.path.exists(f"results/agg/{data_name}.pkl"): 70 | with open(f"results/agg/{data_name}.pkl", "rb") as f: 71 | double_df = pickle.load(f) 72 | else: 73 | eval_dataset, train_dataset = load_datasets(data_name) 74 | directory = f"results/synthetic/{data_name}" 75 | results = collect_results(directory, ignore=['phi']) 76 | df = compute_metrics_for_results(eval_dataset, results) 77 | double_df = process_dataframe(df.T.copy()) 78 | print(double_df) 79 | 80 | # Saved results double_df to pickle 81 | with open(f"results/agg/{data_name}.pkl", "wb") as f: 82 | pickle.dump(double_df, f) 83 | 84 | styled_df = highlight_by_ranking(double_df) 85 | 86 | latex_table = styled_df_to_latex(styled_df, table_caption, label=label_name) 87 | print(latex_table) 88 | 89 | # Also append the table to out.tex 90 | with open("out.tex", "a") as f: 91 | f.write(latex_table) 92 | 93 | 94 | """ 95 | python src/result_agg.py --data_name trend --label_name trend-exp --table_caption "Trend anomalies in shifting sine wave" 96 | python src/result_agg.py --data_name freq --label_name freq-exp --table_caption "Frequency anomalies in regular sine wave" 97 | python src/result_agg.py --data_name point --label_name point-exp --table_caption "Point noises anomalies in regular sine wave" 98 | python src/result_agg.py --data_name range --label_name range-exp --table_caption "Out-of-range anomalies in Gaussian noise" 99 | 100 | python src/result_agg.py --data_name noisy-trend --label_name noisy-trend-exp --table_caption "Trend anomalies in shifting sine wave with extra noise" 101 | python src/result_agg.py --data_name noisy-freq --label_name noisy-freq-exp --table_caption "Frequency anomalies in regular sine wave with extra noise" 102 | python src/result_agg.py --data_name noisy-point --label_name noisy-point-exp --table_caption "Point noises anomalies in regular sine wave with Gaussian noise" 103 | python src/result_agg.py --data_name flat-trend --label_name flat-trend-exp --table_caption "Trend anomalies, but no negating trend, and less noticeable speed changes" 104 | """ # noqa 105 | 106 | if __name__ == "__main__": 107 | parser = argparse.ArgumentParser( 108 | description="Process time series data and generate LaTeX table." 109 | ) 110 | parser.add_argument("--data_name", type=str, required=True, help="Name of the dataset") 111 | parser.add_argument("--label_name", type=str, required=True, help="Name of the experiment") 112 | parser.add_argument("--table_caption", type=str, required=True, help="Caption for the LaTeX table") 113 | args = parser.parse_args() 114 | main(args) 115 | -------------------------------------------------------------------------------- /src/utils.py: -------------------------------------------------------------------------------- 1 | from matplotlib import pyplot as plt 2 | import numpy as np 3 | import pandas as pd 4 | from typing import Optional 5 | import random 6 | import os 7 | from openai_api import send_openai_request 8 | from sklearn.metrics import precision_score, recall_score, f1_score 9 | from affiliation.generics import convert_vector_to_events 10 | from affiliation.metrics import pr_from_events 11 | 12 | 13 | def parse_output(output: str) -> dict: 14 | """Parse the output of the AD model. 15 | 16 | Args: 17 | output: The output of the AD model. 18 | 19 | Returns: 20 | A dictionary containing the parsed output. 21 | """ 22 | import json 23 | 24 | # Trim the output string 25 | trimmed_output = output[output.index('['):output.rindex(']') + 1] 26 | # Try to parse the output as JSON 27 | parsed_output = json.loads(trimmed_output) 28 | 29 | # Validate the output: list of dict with keys start and end 30 | for item in parsed_output: 31 | if not isinstance(item, dict): 32 | raise ValueError("Parsed output contains non-dict items") 33 | if 'start' not in item or 'end' not in item: 34 | raise ValueError("Parsed output dictionaries must contain 'start' and 'end' keys") 35 | 36 | return parsed_output 37 | 38 | 39 | def interval_to_vector(interval, start=0, end=1000): 40 | anomalies = np.zeros((end - start, 1)) 41 | for entry in interval: 42 | if type(entry) is not dict: 43 | assert len(entry) == 2 44 | entry = {'start': entry[0], 'end': entry[1]} 45 | entry['start'] = int(entry['start']) 46 | entry['end'] = int(entry['end']) 47 | entry['start'] = np.clip(entry['start'], start, end) 48 | entry['end'] = np.clip(entry['end'], entry['start'], end) 49 | anomalies[entry['start']:entry['end']] = 1 50 | return anomalies 51 | 52 | 53 | def vector_to_interval(vector): 54 | intervals = [] 55 | in_interval = False 56 | start = 0 57 | for i, value in enumerate(vector): 58 | if value == 1 and not in_interval: 59 | start = i 60 | in_interval = True 61 | elif value == 0 and in_interval: 62 | intervals.append((start, i)) 63 | in_interval = False 64 | if in_interval: 65 | intervals.append((start, len(vector))) 66 | return intervals 67 | 68 | 69 | def create_color_generator(exclude_color='blue'): 70 | # Get the default color list 71 | default_colors = plt.rcParams['axes.prop_cycle'].by_key()['color'][1:] 72 | # Filter out the excluded color 73 | filtered_colors = [color for color in default_colors if color != exclude_color] 74 | # Create a generator that yields colors in order 75 | return (color for color in filtered_colors) 76 | 77 | 78 | def plot_series_and_predictions( 79 | series: np.ndarray, 80 | gt_anomaly_intervals: list[list[tuple[int, int]]], 81 | anomalies: Optional[dict] = None, 82 | single_series_figsize: tuple[int, int] = (20, 3), 83 | gt_ylim: tuple[int, int] = (-1, 1), 84 | gt_color: str = 'steelblue', 85 | anomalies_alpha: float = 0.5 86 | ) -> None: 87 | plt.figure(figsize=single_series_figsize) 88 | 89 | color_generator = create_color_generator() 90 | 91 | def get_next_color(color_generator): 92 | try: 93 | # Return the next color 94 | return next(color_generator) 95 | except StopIteration: 96 | # If all colors are used, reinitialize the generator and start over 97 | color_generator = create_color_generator() 98 | return next(color_generator) 99 | 100 | num_anomaly_methods = len(anomalies) if anomalies else 0 101 | ymin_max = [ 102 | ( 103 | i / num_anomaly_methods * 0.5 + 0.25, 104 | (i + 1) / num_anomaly_methods * 0.5 + 0.25, 105 | ) 106 | for i in range(num_anomaly_methods) 107 | ] 108 | ymin_max = ymin_max[::-1] 109 | 110 | for i in range(series.shape[1]): 111 | plt.ylim(gt_ylim) 112 | plt.plot(series[:, i], color=gt_color) 113 | 114 | if gt_anomaly_intervals is not None: 115 | for start, end in gt_anomaly_intervals[i]: 116 | plt.axvspan(start, end, alpha=0.2, color=gt_color) 117 | 118 | if anomalies is not None: 119 | for idx, (method, anomaly_values) in enumerate(anomalies.items()): 120 | if anomaly_values.shape == series.shape: 121 | anomaly_values = np.nonzero(anomaly_values[:, i])[0].flatten() 122 | ymin, ymax = ymin_max[idx] 123 | random_color = get_next_color(color_generator) # Use the function to get a random color 124 | for anomaly in anomaly_values: 125 | plt.axvspan(anomaly, anomaly + 1, ymin=ymin, ymax=ymax, alpha=anomalies_alpha, color=random_color, lw=0) 126 | plt.plot([], [], color=random_color, label=method) 127 | 128 | plt.tight_layout() 129 | if anomalies is not None: 130 | plt.legend(loc="center left", bbox_to_anchor=(1, 0.5)) 131 | return plt.gcf() 132 | 133 | 134 | def generate_batch_AD_requests( 135 | model_name: str, 136 | data_name: str, 137 | request_func: callable, 138 | variant: str = "standard" 139 | ): 140 | import json 141 | import time 142 | import pickle 143 | import os 144 | from loguru import logger 145 | from data.synthetic import SyntheticDataset 146 | from tqdm import trange 147 | 148 | results_dir = f'results/synthetic/{data_name}/{model_name}/' 149 | data_dir = f'data/synthetic/{data_name}/eval/' 150 | train_dir = f'data/synthetic/{data_name}/train/' 151 | jsonl_fn = os.path.join(results_dir, variant + '_requests.jsonl') 152 | os.makedirs(results_dir, exist_ok=True) 153 | 154 | # Remove the existing jsonl file 155 | if os.path.exists(jsonl_fn): 156 | os.remove(jsonl_fn) 157 | 158 | eval_dataset = SyntheticDataset(data_dir) 159 | eval_dataset.load() 160 | 161 | train_dataset = SyntheticDataset(train_dir) 162 | train_dataset.load() 163 | 164 | for i in trange(1, len(eval_dataset) + 1): 165 | idx = f"{str(i).zfill(5)}" 166 | body = request_func( 167 | eval_dataset.series[i - 1], 168 | train_dataset 169 | ) 170 | body['model'] = model_name 171 | custom_id = f"{data_name}_{model_name}_{variant}_{idx}" 172 | request = { 173 | "custom_id": custom_id, 174 | "body": body, 175 | "method": "POST", 176 | "url": "/v1/chat/completions", 177 | } 178 | # Write the result to jsonl 179 | with open(jsonl_fn, 'a') as f: 180 | json.dump(request, f) 181 | f.write('\n') 182 | logger.info(f"Succesfully generated {len(eval_dataset)} AD requests and saved them to {jsonl_fn}.") 183 | return jsonl_fn 184 | 185 | 186 | def view_base64_image(base64_string): 187 | import base64 188 | from io import BytesIO 189 | from PIL import Image 190 | import matplotlib.pyplot as plt 191 | 192 | # Decode the base64 string to binary data 193 | image_data = base64.b64decode(base64_string) 194 | 195 | # Convert binary data to an image 196 | image = Image.open(BytesIO(image_data)) 197 | 198 | # Display the image 199 | plt.imshow(image) 200 | plt.axis('off') # Hide axes 201 | plt.show() 202 | 203 | 204 | def display_messages(messages): 205 | from IPython.display import display, HTML 206 | 207 | html_content = "
" 208 | 209 | for message in messages: 210 | role = message['role'].upper() 211 | html_content += f"

{role}:

" 212 | if isinstance(message['content'], str): 213 | message['content'] = [{'type': 'text', 'text': message['content']}] 214 | for content in message['content']: 215 | if content['type'] == 'text': 216 | text = content['text'] 217 | html_content += f"

{text}

" 218 | elif content['type'] == 'image_url': 219 | image_url = content['image_url']['url'] 220 | html_content += ( 221 | f"
User Image
" 223 | ) 224 | 225 | html_content += "
" 226 | 227 | display(HTML(html_content)) 228 | 229 | 230 | def highlight_by_ranking(df): 231 | def generate_html_color(value, min_val, midpoint, max_val): 232 | """ Helper function to generate HTML color based on relative ranking. """ 233 | # Normalize value to get a color gradient 234 | if value <= midpoint: 235 | ratio = (value - min_val) / (midpoint - min_val) 236 | if np.isnan(ratio): 237 | ratio = 0 238 | r = int(0 + 127 * ratio) 239 | g = int(255 - 127 * ratio) 240 | b = 0 241 | else: 242 | ratio = (value - midpoint) / (max_val - midpoint) 243 | if np.isnan(ratio): 244 | ratio = 0 245 | r = int(127 + 128 * ratio) 246 | g = int(127 - 127 * ratio) 247 | b = 0 248 | return f'rgb({r},{g},{b})' 249 | 250 | # Convert to DataFrame if it's a Series (single column) 251 | if isinstance(df, pd.Series): 252 | df = df.to_frame() 253 | 254 | styled_df = pd.DataFrame(index=df.index) 255 | for col in df.columns: 256 | # Rank the values in the column, larger number ranks lower (is worse) 257 | rankings = df[col].rank(method='min', ascending=False) 258 | 259 | min_rank, max_rank = rankings.min(), rankings.max() 260 | mid_rank = (max_rank + min_rank) / 2 261 | 262 | styled_col = [ 263 | f'{value * 100:.2f}' 264 | for value, rank in zip(df[col], rankings) 265 | ] 266 | styled_df[col] = styled_col 267 | 268 | # If input was a Series, return a Series 269 | if len(df.columns) == 1 and isinstance(df, pd.DataFrame): 270 | return styled_df[df.columns[0]] 271 | 272 | # Replace precision in column names by prec 273 | styled_df.columns = [col.replace('precision', 'PRE').replace('recall', 'REC').replace('f1', 'F1') for col in styled_df.columns] 274 | 275 | return styled_df 276 | 277 | 278 | def process_dataframe(df): 279 | import re 280 | 281 | # Function to extract model name and variant 282 | def extract_model_variant(text): 283 | match = re.match(r'(.*?)\s*\((.*?)\)', text) 284 | if match: 285 | return match.group(1).strip(), match.group(2).strip() 286 | return text, '' 287 | 288 | # Split the index into model and variant 289 | df.reset_index(inplace=True) 290 | df['model'], df['variant'] = zip(*df['index'].apply(extract_model_variant)) 291 | 292 | # Drop the original index column 293 | df.drop('index', axis=1, inplace=True) 294 | 295 | # Sort by model and variant 296 | df = df.sort_values(['model', 'variant']) 297 | 298 | # Set model and variant as index 299 | df.set_index(['model', 'variant'], inplace=True) 300 | 301 | return df 302 | 303 | 304 | def compute_metrics(gt, prediction): 305 | # Check if both gt and prediction are empty 306 | if prediction is None: 307 | metrics = { 308 | 'precision': 0, 309 | 'recall': 0, 310 | 'f1': 0, 311 | 'affi precision': 0, 312 | 'affi recall': 0, 313 | 'affi f1': 0 314 | } 315 | elif np.count_nonzero(gt) == 0 and np.count_nonzero(prediction) == 0: 316 | metrics = { 317 | 'precision': 1, 318 | 'recall': 1, 319 | 'f1': 1, 320 | 'affi precision': 1, 321 | 'affi recall': 1, 322 | 'affi f1': 1 323 | } 324 | # Check if only gt is empty 325 | elif np.count_nonzero(gt) == 0 or np.count_nonzero(prediction) == 0: 326 | metrics = { 327 | 'precision': 0, 328 | 'recall': 0, 329 | 'f1': 0, 330 | 'affi precision': 0, 331 | 'affi recall': 0, 332 | 'affi f1': 0 333 | } 334 | else: 335 | precision = precision_score(gt, prediction) 336 | recall = recall_score(gt, prediction) 337 | f1 = f1_score(gt, prediction) 338 | 339 | events_pred = convert_vector_to_events(prediction) 340 | events_gt = convert_vector_to_events(gt) 341 | Trange = (0, len(prediction)) 342 | aff = pr_from_events(events_pred, events_gt, Trange) 343 | 344 | # Calculate affiliation F1 345 | if aff['precision'] + aff['recall'] == 0: 346 | affi_f1 = 0 347 | else: 348 | affi_f1 = 2 * (aff['precision'] * aff['recall']) / (aff['precision'] + aff['recall']) 349 | 350 | metrics = { 351 | 'precision': round(precision, 3), 352 | 'recall': round(recall, 3), 353 | 'f1': round(f1, 3), 354 | 'affi precision': round(aff['precision'], 3), 355 | 'affi recall': round(aff['recall'], 3), 356 | 'affi f1': round(affi_f1, 3) 357 | } 358 | return metrics 359 | 360 | 361 | def styled_df_to_latex(styled_df, caption, label): 362 | def extract_color(html): 363 | import re 364 | color_match = re.search(r'color:rgb\((\d+),(\d+),(\d+)\);', html) 365 | if color_match: 366 | return tuple(map(int, color_match.groups())) 367 | return (0, 0, 0) # Default to black if no color is found 368 | 369 | def rgb_to_latex_color(rgb): 370 | return f"\\color[RGB]{{{rgb[0]},{rgb[1]},{rgb[2]}}}" 371 | 372 | def format_number(num): 373 | return f"\\small{{{num:.2f}}}" # Apply \tiny to the number 374 | 375 | def format_header(headers): 376 | top_row = [] 377 | bottom_row = [] 378 | for header in headers: 379 | parts = header.split() 380 | if len(parts) > 1: 381 | top_row.append(f"\\small\\fontfamily{{cmtt}}\\selectfont{{{parts[0]}}}") 382 | bottom_row.append(f"\\small\\fontfamily{{cmtt}}\\selectfont{{{' '.join(parts[1:])}}}") 383 | else: 384 | top_row.append(f"\\small\\fontfamily{{cmtt}}\\selectfont{{{header}}}") 385 | bottom_row.append('') 386 | return ' & '.join(top_row) + ' \\\\', ' & '.join(bottom_row) + ' \\\\' 387 | 388 | def format_index(idx): 389 | def replace(s): 390 | return s.replace('classical ', '') 391 | if isinstance(idx, tuple): 392 | return replace(' '.join(idx)) 393 | return replace(idx) 394 | 395 | def camel_style_with_dash(s): 396 | def format_word(word): 397 | if len(word) == 3: 398 | return word.upper() 399 | return word.capitalize() 400 | 401 | words = s.split('-') 402 | return '-'.join(format_word(word) for word in words) 403 | 404 | latex_lines = [ 405 | "\\begin{longtable}{" + "l" * (styled_df.index.nlevels) + "r" * (len(styled_df.columns)) + "}", 406 | "\\caption{" + caption + "} \\label{tab:" + label + "} \\\\", 407 | "\\toprule" 408 | ] 409 | 410 | top_header, bottom_header = format_header(styled_df.columns) 411 | latex_lines.extend([ 412 | "&" * styled_df.index.nlevels + " " + top_header, 413 | "&" * styled_df.index.nlevels + " " + bottom_header + " \\endfirsthead", 414 | "\\multicolumn{" + str(styled_df.index.nlevels + len(styled_df.columns)) + "}{c}{\\tablename\\ \\thetable\\ -- continued from previous page} \\\\", # noqa 415 | "\\toprule", 416 | "&" * styled_df.index.nlevels + " " + top_header, 417 | "&" * styled_df.index.nlevels + " " + bottom_header + " \\endhead", 418 | "\\midrule \\multicolumn{" + str(styled_df.index.nlevels + len(styled_df.columns)) + "}{r}{Continued on next page} \\\\ \\endfoot", 419 | "\\bottomrule \\endlastfoot", 420 | "\\midrule" 421 | ]) 422 | 423 | prev_model = None 424 | model_row_count = 0 425 | for i, (idx, row) in enumerate(styled_df.iterrows()): 426 | cell_color = "\\cellcolor{gray!15}" if i % 2 == 0 else "" 427 | row_values = [] 428 | for value in row: 429 | color = extract_color(value) 430 | numeric_value = float(value.split('>')[1].split('<')[0]) 431 | latex_color = rgb_to_latex_color(color) 432 | formatted_value = format_number(numeric_value) 433 | row_values.append(f"{cell_color}{latex_color}{formatted_value}") 434 | 435 | if isinstance(idx, tuple): 436 | model, variant = idx 437 | if model != prev_model: 438 | if prev_model is not None: 439 | latex_lines.append("\\midrule") 440 | model_row_count = 1 441 | latex_lines.append(f"\\multirow{{-1}}{{*}}{{\\footnotesize\\fontfamily{{cmtt}}\\selectfont{{{camel_style_with_dash(format_index(model))}}}}} & {cell_color}\\footnotesize\\fontfamily{{cmtt}}\\selectfont{{{camel_style_with_dash(format_index(variant))}}} & " + " & ".join(row_values) + " \\\\") # noqa 442 | prev_model = model 443 | else: 444 | model_row_count += 1 445 | latex_lines.append( 446 | f"& {cell_color}\\footnotesize\\fontfamily{{cmtt}}\\selectfont{{{camel_style_with_dash(format_index(variant))}}} & " 447 | + " & ".join(row_values) 448 | + " \\\\" 449 | ) 450 | else: 451 | latex_lines.append( 452 | f"\\footnotesize\\fontfamily{{cmtt}}\\selectfont{{{camel_style_with_dash(format_index(idx))}}} & {cell_color}" 453 | + " & ".join(row_values) 454 | + " \\\\" 455 | ) 456 | 457 | latex_lines.append("\\end{longtable}") 458 | 459 | return "\n".join(latex_lines) 460 | 461 | 462 | def load_results(result_fn, raw=False, postprocess_func: callable = None): 463 | """ 464 | Load and process results from a result JSON lines file. 465 | 466 | Parameters 467 | ---------- 468 | result_fn : str 469 | The filename of the JSON lines file containing the results. 470 | raw : bool, optional 471 | If True, return raw JSON objects. If False, parse the response 472 | and convert it to a vector. Default is False. 473 | postprocess_func : callable, optional 474 | A function to postprocess the results (e.g., scaling down). Default is None. 475 | 476 | Returns 477 | ------- 478 | list 479 | A list of processed results. Each item is either a raw JSON object 480 | or a vector representation of anomalies, depending on the 481 | `raw` parameter. 482 | 483 | Notes 484 | ----- 485 | The function attempts to parse each line in the file. If parsing fails, 486 | it appends an empty vector to the results. 487 | 488 | Raises 489 | ------ 490 | FileNotFoundError 491 | If the specified file does not exist. 492 | JSONDecodeError 493 | If a line in the file is not valid JSON. 494 | """ 495 | import json 496 | import pandas as pd 497 | from utils import parse_output, interval_to_vector 498 | 499 | if postprocess_func is None: 500 | postprocess_func = lambda x: x 501 | 502 | with open(result_fn, 'r') as f: 503 | results = [] 504 | for line in f: 505 | info = json.loads(line) 506 | if raw: 507 | results.append(info) 508 | else: 509 | try: 510 | response_parsed = parse_output(postprocess_func(info['response'])) 511 | results.append(interval_to_vector(response_parsed)) 512 | except Exception: 513 | results.append(None) 514 | continue 515 | 516 | return results 517 | 518 | 519 | def collect_results(directory, raw=False, ignore=[]): 520 | """ 521 | Collect and process results from JSON lines files in a directory. 522 | 523 | Parameters 524 | ---------- 525 | directory : str 526 | The path to the directory containing the JSON lines files. 527 | raw : bool, optional 528 | If True, return raw JSON objects. If False, parse the responses. 529 | Default is False. 530 | ignore: list[str], optional 531 | Skip folders containing these names. Default is an empty list. 532 | 533 | Returns 534 | ------- 535 | dict 536 | A dictionary where keys are model names with variants, and values 537 | are lists of processed results from each file. 538 | 539 | Notes 540 | ----- 541 | This function walks through the given directory, processing each 542 | `.jsonl` file except those with 'requests' in the filename. It uses 543 | the directory name as the model name and the filename (sans extension) 544 | as the variant. 545 | 546 | Raises 547 | ------ 548 | FileNotFoundError 549 | If the specified directory does not exist. 550 | """ 551 | import os 552 | from config import postprocess_configs 553 | 554 | results = {} 555 | config = postprocess_configs() 556 | for root, _, files in os.walk(directory): 557 | for file in files: 558 | skip = False 559 | for ignore_folder in ignore: 560 | if ignore_folder in root: 561 | skip = True 562 | break 563 | if skip: 564 | continue 565 | if 'requests' not in file and file.endswith('.jsonl'): 566 | model_name = os.path.basename(root) 567 | variant = file.replace('.jsonl', '') 568 | if variant in config: 569 | pf = config[variant] 570 | else: 571 | pf = None 572 | result_fn = os.path.join(root, file) 573 | model_key = f'{model_name} ({variant})' 574 | results[model_key] = load_results(result_fn, raw=raw, postprocess_func=pf) 575 | return results 576 | 577 | 578 | def EDA(eval_dataset): 579 | total_anom = 0 580 | total = 0 581 | time_series_without_anomalies = 0 582 | anomaly_counts = [] 583 | anomaly_lengths = [] 584 | 585 | for i in range(400): 586 | data = eval_dataset[i] 587 | series_anom = 0 588 | series_anomaly_count = 0 589 | 590 | for start, end in data[0][0]: 591 | length = end - start 592 | series_anom += length 593 | series_anomaly_count += 1 594 | anomaly_lengths.append(length) 595 | 596 | total_anom += series_anom 597 | total += 1000 598 | 599 | if series_anom == 0: 600 | time_series_without_anomalies += 1 601 | 602 | anomaly_counts.append(series_anomaly_count) 603 | 604 | # Calculate statistics 605 | avg_anomaly_ratio = total_anom / total 606 | percent_without_anomalies = (time_series_without_anomalies / 400) * 100 607 | avg_anomalies_per_series = sum(anomaly_counts) / 400 608 | max_anomalies_in_series = max(anomaly_counts) 609 | avg_anomaly_length = sum(anomaly_lengths) / len(anomaly_lengths) 610 | max_anomaly_length = max(anomaly_lengths) 611 | 612 | print(f"Average anomaly ratio: {avg_anomaly_ratio:.4f}") 613 | print(f"Number of time series without anomalies: {time_series_without_anomalies}") 614 | print(f"Percentage of time series without anomalies: {percent_without_anomalies:.2f}%") 615 | print(f"Average number of anomalies per time series: {avg_anomalies_per_series:.2f}") 616 | print(f"Maximum number of anomalies in a single time series: {max_anomalies_in_series}") 617 | print(f"Average length of an anomaly: {avg_anomaly_length:.2f}") 618 | print(f"Maximum length of an anomaly: {max_anomaly_length}") 619 | 620 | 621 | if __name__ == '__main__': 622 | from data.synthetic import SyntheticDataset 623 | 624 | for name in ['point', 'range', 'trend', 'freq', 'noisy-point', 'noisy-trend', 'noisy-freq']: 625 | print(f"Dataset: {name}") 626 | eval_dataset = SyntheticDataset(f'data/synthetic/{name}/eval/') 627 | eval_dataset.load() 628 | EDA(eval_dataset) 629 | -------------------------------------------------------------------------------- /synthesize.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | python src/data/synthetic.py --generate --data_dir data/synthetic/range/eval --synthetic_func synthetic_dataset_with_out_of_range_anomalies --seed 42 3 | python src/data/synthetic.py --generate --data_dir data/synthetic/range/train --synthetic_func synthetic_dataset_with_out_of_range_anomalies --seed 3407 4 | 5 | python src/data/synthetic.py --generate --data_dir data/synthetic/point/eval --synthetic_func synthetic_dataset_with_point_anomalies --seed 42 6 | python src/data/synthetic.py --generate --data_dir data/synthetic/point/train --synthetic_func synthetic_dataset_with_point_anomalies --seed 3407 7 | 8 | python src/data/synthetic.py --generate --data_dir data/synthetic/freq/eval --synthetic_func synthetic_dataset_with_frequency_anomalies --seed 42 9 | python src/data/synthetic.py --generate --data_dir data/synthetic/freq/train --synthetic_func synthetic_dataset_with_frequency_anomalies --seed 3407 10 | 11 | python src/data/synthetic.py --generate --data_dir data/synthetic/trend/eval --synthetic_func synthetic_dataset_with_trend_anomalies --seed 42 12 | python src/data/synthetic.py --generate --data_dir data/synthetic/trend/train --synthetic_func synthetic_dataset_with_trend_anomalies --seed 3407 13 | 14 | python src/data/synthetic.py --generate --data_dir data/synthetic/flat-trend/eval --synthetic_func synthetic_dataset_with_flat_trend_anomalies --seed 42 15 | python src/data/synthetic.py --generate --data_dir data/synthetic/flat-trend/train --synthetic_func synthetic_dataset_with_flat_trend_anomalies --seed 3407 16 | 17 | python src/data/synthetic.py --generate --add_noise --data_dir data/synthetic/noisy-point/eval --synthetic_func synthetic_dataset_with_point_anomalies --seed 42 18 | python src/data/synthetic.py --generate --add_noise --data_dir data/synthetic/noisy-point/train --synthetic_func synthetic_dataset_with_point_anomalies --seed 3407 19 | 20 | python src/data/synthetic.py --generate --add_noise --data_dir data/synthetic/noisy-freq/eval --synthetic_func synthetic_dataset_with_frequency_anomalies --seed 42 21 | python src/data/synthetic.py --generate --add_noise --data_dir data/synthetic/noisy-freq/train --synthetic_func synthetic_dataset_with_frequency_anomalies --seed 3407 22 | 23 | python src/data/synthetic.py --generate --add_noise --data_dir data/synthetic/noisy-trend/eval --synthetic_func synthetic_dataset_with_trend_anomalies --seed 42 24 | python src/data/synthetic.py --generate --add_noise --data_dir data/synthetic/noisy-trend/train --synthetic_func synthetic_dataset_with_trend_anomalies --seed 3407 25 | -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # data=("trend" "range" "point" "freq" "noisy-point" "noisy-freq" "noisy-trend") 4 | 5 | # data=("trend" "range" "point" "freq") 6 | # variants=( 7 | # "0shot-text-s0.3-calc" 8 | # "0shot-text-s0.3-dyscalc" 9 | # "1shot-vision-calc" 10 | # "1shot-vision-dyscalc" 11 | # "0shot-vision-calc" 12 | # "0shot-vision-dyscalc" 13 | # ) 14 | 15 | data=("flat-trend") 16 | variants=( 17 | "1shot-vision" "0shot-vision" "1shot-text-s0.3" "0shot-text-s0.3" "0shot-text" \ 18 | "0shot-text-s0.3-cot" "1shot-text-s0.3-cot" "0shot-vision-cot" "1shot-vision-cot" 19 | "0shot-text-s0.3-csv" "0shot-text-s0.3-cot-csv" 20 | "0shot-text-s0.3-tpd" "0shot-text-s0.3-cot-tpd" 21 | "0shot-text-s0.3-pap" "0shot-text-s0.3-cot-pap" 22 | ) 23 | 24 | # models=("internvlm-76b") 25 | 26 | models=("gpt-4o-mini") 27 | for model in "${models[@]}"; do 28 | for datum in "${data[@]}"; do 29 | for variant in "${variants[@]}"; do 30 | session_name="${datum}_${model}_${variant}" 31 | command="python src/batch_api.py --data $datum --model $model --variant $variant" 32 | echo "Runing \`$command\` ..." 33 | tmux new-session -d -s "$session_name" "$command" 34 | done 35 | done 36 | done 37 | 38 | # variants=( 39 | # "0shot-vision" "1shot-vision" "0shot-text-s0.3" "0shot-text" \ 40 | # "0shot-text-s0.3-cot" "1shot-text-s0.3-cot" "0shot-vision-cot" "1shot-vision-cot" 41 | # "0shot-text-s0.3-csv" "0shot-text-s0.3-cot-csv" 42 | # "0shot-text-s0.3-tpd" "0shot-text-s0.3-cot-tpd" 43 | # "0shot-text-s0.3-pap" "0shot-text-s0.3-cot-pap" 44 | # ) 45 | 46 | # # variants=("0shot-vision-cot") 47 | 48 | # for model in "${models[@]}"; do 49 | # for datum in "${data[@]}"; do 50 | # for variant in "${variants[@]}"; do 51 | # session_name="${datum}_${model}_${variant}" 52 | # command="python src/online_api.py --data $datum --model $model --variant $variant" 53 | # echo "Runing \`$command\` ..." 54 | # tmux new-session -d -s "$session_name" "$command" 55 | # done 56 | # done 57 | # done 58 | 59 | 60 | ############################################## 61 | # Baselines 62 | ############################################## 63 | 64 | # for datum in "${data[@]}"; do 65 | # session_name="iso_${datum}" 66 | # # Kill the existing session if it exists 67 | # tmux has-session -t "$session_name" 2>/dev/null 68 | # if [ $? -eq 0 ]; then 69 | # tmux kill-session -t "$session_name" 70 | # fi 71 | # command="python src/baselines/isoforest.py --data $datum --model isolation-forest" 72 | # tmux new-session -d -s "$session_name" "$command" 73 | # done 74 | 75 | 76 | 77 | 78 | 79 | # tmux list-sessions -F '#{session_name}' | grep internvlm | xargs -I {} tmux kill-session -t {} 80 | # find results/synthetic -type d -name "text" -exec rm -rf {} + -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Rose-STL-Lab/AnomLLM/bf7827dfd63e5cd134f5b2412efc54175e5aee72/tests/__init__.py --------------------------------------------------------------------------------