├── README.md
├── figures
    ├── all_tools.png
    ├── dataset_examples.png
    ├── dataset_gen.png
    └── dataset_stats.png
├── mnms
    ├── __init__.py
    ├── agent.py
    ├── constants.py
    ├── evaluation
    │   ├── __init__.py
    │   ├── __pycache__
    │   │   ├── __init__.cpython-39.pyc
    │   │   ├── evaluator.cpython-39.pyc
    │   │   ├── metrics.cpython-39.pyc
    │   │   └── run.cpython-39.pyc
    │   ├── evaluator.py
    │   ├── metrics.py
    │   ├── run.py
    │   └── run_final_result_eval.py
    ├── execution
    │   ├── __init__.py
    │   ├── config.py
    │   ├── data
    │   │   ├── 06483.jpg
    │   │   ├── 07115.jpg
    │   │   ├── 07600.jpg
    │   │   ├── 08078.jpg
    │   │   ├── 08312.jpg
    │   │   ├── 08773.jpg
    │   │   ├── 08788.jpg
    │   │   ├── 09531.jpg
    │   │   ├── 09742.jpg
    │   │   ├── 09844.jpg
    │   │   ├── 09966.jpg
    │   │   ├── 100081-input.png
    │   │   ├── 10050.jpg
    │   │   ├── 100558-input.png
    │   │   ├── 10084.jpg
    │   │   ├── 10261.jpg
    │   │   ├── 10358.jpg
    │   │   ├── 10401.jpg
    │   │   ├── 10404.jpg
    │   │   ├── 10573.jpg
    │   │   ├── 10899.jpg
    │   │   ├── 11120.jpg
    │   │   ├── 11123.jpg
    │   │   ├── 11135.jpg
    │   │   ├── 111376-input.png
    │   │   ├── 11170.jpg
    │   │   ├── 11226.jpg
    │   │   ├── 11409.jpg
    │   │   ├── 11427.jpg
    │   │   ├── 11534.jpg
    │   │   ├── 11561.jpg
    │   │   ├── 1159654.jpg
    │   │   ├── 11665.jpg
    │   │   ├── 11674.jpg
    │   │   ├── 11705.jpg
    │   │   ├── 11706.jpg
    │   │   ├── 11807.jpg
    │   │   ├── 11899.jpg
    │   │   ├── 12300.jpg
    │   │   ├── 12381.jpg
    │   │   ├── 12552.jpg
    │   │   ├── 12603.jpg
    │   │   ├── 12633.jpg
    │   │   ├── 1284-1180-0003.flac
    │   │   ├── 12904.jpg
    │   │   ├── 12946.jpg
    │   │   ├── 12955.jpg
    │   │   ├── 13181.jpg
    │   │   ├── 13393.jpg
    │   │   ├── 13426.jpg
    │   │   ├── 134597-input.png
    │   │   ├── 13480.jpg
    │   │   ├── 13538.jpg
    │   │   ├── 13644.jpg
    │   │   ├── 13682.jpg
    │   │   ├── 14021.jpg
    │   │   ├── 14043.jpg
    │   │   ├── 14095.jpg
    │   │   ├── 14098.jpg
    │   │   ├── 14168.jpg
    │   │   ├── 14208.jpg
    │   │   ├── 14306.jpg
    │   │   ├── 14317.jpg
    │   │   ├── 14382.jpg
    │   │   ├── 14450.jpg
    │   │   ├── 14496.jpg
    │   │   ├── 14514.jpg
    │   │   ├── 14553.jpg
    │   │   ├── 147546-input.png
    │   │   ├── 14757.jpg
    │   │   ├── 14791.jpg
    │   │   ├── 14799.jpg
    │   │   ├── 14809.jpg
    │   │   ├── 15021.jpg
    │   │   ├── 150517.jpg
    │   │   ├── 153639-input.png
    │   │   ├── 15387.jpg
    │   │   ├── 15491.jpg
    │   │   ├── 15595.jpg
    │   │   ├── 15610.jpg
    │   │   ├── 15653-input.png
    │   │   ├── 15764.jpg
    │   │   ├── 15792.jpg
    │   │   ├── 1592294.jpg
    │   │   ├── 1592452.jpg
    │   │   ├── 16148.jpg
    │   │   ├── 16273.jpg
    │   │   ├── 16386.jpg
    │   │   ├── 16611.jpg
    │   │   ├── 16657.jpg
    │   │   ├── 16749.jpg
    │   │   ├── 16942.jpg
    │   │   ├── 16986.jpg
    │   │   ├── 17042.jpg
    │   │   ├── 17320-input.png
    │   │   ├── 17412.jpg
    │   │   ├── 17464.jpg
    │   │   ├── 17716.jpg
    │   │   ├── 177572-input.png
    │   │   ├── 17862.jpg
    │   │   ├── 18104.jpg
    │   │   ├── 18225.jpg
    │   │   ├── 18404.jpg
    │   │   ├── 18535.jpg
    │   │   ├── 19027.jpg
    │   │   ├── 19068.jpg
    │   │   ├── 19120.jpg
    │   │   ├── 19358.jpg
    │   │   ├── 19929.jpg
    │   │   ├── 20864.jpg
    │   │   ├── 21203.jpg
    │   │   ├── 21740.jpg
    │   │   ├── 21803.jpg
    │   │   ├── 22638.jpg
    │   │   ├── 2300-131720-0000.flac
    │   │   ├── 2315602.jpg
    │   │   ├── 2315924.jpg
    │   │   ├── 2316209.jpg
    │   │   ├── 2316316.jpg
    │   │   ├── 2317756.jpg
    │   │   ├── 2317766.jpg
    │   │   ├── 2318145.jpg
    │   │   ├── 2318392.jpg
    │   │   ├── 2319267.jpg
    │   │   ├── 2320104.jpg
    │   │   ├── 2321508.jpg
    │   │   ├── 2321647.jpg
    │   │   ├── 2321676.jpg
    │   │   ├── 2321930.jpg
    │   │   ├── 2322211.jpg
    │   │   ├── 2322245.jpg
    │   │   ├── 2322902.jpg
    │   │   ├── 2323530.jpg
    │   │   ├── 2324260.jpg
    │   │   ├── 2324812.jpg
    │   │   ├── 2326465.jpg
    │   │   ├── 2327107.jpg
    │   │   ├── 2327921.jpg
    │   │   ├── 2327939.jpg
    │   │   ├── 2328271.jpg
    │   │   ├── 2328879.jpg
    │   │   ├── 2329245.jpg
    │   │   ├── 2329676.jpg
    │   │   ├── 2330591.jpg
    │   │   ├── 2330610.jpg
    │   │   ├── 2330986.jpg
    │   │   ├── 2332593.jpg
    │   │   ├── 2332697.jpg
    │   │   ├── 2333456.jpg
    │   │   ├── 2333829.jpg
    │   │   ├── 2334315.jpg
    │   │   ├── 2334677.jpg
    │   │   ├── 2335269.jpg
    │   │   ├── 2336649.jpg
    │   │   ├── 2336917.jpg
    │   │   ├── 2337344.jpg
    │   │   ├── 2338193.jpg
    │   │   ├── 2338250.jpg
    │   │   ├── 2340480.jpg
    │   │   ├── 2340962.jpg
    │   │   ├── 2341610.jpg
    │   │   ├── 2342830.jpg
    │   │   ├── 2342847.jpg
    │   │   ├── 2343942.jpg
    │   │   ├── 2344704.jpg
    │   │   ├── 2344819.jpg
    │   │   ├── 2345745.jpg
    │   │   ├── 2346590.jpg
    │   │   ├── 2346651.jpg
    │   │   ├── 2347052.jpg
    │   │   ├── 2347302.jpg
    │   │   ├── 2347441.jpg
    │   │   ├── 2348288.jpg
    │   │   ├── 2348943.jpg
    │   │   ├── 2349334.jpg
    │   │   ├── 2349496.jpg
    │   │   ├── 2350021.jpg
    │   │   ├── 2350057.jpg
    │   │   ├── 2350119.jpg
    │   │   ├── 2350499.jpg
    │   │   ├── 2351336.jpg
    │   │   ├── 2351464.jpg
    │   │   ├── 2353266.jpg
    │   │   ├── 2354281.jpg
    │   │   ├── 2354285.jpg
    │   │   ├── 2354461.jpg
    │   │   ├── 2354497.jpg
    │   │   ├── 2355794.jpg
    │   │   ├── 2356131.jpg
    │   │   ├── 2356182.jpg
    │   │   ├── 2356736.jpg
    │   │   ├── 2357888.jpg
    │   │   ├── 2358679.jpg
    │   │   ├── 2359351.jpg
    │   │   ├── 2359509.jpg
    │   │   ├── 2359565.jpg
    │   │   ├── 2362052.jpg
    │   │   ├── 2362289.jpg
    │   │   ├── 2364151.jpg
    │   │   ├── 2364622.jpg
    │   │   ├── 2365156.jpg
    │   │   ├── 2365901.jpg
    │   │   ├── 2366358.jpg
    │   │   ├── 2366671.jpg
    │   │   ├── 2368356.jpg
    │   │   ├── 237-134500-0000.flac
    │   │   ├── 2370112.jpg
    │   │   ├── 2370590.jpg
    │   │   ├── 2371415.jpg
    │   │   ├── 2371677.jpg
    │   │   ├── 2372348.jpg
    │   │   ├── 2372497.jpg
    │   │   ├── 2372928.jpg
    │   │   ├── 2373927.jpg
    │   │   ├── 2374322.jpg
    │   │   ├── 2374973.jpg
    │   │   ├── 2375698.jpg
    │   │   ├── 2376309.jpg
    │   │   ├── 2377004.jpg
    │   │   ├── 2377385.jpg
    │   │   ├── 2378137.jpg
    │   │   ├── 2378292.jpg
    │   │   ├── 2378563.jpg
    │   │   ├── 2379887.jpg
    │   │   ├── 2380325.jpg
    │   │   ├── 2380691.jpg
    │   │   ├── 2381704.jpg
    │   │   ├── 2382560.jpg
    │   │   ├── 2382602.jpg
    │   │   ├── 2384900.jpg
    │   │   ├── 2385634.jpg
    │   │   ├── 2387080.jpg
    │   │   ├── 2387947.jpg
    │   │   ├── 2388803.jpg
    │   │   ├── 2390981.jpg
    │   │   ├── 2393510.jpg
    │   │   ├── 23936.jpg
    │   │   ├── 2393797.jpg
    │   │   ├── 2394610.jpg
    │   │   ├── 2394635.jpg
    │   │   ├── 2394828.jpg
    │   │   ├── 2395343.jpg
    │   │   ├── 2395951.jpg
    │   │   ├── 2396295.jpg
    │   │   ├── 2397301.jpg
    │   │   ├── 2398150.jpg
    │   │   ├── 2400761.jpg
    │   │   ├── 2401001.jpg
    │   │   ├── 2401028.jpg
    │   │   ├── 2401271.jpg
    │   │   ├── 2401298.jpg
    │   │   ├── 2401498.jpg
    │   │   ├── 2401635.jpg
    │   │   ├── 2402643.jpg
    │   │   ├── 2406320.jpg
    │   │   ├── 2406339.jpg
    │   │   ├── 2407124.jpg
    │   │   ├── 2407807.jpg
    │   │   ├── 2408984.jpg
    │   │   ├── 2409068.jpg
    │   │   ├── 2409304.jpg
    │   │   ├── 2409579.jpg
    │   │   ├── 2409782.jpg
    │   │   ├── 2409896.jpg
    │   │   ├── 2410989.jpg
    │   │   ├── 2411180.jpg
    │   │   ├── 2413340.jpg
    │   │   ├── 2413350.jpg
    │   │   ├── 2415263.jpg
    │   │   ├── 2416088.jpg
    │   │   ├── 2416387.jpg
    │   │   ├── 2416914.jpg
    │   │   ├── 2416928.jpg
    │   │   ├── 270650-input.png
    │   │   ├── 282572-input.png
    │   │   ├── 286059.jpg
    │   │   ├── 291614-input.png
    │   │   ├── 319096-input.png
    │   │   ├── 3575-170457-0001.flac
    │   │   ├── 357903-input.png
    │   │   ├── 360871-input.png
    │   │   ├── 362944-input.png
    │   │   ├── 398575-input.png
    │   │   ├── 407550-input.png
    │   │   ├── 417814-input.png
    │   │   ├── 417830-input.png
    │   │   ├── 4446-2271-0000.flac
    │   │   ├── 4507-16021-0002.flac
    │   │   ├── 456782-input.png
    │   │   ├── 498121.jpg
    │   │   ├── 510027-input.png
    │   │   ├── 543942-input.png
    │   │   ├── 581668-input.png
    │   │   ├── 61-70968-0000.flac
    │   │   ├── 672-122797-0000.flac
    │   │   ├── 6829-68769-0002.flac
    │   │   ├── 70952-input.png
    │   │   ├── 8224-274384-0000.flac
    │   │   ├── 8455-210777-0002.flac
    │   │   ├── 86582-input.png
    │   │   ├── 908-157963-0001.flac
    │   │   ├── COCO_train2014_000000028742.jpg
    │   │   ├── COCO_train2014_000000113236.jpg
    │   │   ├── COCO_train2014_000000134502.jpg
    │   │   ├── COCO_train2014_000000256855.jpg
    │   │   ├── COCO_train2014_000000306393.jpg
    │   │   └── ocr_temp.jpg
    │   ├── executor.py
    │   ├── run.py
    │   ├── tool_api.py
    │   └── utils.py
    ├── parser.py
    ├── prompt.py
    ├── run_plan_agent.py
    ├── tool_graph.pkl
    └── verifier.py
└── requirements.txt


/README.md:
--------------------------------------------------------------------------------
  1 | # m&m's: A Benchmark to Evaluate Tool-Use Agents for multi-step multi-modal Tasks
  2 | [**🌐 Website**](https://mnms-project.github.io/) | [**🤗 Dataset**](https://huggingface.co/datasets/zixianma/mms) | [**📖 Paper**](https://arxiv.org/abs/2403.11085) 
  3 | 
  4 | m&ms is a benchmark for evaluating large language model (LLM) agents' tool-use abilities on multi-step multi-modal tasks.
  5 | 
  6 | ## Dataset examples
  7 | <img src="figures/dataset_examples.png">
  8 | 
  9 | ## Dataset details
 10 | This dataset contains 4K+ multi-step multi-modal tasks involving 33 tools that include 13 multi-modal models, 9 (free) public APIs, and 11 image processing modules. For each of these task queries, we provide automatically generated plans using this realistic toolset. We further provide a high-quality subset of 1,565 human-verified task plans and 882 human-verified, filtered, and correctly executable plans. Below is a table summarizing the statistics of m&ms:
 11 | <p align="center">
 12 |   <img src="figures/dataset_stats.png" width="500px">
 13 | </p>
 14 | And a table listing all 33 available tools in our tool API:
 15 | <p align="center">
 16 |   <img src="figures/all_tools.png" width="800px">
 17 | </p>
 18 | 
 19 | 
 20 | 
 21 | ## Dataset generation
 22 | <p align="center">
 23 |   <img src="figures/dataset_gen.png" width="800px">
 24 | </p>
 25 | 
 26 | ## Setup
 27 | ### Installation
 28 | Please make sure you install all the required packages in ```requirements.txt``` by running:
 29 | ```
 30 | pip install -r requirements.txt
 31 | ```
 32 | ### Environment variables
 33 | You can set up your environment variables by running the command lines below line by line or store them locally in a file (e.g. ```.bashrc```)
 34 | ```
 35 | export HUGGINGFACE_HUB_CACHE=<your_cache_dir>
 36 | export RAPID_API_KEY=<your_rapid_key>
 37 | export OMDB_API_KEY=<your_omdb_key>
 38 | export OPENAI_API_KEY=<your_openai_key> 
 39 | export GOOGLE_API_KEY=<your_google_key> (only if you experiment with Google's models)
 40 | export DEEPINFRA_API_KEY=<your_deepinfra_key> (only if you run models hosted on DeepInfra)
 41 | ```
 42 | And then run this line for the changes to take effect:
 43 | ```
 44 | source .bashrc
 45 | ```
 46 | You can generate your own API keys on [Rapid](https://rapidapi.com/), [OMDb](https://www.omdbapi.com/), [OpenAI](https://openai.com/), [Google](https://ai.google.dev/tutorials/quickstart), and [DeepInfra](https://deepinfra.com/).
 47 | 
 48 | ## Planning
 49 | To run planning experiments on m&ms with LLM agents, below is an example command line
 50 | ```
 51 | python -m mnms.run_plan_agent  --action-format json --plan-mode multi-step --model gemini-pro --max-reply 10 --verify --execute --exp-id 0331 
 52 | ```
 53 | where the arguments:
 54 | - ```action-format``` specifies the output format of the LLM agent i.e. ```json``` or ```code```;
 55 | - ```plan-mode``` refers to the planning strategy ```multi-step``` or ```step-by-step```;
 56 | - ```max-reply``` is the max number of turns the LLM agent can take to refine its initial plan (```max-reply=0``` means no iterative refinement);
 57 | - ```verifiy``` and ```execute``` specify whether to turn on verification and execution feedback during planning.
 58 | 
 59 | Additionally, you can also run experiments on your own queries by adding ```--input-file <your_input_file>```, where each example is expected to contain ```id``` and ```user_request```.
 60 | 
 61 | ## Execution
 62 | 
 63 | To execute the groundtruth plans, you can run this line:
 64 | ```
 65 | python -m mnms.execution.run --plan-format code
 66 | ```
 67 | 
 68 | 
 69 | To execute predicted plans, you will need to additionally provide a ```predictions.json``` file like this:
 70 | ```
 71 | python -m mnms.execution.run --plan-format json --input-file <predictions.json>
 72 | ```
 73 | 
 74 | ## Evaluation
 75 | To evaluate the predicted plans against the groundtruth plans in m&ms, simply run this line:
 76 | ```
 77 | python -m mnms.evaluation.run --plan-format json --preds-file <predictions.json> --output-csv <output.csv> 
 78 | ```
 79 | Note that our code works with a ```predictions.json``` file where each line is a dictionary containing "id" and "prediction" like the example below:
 80 | ```
 81 | {"id": 10, "prediction": [{"id": 0, "name": "text classification", "args": {"text": "forced, familiar and thoroughly condescending."}}]}
 82 | {"id": 24, "prediction": [{"id": 0, "name": "text generation", "args": {"text": "Who exactly became the King of the Canary Islands?"}}]}
 83 | ...
 84 | ```
 85 | 
 86 | But you are welcome to modify it to work with other file formats. 
 87 | 
 88 | To evaluate the final execution results against the groundtruth results in m&ms for selected tasks, first download the groundtruth execution results from [here](https://huggingface.co/datasets/zixianma/mnms_exec_results/blob/main/gt_results.zip) and unzip it into ```<gt_results_dir>```. Then, run this line:
 89 | ```
 90 | python -m mnms.evaluation.run_final_result_eval --gt-dir <gt_results_dir>  --pred-dir <pred_results_dir> --output-csv <output.csv> 
 91 | ```
 92 | 
 93 | 
 94 | ## Citation
 95 | If you find our work helpful, please consider starring our repo and citing our paper. Thanks!
 96 | ```
 97 | @article{ma2024mms,
 98 |   title={m&m's: A Benchmark to Evaluate Tool-Use for multi-step multi-modal Tasks}, 
 99 |   author={Zixian Ma and Weikai Huang and Jieyu Zhang and Tanmay Gupta and Ranjay Krishna},
100 |   year={2024},
101 |   journal={EECV 2024},
102 | }
103 | ```
104 | 


--------------------------------------------------------------------------------
/figures/all_tools.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/figures/all_tools.png


--------------------------------------------------------------------------------
/figures/dataset_examples.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/figures/dataset_examples.png


--------------------------------------------------------------------------------
/figures/dataset_gen.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/figures/dataset_gen.png


--------------------------------------------------------------------------------
/figures/dataset_stats.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/figures/dataset_stats.png


--------------------------------------------------------------------------------
/mnms/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/__init__.py


--------------------------------------------------------------------------------
/mnms/agent.py:
--------------------------------------------------------------------------------
  1 | from autogen.agentchat import Agent, UserProxyAgent
  2 | from typing import Dict, Optional, Union
  3 | 
  4 | class MnmsUserAgent(UserProxyAgent):
  5 | 
  6 |     def __init__(
  7 |             self,
  8 |             name,
  9 |             prompt_generator, 
 10 |             feedback_generator,
 11 |             parser,
 12 |             verifier,
 13 |             executor,
 14 |             **config,
 15 |     ):
 16 |         super().__init__(name, **config)
 17 | 
 18 |         self.current_plan = []
 19 |         self.prompt_generator = prompt_generator
 20 |         self.feedback_generator = feedback_generator
 21 |         self.verifier = verifier
 22 |         self.parser = parser
 23 |         self.executor = executor
 24 |         self.current_task_id = None
 25 |         self.feedback_types = []
 26 | 
 27 |     def sender_hits_max_reply(self, sender: Agent):
 28 |         return self._consecutive_auto_reply_counter[sender.name] >= self._max_consecutive_auto_reply
 29 | 
 30 |     def receive(
 31 |             self,
 32 |             message: Union[Dict, str],
 33 |             sender: Agent,
 34 |             request_reply: Optional[bool] = None,
 35 |             silent: Optional[bool] = False,
 36 |     ):
 37 |         """Receive a message from the sender agent.
 38 |         Once a message is received, this function sends a reply to the sender or simply stop.
 39 |         The reply can be generated automatically or entered manually by a human.
 40 |         """
 41 |         print("COUNTER:", self._consecutive_auto_reply_counter[sender.name])
 42 |         self._process_received_message(message, sender, silent)
 43 |         
 44 |         parsed_results = self.parser.parse(message)
 45 |         parsed_content = parsed_results['content']
 46 |         parsed_error_message = parsed_results['message']
 47 |         parsed_status = parsed_results['status']
 48 |         parsed_error_code = parsed_results['error_code']
 49 | 
 50 |         # Terminate planning if parsing fails and termination check returns true
 51 |         # Otherwise proceed to parsing feedback or verification/execution
 52 |         if not parsed_status and self._is_termination_msg(message):
 53 |             return
 54 |         
 55 |         if parsed_status: 
 56 |             # if parsing succeeds, update the current plan to be the newly parsed one
 57 |             self.current_plan = parsed_content
 58 | 
 59 |             if self.verifier:
 60 |                 verified_results = self.verifier.verify(parsed_content)
 61 |                 verified_error_message = verified_results['message']
 62 |                 verified_status = verified_results['status']
 63 |                 verified_error_code = verified_results['error_code']
 64 | 
 65 |                 if verified_status: 
 66 |                     # if verification succeeds
 67 |                     if self.executor:
 68 |                         # go to execution stage if there is an executor module
 69 |                         executed_results = self.executor.execute(self.current_task_id, parsed_content)
 70 |                         executed_error_message = executed_results['message']
 71 |                         executed_status = executed_results['status']
 72 | 
 73 |                         # send the message from running the execution module
 74 |                         reply = self.feedback_generator.get_prompt("execution", executed_status, executed_error_message)
 75 |                         if executed_status:
 76 |                             self.send(reply, sender, request_reply=False) # no need to reply when the global plan is executed correctly
 77 |                             self._consecutive_auto_reply_counter[sender.name] = 0
 78 |                             return
 79 |                         else:
 80 |                             # always check the counter before requesting any reply
 81 |                             if self.sender_hits_max_reply(sender): 
 82 |                                 # reset the consecutive_auto_reply_counter
 83 |                                 self._consecutive_auto_reply_counter[sender.name] = 0
 84 |                                 return
 85 |                         
 86 |                             self._consecutive_auto_reply_counter[sender.name] += 1
 87 |                             self.feedback_types.append("execution")
 88 |                             self.send(reply, sender, request_reply=True)
 89 |                     else:
 90 |                         # if no exeuction module, send the message from verifier
 91 |                         reply = self.feedback_generator.get_prompt("verification", verified_status, verified_error_message, verified_error_code)
 92 |                         self.send(reply, sender, request_reply=False) # request_reply=false because verification is successful
 93 |                         self._consecutive_auto_reply_counter[sender.name] = 0
 94 |                         return
 95 |                 else:
 96 |                     if self.sender_hits_max_reply(sender):
 97 |                         # reset the consecutive_auto_reply_counter
 98 |                         self._consecutive_auto_reply_counter[sender.name] = 0
 99 |                         return
100 |                     
101 |                     # if verification fails, construct a feedback message from the error code and message of the verifier
102 |                     # send the feedback message, and request a reply
103 |                     self._consecutive_auto_reply_counter[sender.name] += 1
104 |                     reply = self.feedback_generator.get_prompt("verification", verified_status, verified_error_message, verified_error_code)
105 |                     self.feedback_types.append("verification")
106 |                     self.send(reply, sender, request_reply=True)
107 |             else:
108 |                 if self.executor: 
109 |                     # if verifier is NOT specified but executor is, run the execution module
110 |                     executed_results = self.executor.execute(self.current_task_id, parsed_content)
111 |                     executed_error_message = executed_results['message']
112 |                     executed_status = executed_results['status']
113 | 
114 |                     # send the message from running the execution module
115 |                     reply = self.feedback_generator.get_prompt("execution", executed_status, executed_error_message)
116 |                     if executed_status:
117 |                         self.send(reply, sender, request_reply=False)
118 |                         self._consecutive_auto_reply_counter[sender.name] = 0
119 |                         return
120 |                     else:
121 |                         if self.sender_hits_max_reply(sender):
122 |                             # reset the consecutive_auto_reply_counter
123 |                             self._consecutive_auto_reply_counter[sender.name] = 0
124 |                             return
125 |                         
126 |                         self._consecutive_auto_reply_counter[sender.name] += 1
127 |                         self.feedback_types.append("execution")
128 |                         self.send(reply, sender, request_reply=True)
129 |                 else:
130 |                     # if parsing succeeds, and neither verifier nor executor is specified, simply return
131 |                     reply = self.feedback_generator.get_prompt("parsing", parsed_status, parsed_error_message, parsed_error_code)
132 |                     self.send(reply, sender, request_reply=False)
133 |                     self._consecutive_auto_reply_counter[sender.name] = 0
134 |                     return
135 |         else:
136 |             if self.sender_hits_max_reply(sender):
137 |                 # reset the consecutive_auto_reply_counter
138 |                 self._consecutive_auto_reply_counter[sender.name] = 0
139 |                 return
140 | 
141 |             # if parsing fails, construct a feedback message from the error code and message of the parser
142 |             # send the feedback message, and request a reply
143 |             self._consecutive_auto_reply_counter[sender.name] += 1
144 |             reply = self.feedback_generator.get_prompt("parsing", parsed_status, parsed_error_message, parsed_error_code)
145 |             self.feedback_types.append("parsing")
146 |             self.send(reply, sender, request_reply=True)
147 | 
148 |     def generate_init_message(self, query):
149 |         content = self.prompt_generator.get_prompt_for_curr_query(query)
150 |         return content
151 | 
152 |     def initiate_chat(self, assistant, message, task_id, log_prompt_only=False):
153 |         self.current_task_id = task_id
154 |         self.current_plan = []
155 |         self.feedback_types = []
156 |         initial_message = self.generate_init_message(message)
157 |         if log_prompt_only:
158 |             print(initial_message)
159 |         else:
160 |             assistant.receive(initial_message, self, request_reply=True)
161 |             
162 |             # get the plan
163 |             plan = self.current_plan
164 |             print("\nFINAL plan:", plan)
165 |         
166 | 
167 | 
168 | class MnmsUserAgentLocal(MnmsUserAgent):
169 |     def receive(
170 |             self,
171 |             message: Union[Dict, str],
172 |             sender: Agent,
173 |             request_reply: Optional[bool] = None,
174 |             silent: Optional[bool] = False,
175 |     ):
176 |         """Receive a message from the sender agent.
177 |         Once a message is received, this function sends a reply to the sender or simply stop.
178 |         The reply can be generated automatically or entered manually by a human.
179 |         """
180 |         print("COUNTER:", self._consecutive_auto_reply_counter[sender.name])
181 |         self._process_received_message(message, sender, silent)
182 |         
183 |         parsed_results = self.parser.parse(message)
184 |         parsed_content = parsed_results['content']
185 |         parsed_status = parsed_results['status']
186 |         parsed_error_message = parsed_results['message']
187 |         parsed_error_code = parsed_results['error_code']
188 | 
189 |         # Terminate planning if parsing fails and termination check returns true
190 |         # Otherwise proceed to parsing feedback or verification/execution
191 |         if not parsed_status and self._is_termination_msg(message):
192 |             return
193 | 
194 |         if parsed_status: 
195 |             # if parsing succeeds
196 |             if self.verifier:
197 |                 # verify a local step
198 |                 verified_results = self.verifier.verify_single_node(parsed_content)
199 |                 verified_error_message = verified_results['message']
200 |                 verified_status = verified_results['status']
201 |                 verified_error_code = verified_results['error_code']
202 | 
203 |                 if verified_status: 
204 |                     # if verification succeeds
205 |                     if self.executor:
206 |                         # go to execution stage if there is an executor module
207 |                         executed_results = self.executor.execute(self.current_task_id, parsed_content)
208 |                         executed_error_message = executed_results['message']
209 |                         executed_status = executed_results['status']
210 |                         # if this step executes successfully, adds it to the current plan
211 |                         if executed_status:
212 |                             self.current_plan.append(parsed_content)
213 | 
214 |                         if self.sender_hits_max_reply(sender) or self._is_termination_msg(message):
215 |                             # reset the consecutive_auto_reply_counter
216 |                             self._consecutive_auto_reply_counter[sender.name] = 0
217 |                             return
218 |                         
219 |                         # send the message from running the execution module
220 |                         self._consecutive_auto_reply_counter[sender.name] += 1
221 |                         reply = self.feedback_generator.get_prompt("execution", executed_status, executed_error_message) 
222 |                         if not executed_status:
223 |                             self.feedback_types.append("execution")
224 |                         # always request a reply for 1) generating the next step if execution succeeds
225 |                         # or 2) fixing this current step if execution fails
226 |                         self.send(reply, sender, request_reply=True)
227 |                     else:
228 |                         # if verification succeeds and no exeuctor is specified, consider it a correct step and add it to plan
229 |                         self.current_plan.append(parsed_content)
230 | 
231 |                         if self.sender_hits_max_reply(sender) or self._is_termination_msg(message):
232 |                             # reset the consecutive_auto_reply_counter
233 |                             self._consecutive_auto_reply_counter[sender.name] = 0
234 |                             return
235 |                         
236 |                         self._consecutive_auto_reply_counter[sender.name] += 1
237 |                         # if no exeuction module, send the message from verifier
238 |                         reply = self.feedback_generator.get_prompt("verification", verified_status, verified_error_message, verified_error_code) 
239 |                         # request a reply for generating the next step
240 |                         self.send(reply, sender, request_reply=True)
241 |                 else:
242 |                     if self.sender_hits_max_reply(sender):
243 |                         # reset the consecutive_auto_reply_counter
244 |                         self._consecutive_auto_reply_counter[sender.name] = 0
245 |                         return
246 |                     
247 |                     # if verification fails, construct a feedback message from the error code and message of the verifier
248 |                     # send the feedback message, and request a reply to fix the current step
249 |                     self._consecutive_auto_reply_counter[sender.name] += 1
250 |                     reply = self.feedback_generator.get_prompt("verification", verified_status, verified_error_message, verified_error_code)
251 |                     self.feedback_types.append("verification")
252 |                     self.send(reply, sender, request_reply=True)
253 |             else:
254 |                 if self.executor:
255 |                     # if verifier is NOT specified but executor is, run the execution module
256 |                     executed_results = self.executor.execute(self.current_task_id, parsed_content)
257 |                     executed_error_message = executed_results['message']
258 |                     executed_status = executed_results['status']
259 |                     # if this step executes successfully, adds it to the current plan
260 |                     if executed_status:
261 |                         self.current_plan.append(parsed_content)
262 | 
263 |                     if self.sender_hits_max_reply(sender) or self._is_termination_msg(message):
264 |                         # reset the consecutive_auto_reply_counter
265 |                         self._consecutive_auto_reply_counter[sender.name] = 0
266 |                         return
267 |                     
268 |                     # send the message from running the execution module
269 |                     self._consecutive_auto_reply_counter[sender.name] += 1
270 |                     reply = self.feedback_generator.get_prompt("execution", executed_status, executed_error_message) 
271 |                     if not executed_status:
272 |                         self.feedback_types.append("execution")
273 |                     # always request a reply for 1) generating the next step if execution succeeds
274 |                     # or 2) fixing this current step if execution fails
275 |                     self.send(reply, sender, request_reply=True)
276 |                 else:
277 |                     # if parsing succeeds, and neither verifier nor executor is specified, add current step to plan
278 |                     # request a reply for generating the next step 
279 |                     self.current_plan.append(parsed_content)
280 | 
281 |                     if self.sender_hits_max_reply(sender) or self._is_termination_msg(message):
282 |                         # reset the consecutive_auto_reply_counter
283 |                         self._consecutive_auto_reply_counter[sender.name] = 0
284 |                         return
285 |                     
286 |                     self._consecutive_auto_reply_counter[sender.name] += 1
287 |                     reply = self.feedback_generator.get_prompt("parsing", parsed_status, parsed_error_message, parsed_error_code)
288 |                     self.send(reply, sender, request_reply=True)
289 |         else:
290 |             if self.sender_hits_max_reply(sender):
291 |                 # reset the consecutive_auto_reply_counter
292 |                 self._consecutive_auto_reply_counter[sender.name] = 0
293 |                 return
294 |             
295 |             # if parsing fails, construct a feedback message from the error code and message of the parser
296 |             # send the feedback message, and request a reply
297 |             self._consecutive_auto_reply_counter[sender.name] += 1
298 |             reply = self.feedback_generator.get_prompt("parsing", parsed_status, parsed_error_message, parsed_error_code)
299 |             self.feedback_types.append("parsing")
300 |             self.send(reply, sender, request_reply=True)
301 | 


--------------------------------------------------------------------------------
/mnms/evaluation/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/evaluation/__init__.py


--------------------------------------------------------------------------------
/mnms/evaluation/__pycache__/__init__.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/evaluation/__pycache__/__init__.cpython-39.pyc


--------------------------------------------------------------------------------
/mnms/evaluation/__pycache__/evaluator.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/evaluation/__pycache__/evaluator.cpython-39.pyc


--------------------------------------------------------------------------------
/mnms/evaluation/__pycache__/metrics.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/evaluation/__pycache__/metrics.cpython-39.pyc


--------------------------------------------------------------------------------
/mnms/evaluation/__pycache__/run.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/evaluation/__pycache__/run.cpython-39.pyc


--------------------------------------------------------------------------------
/mnms/evaluation/evaluator.py:
--------------------------------------------------------------------------------
  1 | import re
  2 | import os
  3 | import ast
  4 | import torch
  5 | import sys
  6 | sys.path.append('/gscratch/krishna/zixianma/miniconda3/envs/sg/lib/python3.9/site-packages')
  7 | sys.path.append('/gscratch/krishna/zixianma/miniconda3/lib/python3.10/site-packages')
  8 | # sys.path.append('./')
  9 | import json
 10 | import pickle
 11 | import tempfile
 12 | import traceback
 13 | import open_clip
 14 | import pandas as pd
 15 | from PIL import Image
 16 | from typing import Any, Dict, List, Literal, Tuple, Optional
 17 | from functools import partial
 18 | from .metrics import PlanAccMetrics, PRFMetrics, EditDistMetrics
 19 | from mnms.constants import TOOL_METADATA
 20 | from nltk.translate.bleu_score import sentence_bleu
 21 | from skimage.metrics import mean_squared_error as mse
 22 | import numpy as np
 23 | from pycocotools.coco import COCO
 24 | from pycocotools.cocoeval import COCOeval
 25 | from datasets import load_dataset
 26 | from tqdm import tqdm
 27 | 
 28 | 
 29 | def find_best_match(gt_code_list: List[str], pred_code: str):
 30 |     """
 31 |     Find the best match between a list of ground truth codes and predicted code.
 32 |     """
 33 |     labeler = partial(get_code_labels, include_arg_names=False)
 34 |     gt_labels_list = [labeler(gt_code) for gt_code in gt_code_list]
 35 |     pred_labels = labeler(pred_code)
 36 |     match_scores = []
 37 |     for gt_labels in gt_labels_list:
 38 |         metric = PRFMetrics(categories=None, average="binary")
 39 |         metric.update(gt_labels, pred_labels)
 40 |         match_scores.append(metric.compute()["f1"])
 41 | 
 42 |     best_match_idx = match_scores.index(max(match_scores))
 43 |     return gt_code_list[best_match_idx]
 44 | 
 45 | def replace_node_with_placeholder(text):
 46 |     """
 47 |     Example:
 48 |     input: 'prefix for node-1.bleh node-2.blah'
 49 |     output: ('prefix for {} {}', [(1, 'bleh'), (2, 'blah')])
 50 |     """
 51 |     # This regular expression matches '<node-k>.blah' patterns
 52 |     pattern = r"<node-\d+>\.\w+"
 53 |     matches = re.findall(pattern, text)
 54 |     format_args = []
 55 |     for match in matches:
 56 |         if match.startswith("<node-"):
 57 |             node_id = int(match[6 : match.index(">")])
 58 |             arg_name = match.split(".")[-1]
 59 |             format_args.append((node_id, arg_name))
 60 |     # Replace matched patterns with '{}'
 61 |     replaced_text = re.sub(pattern, "{}", text)
 62 |     return replaced_text, format_args
 63 | 
 64 | 
 65 | def nodes_to_code(nodes):
 66 |     """
 67 |     Turns the json-format nodes into python code lines (without the solve() header and output used in code_str)
 68 |     Example:
 69 |     Input:
 70 |     [{"id": 0, "name": "image captioning", "args": {"image": "2327921.jpg"}}, 
 71 |     {"id": 1, "name": "text summarization", "args": {"text": output0["text"]}}]
 72 |     Output:
 73 |     output0 = image_captioning(image="2327921.jpg")
 74 |     output1 = text_summarization(text=output0['text'])
 75 |     """
 76 |     assert isinstance(nodes, list), f"nodes is of type {type(nodes)} but expected list"
 77 | 
 78 |     code = []
 79 |     for node in nodes:
 80 |         node_name = "output{}".format(node["id"])
 81 | 
 82 |         fn_name = node["name"].replace(" ", "_")
 83 |         arg_strs = []
 84 |         for arg_name, arg_value in node["args"].items():
 85 |             if isinstance(arg_value, str):
 86 |                 if arg_value.startswith("<node-") and " " not in arg_value:
 87 |                     node_id = int(arg_value[6 : arg_value.index(">")])
 88 |                     node_key = arg_value[arg_value.index(".") + 1 :]
 89 |                     arg_value = f"output{node_id}['{node_key}']"
 90 |                 elif "<node-" in arg_value:
 91 |                     placeholder_str, node_infos = replace_node_with_placeholder(
 92 |                         arg_value
 93 |                     )
 94 |                     node_values = []
 95 |                     for node_id, node_key in node_infos:
 96 |                         node_values.append(f"output{node_id}['{node_key}']")
 97 |                     arg_value = placeholder_str.format(*node_values)
 98 |                     arg_value = arg_value.replace('"', '\\"')
 99 |                     arg_value = 'f"{}"'.format(arg_value)
100 |                 else:
101 |                     arg_value = arg_value.replace('"', '\\"')
102 |                     arg_value = f'"{arg_value}"'
103 |             else:
104 |                 # Turn argument values for e.g. number, year etc into strings
105 |                 arg_value = f'"{arg_value}"'
106 | 
107 |             arg_strs.append(f"{arg_name}={arg_value}")
108 |         arg_str = ", ".join(arg_strs)
109 |         code.append(f"{node_name} = {fn_name}({arg_str})")
110 |     return "\n".join(code)
111 | 
112 | 
113 | def extract_func_info_from_single_line(code):
114 |     """
115 |     Example:
116 |     input: output1 = object_detection(image=output0['image'])
117 |     output: {
118 |         'output_var': 'output1',
119 |         'func_name': 'object_detection',
120 |         'args': {'image': 'output0[\'image\']'}}
121 |     """
122 |     # Parse the given code to an AST
123 |     try:
124 |         tree = ast.parse(code)
125 | 
126 |         # Initialize the result dictionary
127 |         result = {"output_var": "", "func_name": "", "args": {}}
128 | 
129 |         # Find the first assignment in the code
130 |         for node in ast.walk(tree):
131 |             if isinstance(node, ast.Assign):
132 |                 # Extract the node name (variable name on the left of the assignment)
133 |                 result["output_var"] = node.targets[0].id
134 | 
135 |                 # Check if the value is a function call
136 |                 if isinstance(node.value, ast.Call):
137 |                     if isinstance(node.value.func, ast.Name):
138 |                         result["func_name"] = node.value.func.id
139 | 
140 |                     # Extract the function arguments
141 |                     for arg in node.value.args:
142 |                         # This example doesn't handle positional arguments, but could be extended
143 |                         pass
144 |                     for keyword in node.value.keywords:
145 |                         result["args"][keyword.arg] = ast.unparse(keyword.value)
146 | 
147 |                 break  # Assuming only one assignment of interest per line of code
148 | 
149 |         if result["output_var"] == "" or result["func_name"] == "":
150 |             return None
151 |     except Exception as e:
152 |         # print(e)
153 |         # print(code)
154 |         return None
155 |     return result
156 | 
157 | 
158 | def extract_func_info_from_code(code):
159 |     info = []
160 |     for line in code.split("\n"):
161 |         result = extract_func_info_from_single_line(line.strip())
162 |         if result:
163 |             info.append(result)
164 | 
165 |     return info
166 | 
167 | def func_info_to_code(info):
168 |     code = []
169 |     for node in info:
170 |         node_name = node["output_var"]
171 |         fn_name = node["func_name"]
172 |         arg_strs = []
173 |         for arg_name, arg_value in node["args"].items():
174 |             arg_value = f"{arg_value}"
175 |             arg_strs.append(f"{arg_name}={arg_value}")
176 |         arg_str = ", ".join(arg_strs)
177 |         code.append(f"{node_name} = {fn_name}({arg_str})")
178 |     return "\n".join(code)
179 | 
180 | def get_code_labels(
181 |     code: str, include_arg_names=False, include_arg_values=False, edge_as_label=False, sep="--"
182 | ):
183 |     """
184 |     Example input:
185 |     output0 = image_captioning(image="2327921.jpg")
186 |     output1 = text_summarization(text=output0['text'])
187 |     output2 = question_answering(text=output1['text'], question="What is the main dessert mentioned in the caption?")
188 |     """
189 | 
190 |     parse = extract_func_info_from_code(code)
191 |     if not include_arg_names and include_arg_values is True:
192 |         raise ValueError("Cannot include arg values without arg names")
193 | 
194 |     labels = []
195 |     for i, func_info in enumerate(parse):
196 |         if "func_name" not in func_info:
197 |             continue
198 |         if edge_as_label:
199 |             if i + 1 >= len(parse) or "func_name" not in parse[i+1]:
200 |                 continue
201 |             label = [parse[i]["func_name"], parse[i+1]['func_name']]
202 |         else:
203 |             label = [func_info["func_name"]]
204 | 
205 |             if include_arg_names:
206 |                 # Sort the arguments by names to align the pred's and label's orderings 
207 |                 sorted_args = {val[0] : val[1] for val in sorted(func_info["args"].items(), key = lambda x: x[0])}
208 |                 for arg_name, arg_value in sorted_args.items():
209 |                     if include_arg_values:
210 |                         if isinstance(arg_value, str):
211 |                             arg_value = arg_value.lower()    
212 |                         label.append(f"{arg_name}={arg_value}")
213 |                     else:
214 |                         label.append(arg_name)
215 | 
216 |         labels.append(sep.join(label))
217 | 
218 |     return labels
219 | 
220 | def list_directories(path) -> List[str]:
221 |     """List all directories within a given path."""
222 |     return [d for d in os.listdir(path) if os.path.isdir(os.path.join(path, d))]
223 | 
224 | 
225 | def image_similarity(gt_image_path, pred_image_path, method="clip"):
226 |     gt_image = Image.open(gt_image_path)
227 |     pred_image = Image.open(pred_image_path)
228 |     if method == "mse":
229 |         score = mse(np.asarray(gt_image), np.asarray(pred_image))
230 |     elif method == "clip":
231 |         score = get_similarity_between_images(gt_image, pred_image)
232 |     return score
233 | 
234 | def get_similarity_between_images(gt_image, pred_image):
235 |     device = "cuda" if torch.cuda.is_available() else "cpu"
236 |     clip_model, _, preprocess = open_clip.create_model_and_transforms('ViT-H-14-378-quickgelu', pretrained='dfn5b', device=device)
237 |     gt_image = preprocess(gt_image).unsqueeze(0)
238 |     pred_image = preprocess(pred_image).unsqueeze(0)
239 | 
240 |     with torch.no_grad(), torch.cuda.amp.autocast():
241 |         gt_image_features = clip_model.encode_image(gt_image.to(device))
242 |         pred_image_features = clip_model.encode_image(pred_image.to(device))
243 |         
244 |         gt_image_features /= gt_image_features.norm(dim=-1, keepdim=True)
245 |         pred_image_features /= pred_image_features.norm(dim=-1, keepdim=True)
246 |     
247 |     similarity = gt_image_features @ pred_image_features.T
248 |     return similarity[0,0].item()
249 | 
250 | def bleu(reference, candidate):
251 |     # Calculating BLEU score
252 |     ref_tokens = [reference.lower().split()]
253 |     cand_tokens = candidate.lower().split()
254 |     n = len(cand_tokens)
255 |     if n == 0:
256 |         return 0
257 |     if n < 4:
258 |         weights = [1/n] * n
259 |     else:
260 |         weights = [0.25] * 4
261 |     score = sentence_bleu(ref_tokens, cand_tokens, weights=weights)
262 |     return score
263 | 
264 | 
265 | def get_average_precision(gt_objects, pred_objects, temp_dir = "/mmfs1/gscratch/krishna/zixianma/mnms/mnms/execution/temp_data"):
266 |     if len(pred_objects) == 0 or len(pred_objects[0]) == 0: # no predicted object
267 |         return 0
268 |     gt_objects = convert_objects_into_coco_format(gt_objects, gt=True)
269 |     
270 |     pred_objects = convert_objects_into_coco_format(pred_objects, gt=False)
271 |     with tempfile.NamedTemporaryFile(mode='w+', dir=temp_dir, suffix=".json", delete=False) as temp_file:
272 |         json.dump(gt_objects, temp_file)
273 |         temp_filename = temp_file.name
274 |         temp_file.seek(0)
275 |         cocoGt = COCO(temp_filename)
276 |     
277 |     with tempfile.NamedTemporaryFile(mode='w+', dir=temp_dir, suffix=".json", delete=False) as temp_file:
278 |         json.dump(pred_objects, temp_file)
279 |         temp_filename = temp_file.name
280 |         temp_file.seek(0)
281 |         cocoDt = cocoGt.loadRes(temp_filename)
282 |     score = get_average_precision_at_iou_50(cocoGt, cocoDt)
283 |     return score   
284 |     
285 | def get_average_precision_at_iou_50(cocoGt, cocoDt):
286 |     # Create COCO Eval object with the annotations and the detections
287 |     cocoEval = COCOeval(cocoGt, cocoDt, 'bbox')
288 |     cocoEval.evaluate()
289 |     cocoEval.accumulate()
290 |     cocoEval.summarize()
291 |     p = cocoEval.params
292 |     # Get Average Precision (AP)
293 |     # print('Average Precision (AP) for IoU threshold 0.5:', cocoEval.stats[1])
294 |     return cocoEval.stats[1]
295 | 
296 | def convert_objects_into_coco_format(objects, gt=False):
297 |     obj_list = []
298 |     image_list = []
299 |     image_id = 1
300 |     categories = {}
301 |     for i, object in enumerate(objects):
302 |         id = i + 1
303 |         obj_cat = object["label"]
304 |         if obj_cat in categories:
305 |             cat_id = categories[obj_cat]
306 |         else:
307 |             cat_id = (max(categories.values()) + 1) if len(categories) > 0 else 0
308 |             categories[obj_cat] = cat_id 
309 |         x, y = int(object["bbox"][0]), int(object["bbox"][1])
310 |         width, height = int(object["bbox"][2] - object["bbox"][0]), int(object["bbox"][3] - object["bbox"][1])
311 |         bbox = [x, y, width, height]
312 |         area = width * height
313 |         
314 |         obj_dict = {
315 |             "id": id,
316 |             "image_id": image_id,
317 |             "category_id": cat_id,
318 |             "bbox": bbox,
319 |             "area": int(area),
320 |             "iscrowd": 0
321 |         }
322 |         if not gt:
323 |             obj_dict["score"] = object['score'] if 'score' in object else 1.0
324 | 
325 |         img_dict = {"id": image_id}
326 |         obj_list.append(obj_dict)
327 |         image_list.append(img_dict)
328 | 
329 |     cat_list = []
330 |     if gt:
331 |         for obj_cat, cat_id in categories.items():
332 |             cat_dict = {"id": cat_id, "name": obj_cat}
333 |             cat_list.append(cat_dict)
334 | 
335 |         final_output = {"categories": cat_list, "images": image_list, "annotations": obj_list}
336 |     else:
337 |         final_output = obj_list
338 |     return final_output 
339 | 
340 | 
341 | class MNMEvaluator:
342 |     def __init__(
343 |         self,
344 |         gt_data: List[Dict[str, Any]],
345 |         pred_data: List[Dict[str, Any]],
346 |         eval_set: Literal["gt", "pred"],
347 |         plan_format: Literal["code", "json"],
348 |         num_tools: int = 33,
349 |         use_best_match: bool = False,
350 |     ) -> None:
351 |         self.gt_data = gt_data
352 |         self.pred_data = pred_data
353 |         self.eval_set = eval_set
354 |         self.plan_format = plan_format
355 |         self.num_tools = num_tools
356 |         self.use_best_match = use_best_match
357 |         self.categories = self.get_categories()
358 |         assert (
359 |             len(self.categories) == self.num_tools
360 |         ), "Found {} categories, expected {}".format(
361 |             len(self.categories), self.num_tools
362 |         )
363 |         self.metrics = {
364 |             "tool_macro": PRFMetrics(categories=self.categories, average="macro"), 
365 |             "argname": PRFMetrics(categories=None, average="binary"),
366 |             "argvalue": PRFMetrics(categories=None, average="binary"),
367 |             "edge": PRFMetrics(categories=None, average="binary"),
368 |             "plan_tool": PlanAccMetrics(),
369 |             "plan_argname": PlanAccMetrics(),
370 |             "plan_argvalue": PlanAccMetrics(),
371 |             "tool": EditDistMetrics(categories=self.categories),
372 |         }
373 |         self.code_labelers = {
374 |             "tool_macro": partial(get_code_labels),
375 |             "argname": partial(get_code_labels, include_arg_names=True),
376 |             "argvalue": partial(get_code_labels, include_arg_names=True, include_arg_values=True),
377 |             "edge": partial(get_code_labels, edge_as_label=True),
378 |             "plan_tool": partial(get_code_labels),
379 |             "plan_argname": partial(get_code_labels, include_arg_names=True),
380 |             "plan_argvalue": partial(get_code_labels, include_arg_names=True, include_arg_values=True),
381 |             "tool": partial(get_code_labels),
382 |         }
383 |         self.wrong_predictions = {"plan_tool": [], "plan_argname": [], "plan_argvalue": []}
384 | 
385 |     def get_categories(self):
386 |         categories = []
387 |         for tool in TOOL_METADATA.keys():
388 |             categories.append(tool.replace(" ", "_"))
389 |         return categories
390 | 
391 |     def get_paired_eval_data(self):
392 |         if self.eval_set == "gt":
393 |             eval_ids = [sample["id"] for sample in self.gt_data]
394 |         elif self.eval_set == "pred":
395 |             eval_ids = [sample["id"] for sample in self.pred_data]
396 |         else:
397 |             raise ValueError("Invalid eval_set")
398 | 
399 |         id2gt = {
400 |             sample["id"]: sample for sample in self.gt_data if sample["id"] in eval_ids
401 |         }
402 |         id2pred = {
403 |             sample["id"]: sample
404 |             for sample in self.pred_data
405 |             if sample["id"] in eval_ids
406 |         }
407 |         for sample_id in eval_ids:
408 |             yield id2gt[sample_id], id2pred[sample_id]
409 | 
410 |     def evaluate(self) -> Tuple[pd.DataFrame, List[Dict[str, Any]]]:
411 |         invalid_predictions = []
412 |         for gt_sample, pred_sample in self.get_paired_eval_data():
413 |             try:
414 |                 pred_code = pred_sample["prediction"] if self.plan_format == 'code' else nodes_to_code(pred_sample["prediction"])
415 |             except Exception as e:
416 |                 print(pred_sample["prediction"])
417 |                 print(e, '\n')
418 |                 pred_code = ""
419 |                 invalid_predictions.append(
420 |                     dict(gt=gt_sample, pred=pred_sample, err=traceback.format_exc())
421 |                 )
422 |             if not isinstance(pred_code, str):
423 |                 pred_code = str(pred_code)
424 |             if self.use_best_match:
425 |                 plans = [eval(gt_sample["plan_str"])] + eval(gt_sample["alt_plans_str"])
426 |                 gt_code_list = [nodes_to_code(plan) for plan in plans]
427 |                 gt_code = find_best_match(gt_code_list, pred_code)
428 |             else:
429 |                 gt_code = nodes_to_code(eval(gt_sample["plan_str"]))
430 |             
431 |             for metric_name, metric in self.metrics.items():
432 |                 gt_labels = self.code_labelers[metric_name](gt_code)
433 |                 pred_labels = self.code_labelers[metric_name](pred_code)
434 |                 metric.update(gt_labels, pred_labels)
435 |                 # Record wrong predictions
436 |                 if metric_name in ["plan_tool", "plan_argname", "plan_argvalue"] and pred_labels != gt_labels:
437 |                     self.wrong_predictions[metric_name].append({'id': gt_sample['id'], 'pred': pred_labels, 'label': gt_labels})
438 |                 
439 | 
440 |         records = []
441 |         for metric_name, metric in self.metrics.items():
442 |             metrics: Dict[str, Any] = metric.compute()
443 |             for k, v in metrics.items():
444 |                 record = {}
445 |                 record["metric"] = f"{metric_name}_{k}"
446 |                 record["value"] = v
447 | 
448 |                 records.append(record)
449 | 
450 |         df = pd.DataFrame.from_records(
451 |             records, columns=["metric", "value"]
452 |         )
453 |         return df, invalid_predictions
454 | 
455 | 
456 | class MNMFinalResultEvaluator:
457 |     def __init__(
458 |         self,
459 |         gt_dir: str,
460 |         pred_dir: str,
461 |         selected_eval_ids: Optional[List] = None
462 |     ) -> None:
463 |         self.gt_dir = gt_dir
464 |         self.pred_dir = pred_dir
465 |         self.all_out_keys = set()
466 |         for tool, tool_meta in TOOL_METADATA.items():
467 |             out_keys = tool_meta['output']['arg_name']
468 |             self.all_out_keys.update(out_keys)
469 |         self.selected_eval_ids = selected_eval_ids 
470 | 
471 |     def evaluate(self) -> Tuple[float, pd.DataFrame]:
472 |         gt_results, _ = self.extract_all_results(self.gt_dir)
473 |         pred_results, _ = self.extract_all_results(self.pred_dir)
474 |         all_task_ids = self.selected_eval_ids if self.selected_eval_ids else list(gt_results.keys())
475 | 
476 |         scores = []
477 |         for task_id in tqdm(all_task_ids):
478 |             task_id = str(task_id)
479 |             if task_id not in gt_results:
480 |                 print(f"skipping {task_id} as it's not in gt execution results")
481 |                 continue
482 |             if task_id not in pred_results:
483 |                 # score is defined as 0 if no execution result is found in prediction directory
484 |                 final_score = 0
485 |             else:
486 |                 gt_res = gt_results[task_id]
487 |                 pred_res = pred_results[task_id]
488 |                 
489 |                 results = {"gt": gt_res, "pred": pred_res}
490 |                 final_results = {}
491 |                 for key, res in results.items():
492 |                     final_ans = {}
493 |                     for node, node_res in res.items():
494 |                         for res_key, res_content in node_res.items():
495 |                             if res_key in final_ans:
496 |                                 final_ans[res_key].append(res_content)
497 |                             else:
498 |                                 final_ans[res_key] = [res_content]
499 |                     final_results[key] = final_ans
500 |                 final_score = self.score_final_results(task_id, final_results['gt'], final_results['pred'])
501 |             # print("FINAL score:", final_score)
502 |             scores.append({'task_id': task_id, 'avg_score': round(final_score, 4)})
503 | 
504 |         df = pd.DataFrame.from_records(scores)
505 |         avg_score = df['avg_score'].mean()
506 |         return avg_score, df
507 |     
508 |     def score_final_results(self, task_id, gt_result, pred_result) -> float:
509 |         all_out_keys = self.all_out_keys
510 |         all_scores = []
511 |         for res_key, gt_content in gt_result.items():
512 |             if res_key not in pred_result or res_key not in all_out_keys:
513 |                 print(f"skipping {res_key}, because it's not in {all_out_keys} or {pred_result.keys()}")
514 |                 continue
515 |             pred_content = pred_result[res_key]
516 |             if res_key == "text" or (isinstance(gt_content, str) and isinstance(pred_content, str)):
517 |                 gt_content = '\n'.join(gt_content)
518 |                 pred_content = '\n'.join(pred_content)
519 |             
520 |             content_score = self.eval_content_by_key(task_id, res_key, gt_content, pred_content)
521 |             if content_score:
522 |                 all_scores.append(content_score)
523 |         total_score = np.mean(all_scores) if len(all_scores) > 0 else 0
524 |         return total_score
525 | 
526 |     def extract_all_results(self, result_dir) -> Tuple[Dict[str, Dict], List[str]]:
527 |         all_task_ids = list_directories(result_dir)
528 |         all_results = {}
529 |         invalid_ids = set()
530 |         for task_id in all_task_ids:
531 |             results = {}
532 |             results_path = os.path.join(result_dir, task_id, "result.json")
533 |             if os.path.exists(results_path):
534 |                 node2path = json.load(open(results_path, "r"))
535 |                 for node_id, node_path in node2path.items():
536 |                     if not os.path.exists(node_path): # the tool node's output path might be a relative path
537 |                         node_path = os.path.join(result_dir, task_id, node_path)
538 |                         if not os.path.exists(node_path):
539 |                             invalid_ids.add(task_id)
540 |                             continue
541 |                     if node_path.endswith(".json"):
542 |                         node_results = json.load(open(node_path, "r"))
543 |                     elif node_path.endswith(".pkl"):
544 |                         node_results = pickle.load(open(node_path, "rb"))
545 |                     else:
546 |                         raise NotImplementedError
547 |                     results[node_id] = node_results
548 |             all_results[task_id] = results
549 |         return all_results, invalid_ids
550 |     
551 | 
552 |     def eval_content_by_key(self, task_id, key, gt_content, pred_content) -> float:
553 |         if key == "text" or (isinstance(gt_content, str) and isinstance(pred_content, str)):
554 |             score = bleu(gt_content, pred_content)
555 |             print(f"BELU score: {score}")
556 |             return score
557 |         elif key == "image":
558 |             scores = []
559 |             for gt_img, pred_img in zip(gt_content, pred_content):
560 |                 gt_img = os.path.join(self.gt_dir, task_id, gt_img) # because the gt_img is a relative path
561 |                 if (gt_img and pred_img) and os.path.exists(gt_img) and os.path.exists(pred_img):
562 |                     score = image_similarity(gt_img, pred_img)
563 |                 else:
564 |                     score = 0
565 |                 scores.append(score)
566 |             final_score = np.mean(scores)
567 |             return final_score
568 |         elif key in ["objects", "object"]:
569 |             gt_objects = gt_content[0]
570 |             pred_objects = pred_content[0]
571 |             if len(gt_objects) == 0:
572 |                 return None
573 |             exact_match_score = int(str(gt_objects) == str(pred_objects))
574 |             if key == "object":
575 |                 gt_objects = [gt_objects]
576 |                 pred_objects = [pred_objects]
577 | 
578 |             if 'label' in gt_objects[0] and 'bbox' in gt_objects[0]:
579 |                 ap_score = get_average_precision(gt_objects, pred_objects)
580 |             else:
581 |                 ap_score = 0
582 |             score = max(exact_match_score, ap_score)
583 |             return score
584 |         else:
585 |             # if str(gt_content) != str(pred_content):
586 |             #     print(f"results for {key}:")
587 |             #     print("GT:", gt_content)
588 |             #     print("PRED:", pred_content)
589 |             res = int(str(gt_content) == str(pred_content))
590 |         return res
591 | 
592 | 
593 |         
594 | 


--------------------------------------------------------------------------------
/mnms/evaluation/metrics.py:
--------------------------------------------------------------------------------
  1 | from typing import Dict, List, Literal, Optional, Tuple
  2 | from sklearn.metrics import precision_recall_fscore_support, accuracy_score
  3 | import Levenshtein
  4 | 
  5 | def flatten_labels(
  6 |     gt_labels: List[str], pred_labels: List[str], categories: Optional[List[str]] = None
  7 | ) -> Tuple[List[int], List[int], List[str]]:
  8 |     """
  9 |     Flatten the ground truth and predicted labels into a single list of int values.
 10 |     """
 11 |     union = list(set(gt_labels + pred_labels))
 12 |     if categories:
 13 | 
 14 |         def gt_feature(label):
 15 |             if label in gt_labels and label in categories:
 16 |                 return categories.index(label) + 1
 17 |             return 0
 18 | 
 19 |         def pred_feature(label):
 20 |             if label in pred_labels and label in categories:
 21 |                 return categories.index(label) + 1
 22 |             return 0
 23 | 
 24 |     else:
 25 | 
 26 |         def gt_feature(label):
 27 |             return 1 if label in gt_labels else 0
 28 | 
 29 |         def pred_feature(label):
 30 |             return 1 if label in pred_labels else 0
 31 | 
 32 |     gt_flat = [gt_feature(label) for label in union]
 33 |     pred_flat = [pred_feature(label) for label in union]
 34 |     return gt_flat, pred_flat, union
 35 | 
 36 | class EditDistMetrics:
 37 |     """
 38 |     Class to calculate normalized edit distance for the MNM task.
 39 |     """
 40 |     def __init__(
 41 |         self,
 42 |         categories: Optional[List[str]],
 43 |         normalized: bool = True,
 44 |     ):
 45 |         self.categories = categories
 46 |         self.cat2label = {cat: i + 1 for i, cat in enumerate(categories)}
 47 |         self.normalized = normalized
 48 |         self._gt: List[int] = []
 49 |         self._pred: List[int] = []
 50 | 
 51 |     def update(self, gt_labels: List[str], pred_labels: List[str]):
 52 |         # Turn tool names into numerical labels starting from 1, or 0 if a tool name does not exist
 53 |         self._gt.append([self.cat2label.get(name, 0) for name in gt_labels])
 54 |         self._pred.append([self.cat2label.get(name, 0) for name in pred_labels])
 55 | 
 56 |     def compute(self) -> Dict[str, float]:
 57 |         assert len(self._gt) == len(self._pred)
 58 |         total = 0
 59 |         for gt, pred in zip(self._gt, self._pred):
 60 |             total += Levenshtein.ratio(pred, gt)
 61 |         ned = 1 - total / len(self._gt)
 62 |         return {
 63 |             "normalized_edit_distance": round(ned * 100, 2)
 64 |         }
 65 | 
 66 | 
 67 | class PlanAccMetrics:
 68 |     """
 69 |     Class to calculate plan accuracy for the MNM task.
 70 |     """
 71 |     def __init__(
 72 |         self
 73 |     ):
 74 |         self._gt: List[int] = []
 75 |         self._pred: List[int] = []
 76 | 
 77 |     def update(self, gt_labels: List[str], pred_labels: List[str]):
 78 |         self._gt.append(gt_labels)
 79 |         self._pred.append(pred_labels)
 80 | 
 81 |     def compute(self) -> Dict[str, float]:
 82 |         # Binarize gt and pred, treating each gt plan as 1
 83 |         binary_gt = [1.0] * len(self._gt)
 84 |         binary_pred = []
 85 |         for gt, pred in zip(self._gt, self._pred):
 86 |             binary_pred += ([1.0] if pred == gt else [0.0])
 87 |         acc = accuracy_score(binary_gt, binary_pred)
 88 |         return {
 89 |             "accuracy": round(100 * acc, 2)
 90 |         }
 91 | 
 92 | 
 93 | class PRFMetrics:
 94 |     """
 95 |     Class to calculate the precision, recall, f1 metrics for the MNM task.
 96 |     """
 97 | 
 98 |     def __init__(
 99 |         self,
100 |         categories: Optional[List[str]],
101 |         average: Optional[Literal["micro", "macro", "binary"]],
102 |     ):
103 |         self.categories = categories
104 |         self.average = average
105 |         self.labels = list(range(1, len(categories) + 1)) if categories else None
106 |         self._gt_flat: List[int] = []
107 |         self._pred_flat: List[int] = []
108 |         self._union: List[str] = []
109 | 
110 |     def update(self, gt_labels: List[str], pred_labels: List[str]):
111 |         gt_flat, pred_flat, union = flatten_labels(
112 |             gt_labels, pred_labels, categories=self.categories
113 |         )
114 |         self._gt_flat.extend(gt_flat)
115 |         self._pred_flat.extend(pred_flat)
116 |         self._union.extend(union)
117 | 
118 |     def compute(self) -> Dict[str, float]:
119 |         precision, recall, f1, _ = precision_recall_fscore_support(
120 |             self._gt_flat, self._pred_flat, labels=self.labels, average=self.average
121 |         )
122 |         return {
123 |             "precision": round(100 * precision, 2),
124 |             "recall": round(100 * recall, 2),
125 |             "f1": round(100 * f1, 2),
126 |         }


--------------------------------------------------------------------------------
/mnms/evaluation/run.py:
--------------------------------------------------------------------------------
 1 | import json 
 2 | import argparse
 3 | from datasets import load_dataset
 4 | from .evaluator import MNMEvaluator
 5 | 
 6 | def load_mnm_data_from_json(path):
 7 |     data = []
 8 |     with open(path, "r") as f:
 9 |         for line in f:
10 |             sample = json.loads(line)
11 |             data.append(sample)
12 |     return data
13 | 
14 | def get_args():
15 |     parser = argparse.ArgumentParser()
16 |     parser.add_argument('--preds-file', default=None, type=str, help="the json file to import the predictions from.", required=True)
17 |     parser.add_argument('--output-csv', default=None, type=str, help="the csv file to save the output to.")
18 |     parser.add_argument('--plan-format', default=None, type=str, choices=['json', 'code'], required=True)
19 |     parser.add_argument('--eval-set', default='gt', type=str, choices=['gt', 'pred'])
20 |     args = parser.parse_args()
21 |     return args
22 | 
23 | def main():
24 |     args = get_args()
25 |     
26 |     gt_data = load_dataset("zixianma/mnms", split="test_human_verified_filtered")
27 |     pred_data = load_mnm_data_from_json(args.preds_file)
28 |     total_n = len(gt_data) if args.eval_set == "gt" else len(pred_data)
29 | 
30 |     evaluator = MNMEvaluator(
31 |         gt_data=gt_data,
32 |         pred_data=pred_data,
33 |         eval_set=args.eval_set,
34 |         plan_format=args.plan_format
35 |     )
36 |     df, invalid_predictions = evaluator.evaluate()
37 |     print(f"There are {len(invalid_predictions)}/{total_n} invalid predictions that fail to be parsed correctly.")
38 |     print(df)
39 |     if args.output_csv:
40 |         df.to_csv(args.output_csv, index=False)
41 | 
42 | 
43 | if __name__ == '__main__':
44 |     main()


--------------------------------------------------------------------------------
/mnms/evaluation/run_final_result_eval.py:
--------------------------------------------------------------------------------
 1 | import json
 2 | import os 
 3 | import argparse
 4 | from datasets import load_dataset
 5 | from .evaluator import MNMEvaluator, MNMFinalResultEvaluator
 6 | 
 7 | def get_args():
 8 |     parser = argparse.ArgumentParser()
 9 |     parser.add_argument('--pred-dir', default=None, type=str, help="the directory to import prediction execution results from.", required=True)
10 |     parser.add_argument('--gt-dir', default="mnms/execution/gt_results", type=str, help="the directory to import gt execution results from.")
11 |     parser.add_argument('--output-csv', default=None, type=str, help="the csv file to save the output to.")
12 |     args = parser.parse_args()
13 |     return args
14 | 
15 | def main():
16 |     args = get_args()
17 |     selected_ids = next(os.walk(args.gt_dir))[1]
18 |     print(len(selected_ids))
19 |     evaluator = MNMFinalResultEvaluator(
20 |         gt_dir=args.gt_dir, 
21 |         pred_dir=args.pred_dir,
22 |         selected_eval_ids=selected_ids[:3]
23 |     )
24 |     avg_score, df = evaluator.evaluate()
25 |     if args.output_csv:
26 |         df.to_csv(args.output_csv, index=False)
27 |     else:
28 |         exp_id = os.path.basename(args.pred_dir)
29 |         output_csv = f"{exp_id}-exec-metrics.csv"
30 |         print(output_csv)
31 |         df.to_csv(output_csv, index=False)
32 | 
33 | if __name__ == '__main__':
34 |     main()


--------------------------------------------------------------------------------
/mnms/execution/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/__init__.py


--------------------------------------------------------------------------------
/mnms/execution/config.py:
--------------------------------------------------------------------------------
 1 | import os 
 2 | 
 3 | OPENAI_API_KEY = os.environ["OPENAI_API_KEY"] 
 4 | RAPID_API_KEY =  os.environ["RAPID_API_KEY"]
 5 | OMDB_API_KEY =  os.environ["OMDB_API_KEY"]
 6 | 
 7 | MODEL_SELECTION = {
 8 |     "text_generation": "gpt-3.5-turbo-0125",
 9 |     "text_summarization": "facebook/bart-large-cnn",
10 |     "text_classification": "distilbert-base-uncased-finetuned-sst-2-english",
11 |     "question_answering": "deepset/roberta-base-squad2", # distilbert-base-uncased-distilled-squad
12 |     "automatic_speech_recognition": "openai/whisper-large-v2",
13 |     "image_generation": "stabilityai/stable-diffusion-xl-base-1.0",
14 |     "image_captioning": "Salesforce/blip-image-captioning-large",
15 |     "image_editing": "stabilityai/stable-diffusion-xl-refiner-1.0",
16 |     "image_classification": "google/vit-base-patch16-224",
17 |     "visual_question_answering": "Salesforce/blip-vqa-base",
18 |     "object_detection": "facebook/detr-resnet-101",
19 |     "image_segmentation": "facebook/maskformer-swin-base-coco",
20 |     "optical_character_recognition": "easyOCR"
21 | }
22 | 
23 | DATA_PATH = "mnms/execution/data"
24 | 
25 | RESULT_PATH = "mnms/execution/results"
26 | 
27 | CACHE_ROOT_PATH = "/gscratch/krishna/zixianma"
28 | 
29 | 


--------------------------------------------------------------------------------
/mnms/execution/data/06483.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/06483.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/07115.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/07115.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/07600.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/07600.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/08078.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/08078.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/08312.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/08312.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/08773.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/08773.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/08788.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/08788.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/09531.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/09531.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/09742.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/09742.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/09844.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/09844.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/09966.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/09966.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/100081-input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/100081-input.png


--------------------------------------------------------------------------------
/mnms/execution/data/10050.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/10050.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/100558-input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/100558-input.png


--------------------------------------------------------------------------------
/mnms/execution/data/10084.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/10084.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/10261.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/10261.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/10358.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/10358.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/10401.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/10401.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/10404.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/10404.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/10573.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/10573.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/10899.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/10899.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/11120.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/11120.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/11123.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/11123.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/11135.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/11135.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/111376-input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/111376-input.png


--------------------------------------------------------------------------------
/mnms/execution/data/11170.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/11170.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/11226.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/11226.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/11409.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/11409.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/11427.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/11427.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/11534.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/11534.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/11561.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/11561.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/1159654.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/1159654.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/11665.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/11665.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/11674.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/11674.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/11705.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/11705.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/11706.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/11706.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/11807.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/11807.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/11899.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/11899.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/12300.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/12300.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/12381.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/12381.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/12552.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/12552.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/12603.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/12603.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/12633.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/12633.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/1284-1180-0003.flac:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/1284-1180-0003.flac


--------------------------------------------------------------------------------
/mnms/execution/data/12904.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/12904.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/12946.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/12946.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/12955.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/12955.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/13181.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/13181.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/13393.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/13393.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/13426.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/13426.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/134597-input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/134597-input.png


--------------------------------------------------------------------------------
/mnms/execution/data/13480.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/13480.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/13538.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/13538.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/13644.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/13644.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/13682.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/13682.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/14021.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/14021.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/14043.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/14043.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/14095.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/14095.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/14098.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/14098.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/14168.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/14168.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/14208.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/14208.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/14306.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/14306.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/14317.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/14317.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/14382.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/14382.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/14450.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/14450.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/14496.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/14496.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/14514.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/14514.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/14553.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/14553.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/147546-input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/147546-input.png


--------------------------------------------------------------------------------
/mnms/execution/data/14757.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/14757.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/14791.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/14791.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/14799.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/14799.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/14809.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/14809.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/15021.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/15021.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/150517.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/150517.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/153639-input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/153639-input.png


--------------------------------------------------------------------------------
/mnms/execution/data/15387.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/15387.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/15491.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/15491.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/15595.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/15595.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/15610.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/15610.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/15653-input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/15653-input.png


--------------------------------------------------------------------------------
/mnms/execution/data/15764.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/15764.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/15792.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/15792.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/1592294.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/1592294.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/1592452.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/1592452.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/16148.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/16148.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/16273.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/16273.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/16386.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/16386.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/16611.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/16611.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/16657.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/16657.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/16749.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/16749.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/16942.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/16942.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/16986.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/16986.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/17042.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/17042.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/17320-input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/17320-input.png


--------------------------------------------------------------------------------
/mnms/execution/data/17412.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/17412.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/17464.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/17464.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/17716.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/17716.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/177572-input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/177572-input.png


--------------------------------------------------------------------------------
/mnms/execution/data/17862.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/17862.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/18104.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/18104.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/18225.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/18225.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/18404.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/18404.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/18535.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/18535.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/19027.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/19027.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/19068.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/19068.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/19120.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/19120.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/19358.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/19358.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/19929.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/19929.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/20864.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/20864.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/21203.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/21203.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/21740.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/21740.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/21803.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/21803.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/22638.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/22638.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2300-131720-0000.flac:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2300-131720-0000.flac


--------------------------------------------------------------------------------
/mnms/execution/data/2315602.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2315602.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2315924.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2315924.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2316209.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2316209.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2316316.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2316316.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2317756.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2317756.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2317766.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2317766.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2318145.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2318145.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2318392.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2318392.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2319267.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2319267.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2320104.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2320104.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2321508.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2321508.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2321647.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2321647.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2321676.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2321676.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2321930.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2321930.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2322211.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2322211.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2322245.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2322245.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2322902.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2322902.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2323530.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2323530.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2324260.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2324260.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2324812.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2324812.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2326465.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2326465.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2327107.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2327107.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2327921.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2327921.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2327939.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2327939.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2328271.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2328271.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2328879.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2328879.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2329245.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2329245.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2329676.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2329676.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2330591.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2330591.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2330610.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2330610.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2330986.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2330986.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2332593.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2332593.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2332697.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2332697.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2333456.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2333456.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2333829.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2333829.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2334315.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2334315.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2334677.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2334677.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2335269.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2335269.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2336649.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2336649.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2336917.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2336917.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2337344.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2337344.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2338193.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2338193.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2338250.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2338250.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2340480.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2340480.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2340962.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2340962.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2341610.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2341610.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2342830.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2342830.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2342847.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2342847.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2343942.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2343942.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2344704.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2344704.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2344819.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2344819.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2345745.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2345745.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2346590.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2346590.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2346651.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2346651.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2347052.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2347052.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2347302.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2347302.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2347441.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2347441.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2348288.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2348288.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2348943.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2348943.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2349334.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2349334.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2349496.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2349496.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2350021.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2350021.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2350057.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2350057.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2350119.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2350119.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2350499.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2350499.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2351336.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2351336.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2351464.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2351464.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2353266.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2353266.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2354281.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2354281.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2354285.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2354285.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2354461.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2354461.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2354497.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2354497.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2355794.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2355794.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2356131.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2356131.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2356182.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2356182.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2356736.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2356736.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2357888.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2357888.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2358679.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2358679.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2359351.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2359351.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2359509.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2359509.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2359565.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2359565.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2362052.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2362052.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2362289.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2362289.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2364151.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2364151.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2364622.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2364622.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2365156.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2365156.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2365901.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2365901.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2366358.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2366358.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2366671.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2366671.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2368356.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2368356.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/237-134500-0000.flac:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/237-134500-0000.flac


--------------------------------------------------------------------------------
/mnms/execution/data/2370112.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2370112.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2370590.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2370590.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2371415.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2371415.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2371677.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2371677.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2372348.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2372348.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2372497.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2372497.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2372928.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2372928.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2373927.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2373927.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2374322.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2374322.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2374973.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2374973.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2375698.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2375698.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2376309.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2376309.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2377004.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2377004.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2377385.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2377385.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2378137.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2378137.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2378292.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2378292.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2378563.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2378563.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2379887.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2379887.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2380325.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2380325.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2380691.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2380691.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2381704.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2381704.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2382560.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2382560.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2382602.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2382602.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2384900.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2384900.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2385634.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2385634.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2387080.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2387080.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2387947.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2387947.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2388803.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2388803.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2390981.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2390981.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2393510.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2393510.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/23936.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/23936.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2393797.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2393797.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2394610.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2394610.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2394635.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2394635.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2394828.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2394828.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2395343.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2395343.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2395951.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2395951.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2396295.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2396295.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2397301.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2397301.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2398150.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2398150.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2400761.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2400761.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2401001.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2401001.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2401028.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2401028.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2401271.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2401271.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2401298.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2401298.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2401498.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2401498.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2401635.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2401635.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2402643.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2402643.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2406320.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2406320.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2406339.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2406339.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2407124.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2407124.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2407807.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2407807.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2408984.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2408984.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2409068.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2409068.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2409304.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2409304.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2409579.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2409579.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2409782.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2409782.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2409896.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2409896.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2410989.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2410989.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2411180.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2411180.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2413340.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2413340.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2413350.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2413350.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2415263.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2415263.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2416088.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2416088.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2416387.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2416387.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2416914.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2416914.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/2416928.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/2416928.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/270650-input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/270650-input.png


--------------------------------------------------------------------------------
/mnms/execution/data/282572-input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/282572-input.png


--------------------------------------------------------------------------------
/mnms/execution/data/286059.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/286059.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/291614-input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/291614-input.png


--------------------------------------------------------------------------------
/mnms/execution/data/319096-input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/319096-input.png


--------------------------------------------------------------------------------
/mnms/execution/data/3575-170457-0001.flac:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/3575-170457-0001.flac


--------------------------------------------------------------------------------
/mnms/execution/data/357903-input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/357903-input.png


--------------------------------------------------------------------------------
/mnms/execution/data/360871-input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/360871-input.png


--------------------------------------------------------------------------------
/mnms/execution/data/362944-input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/362944-input.png


--------------------------------------------------------------------------------
/mnms/execution/data/398575-input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/398575-input.png


--------------------------------------------------------------------------------
/mnms/execution/data/407550-input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/407550-input.png


--------------------------------------------------------------------------------
/mnms/execution/data/417814-input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/417814-input.png


--------------------------------------------------------------------------------
/mnms/execution/data/417830-input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/417830-input.png


--------------------------------------------------------------------------------
/mnms/execution/data/4446-2271-0000.flac:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/4446-2271-0000.flac


--------------------------------------------------------------------------------
/mnms/execution/data/4507-16021-0002.flac:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/4507-16021-0002.flac


--------------------------------------------------------------------------------
/mnms/execution/data/456782-input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/456782-input.png


--------------------------------------------------------------------------------
/mnms/execution/data/498121.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/498121.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/510027-input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/510027-input.png


--------------------------------------------------------------------------------
/mnms/execution/data/543942-input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/543942-input.png


--------------------------------------------------------------------------------
/mnms/execution/data/581668-input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/581668-input.png


--------------------------------------------------------------------------------
/mnms/execution/data/61-70968-0000.flac:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/61-70968-0000.flac


--------------------------------------------------------------------------------
/mnms/execution/data/672-122797-0000.flac:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/672-122797-0000.flac


--------------------------------------------------------------------------------
/mnms/execution/data/6829-68769-0002.flac:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/6829-68769-0002.flac


--------------------------------------------------------------------------------
/mnms/execution/data/70952-input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/70952-input.png


--------------------------------------------------------------------------------
/mnms/execution/data/8224-274384-0000.flac:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/8224-274384-0000.flac


--------------------------------------------------------------------------------
/mnms/execution/data/8455-210777-0002.flac:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/8455-210777-0002.flac


--------------------------------------------------------------------------------
/mnms/execution/data/86582-input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/86582-input.png


--------------------------------------------------------------------------------
/mnms/execution/data/908-157963-0001.flac:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/908-157963-0001.flac


--------------------------------------------------------------------------------
/mnms/execution/data/COCO_train2014_000000028742.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/COCO_train2014_000000028742.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/COCO_train2014_000000113236.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/COCO_train2014_000000113236.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/COCO_train2014_000000134502.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/COCO_train2014_000000134502.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/COCO_train2014_000000256855.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/COCO_train2014_000000256855.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/COCO_train2014_000000306393.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/COCO_train2014_000000306393.jpg


--------------------------------------------------------------------------------
/mnms/execution/data/ocr_temp.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/execution/data/ocr_temp.jpg


--------------------------------------------------------------------------------
/mnms/execution/executor.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | import pickle
  3 | from .config import *
  4 | from .utils import pipeline, save_result, save_output
  5 | 
  6 | class Executor:
  7 |     def __init__(
  8 |         self, 
  9 |         save_result: bool = True, 
 10 |         log_obs: bool = True, 
 11 |         result_folder: str = RESULT_PATH
 12 |     ) -> None:
 13 |         self.save_result = save_result
 14 |         self.log_obs = log_obs
 15 |         self.result_folder = result_folder
 16 | 
 17 |     def execute(self, task_id, task_nodes):
 18 |         """
 19 |         Example input:
 20 |         task_nodes = {'id': 0, 'name': 'image generation', 'args': {'text': 'an image depicting a woman worker looking at rear view mirror smiling'}
 21 |         or
 22 |         task_nodes = [{'id': 0, 'name': 'image generation', 'args': {'text': 'an image depicting a woman worker looking at rear view mirror smiling'}}, 
 23 |         {'id': 1, 'name': 'image captioning', 'args': {'image': '<node-0>.image'}}]   
 24 |         """
 25 |         try:
 26 |             if isinstance(task_nodes, dict):
 27 |                 output =  pipeline(task_id, [task_nodes], result_folder=self.result_folder)
 28 |             elif isinstance(task_nodes, list):
 29 |                 output =  pipeline(task_id, task_nodes, result_folder=self.result_folder)
 30 |             else:
 31 |                 print("input nodes are invalid.")
 32 |                 raise ValueError
 33 |             if self.save_result:
 34 |                 save_result(task_id, output['path'], result_folder=self.result_folder)
 35 |             if self.log_obs:
 36 |                 last_node_id = max(list(output['value'].keys()))
 37 |                 return {'status': True, 'message': f"Execution succeeded. The output of ACTION {last_node_id} is {output['value'][last_node_id]}.\n"}
 38 |             else:
 39 |                 return {'status': True, 'message': f"Execution succeeded."}
 40 |         except Exception as err:
 41 |             return {'status': False, 'message': f"Execution failed with {type(err)}: {err}."}
 42 | 
 43 | 
 44 | 
 45 | class CodeExecutor(Executor):
 46 |     def __init__(
 47 |         self, 
 48 |         save_result: bool = True, 
 49 |         log_obs: bool = True, 
 50 |         result_folder: str = RESULT_PATH
 51 |         ):
 52 |         super().__init__(save_result, log_obs, result_folder)
 53 | 
 54 |     def save_result_as_pickle(self, task_id, result):
 55 |         full_result_path = os.path.join(self.result_folder, str(task_id))
 56 |         output_dict_path_pickle = os.path.join(full_result_path, f"result.pkl")
 57 |         with open(output_dict_path_pickle, "wb") as file:
 58 |             pickle.dump(result, file)   
 59 | 
 60 |     def execute(self, task_id, code_snippet):
 61 |         """
 62 |         Example input:
 63 |         def solve():
 64 |             output0 = image_generation(text="an image depicting a woman worker looking at rear view mirror smiling")
 65 |             output1 = image_captioning(image=output0['image'])
 66 |             result = {0: output0, 1: output1}
 67 |             return result
 68 |         """
 69 |         try:
 70 |             # Avoid execution getting stuck waiting for inputs
 71 |             if code_snippet.find('input(') > -1 or code_snippet.find('stdin') > -1:
 72 |                 return {'status': False, 'message': "Code snippet contains input() or stdin function, which is not supported."}
 73 |         
 74 |             import_lib = "import json\n"
 75 |             import_lib += "from mnms.execution.tool_api import *\n"""
 76 | 
 77 |             final_code = import_lib + code_snippet
 78 |             print(final_code)
 79 |             global_vars = {}
 80 |             exec(final_code, global_vars)
 81 |             result = global_vars['solve']() 
 82 |             outputs_path = {} 
 83 |             if isinstance(result, dict):
 84 |                 for node_id, content in result.items():
 85 |                     if isinstance(content, dict):
 86 |                         output_path = save_output(task_id, node_id, content, self.result_folder)
 87 |                         outputs_path[str(node_id)] = output_path
 88 |                     else:
 89 |                         self.save_result_as_pickle(task_id, result)
 90 |                         break
 91 |                 if len(outputs_path) > 0:
 92 |                     save_result(task_id, outputs_path, result_folder=self.result_folder)
 93 |             else:
 94 |                 self.save_result_as_pickle(task_id, result)
 95 | 
 96 |             if self.log_obs:
 97 |                 return {'status': True, 'message': f"Execution succeeded. The output is {result}.\n"}
 98 |             else:
 99 |                 return {'status': True, 'message': f"Execution succeeded."}
100 |         except Exception as err:
101 |             return {'status': False, 'message': f"Execution failed with {type(err)}: {err}."}
102 | 
103 | 
104 | 
105 |         


--------------------------------------------------------------------------------
/mnms/execution/run.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | import json
 3 | import argparse
 4 | from .executor import *
 5 | from .tool_api import *
 6 | from datasets import load_dataset
 7 | 
 8 | def get_args():
 9 |     parser = argparse.ArgumentParser()
10 |     parser.add_argument('--input-file', default=None, type=str, help="the file to import the input from.")
11 |     parser.add_argument('--output-dir', default=None, type=str, help="the directory to save the output to.")
12 |     parser.add_argument('--plan-format', default=None, type=str, choices=['json', 'code'], required=True)
13 |     args = parser.parse_args()
14 |     return args
15 | 
16 | def load_mnm_data_from_json(path):
17 |     data = []
18 |     with open(path, "r") as f:
19 |         for line in f:
20 |             sample = json.loads(line)
21 |             data.append(sample)
22 |     return data
23 | 
24 | def extract_exp_id_from_json_filename(input_file):
25 |     '''
26 |     Find the unique experiment id. You should change this based on your custom file name
27 |     '''
28 |     start_idx = input_file.rfind('/') + 1
29 |     end_idx = -len('.json')
30 |     exp_id = input_file[start_idx:end_idx]
31 |     return exp_id
32 | 
33 | def main():
34 |     args = get_args()
35 | 
36 |     result_folder = args.output_dir if args.output_dir else "execution/results"
37 |     if not os.path.exists(result_folder):
38 |         os.makedirs(result_folder)
39 |     
40 |     if args.input_file:
41 |         exp_id = extract_exp_id_from_json_filename(args.input_file)
42 |         tasks = load_mnm_data_from_json(args.input_file)
43 |     else:
44 |         exp_id = "gt"
45 |         tasks = load_dataset("zixianma/mnms", split="test_human_verified_filtered")
46 |     # print(exp_id)
47 |     # print(len(tasks))
48 | 
49 |     final_result_folder = os.path.join(result_folder, f"{exp_id}-{args.plan_format}")
50 |     if args.plan_format == 'code':
51 |         executor = CodeExecutor(result_folder=final_result_folder)
52 |     else:
53 |         executor = Executor(result_folder=final_result_folder)
54 |     is_code_executor = isinstance(executor, CodeExecutor)
55 |     
56 |     output = os.path.join(result_folder, f"{exp_id}-{args.plan_format}-exec-results.json")
57 |     has_executed = []
58 |     # Record the ids of the examples that have been executed
59 |     if os.path.exists(output):
60 |         rf = open(output, "r")
61 |         for line in rf:
62 |             data = json.loads(line)
63 |             has_executed.append(data["id"])
64 |         rf.close()
65 |     
66 |     wf = open(output, "a")
67 |     for task in tasks:
68 |         idx = task['id']
69 |         # Skip the examples that have been executed
70 |         if idx in has_executed:
71 |             continue
72 |         print(f"Executing the plan for task {idx}...")
73 |         if is_code_executor:
74 |             plan = task['prediction'] if 'prediction' in task else task['code_str']
75 |         else:
76 |             plan = task['prediction'] if 'prediction' in task else eval(task['plan_str'])
77 |         print(plan)
78 |         result = executor.execute(idx, plan)
79 | 
80 |         if result['status']:
81 |             result_dict = {'id': idx, 'res': 1, 'msg': 'success'}
82 |         else:
83 |             print(result['message'])
84 |             result_dict = {'id': idx, 'res': 0, 'msg': result['message']}
85 |             
86 |         wf.write(json.dumps(result_dict) + "\n")
87 |         wf.flush()
88 | 
89 | 
90 | if __name__ == "__main__":
91 |     main()


--------------------------------------------------------------------------------
/mnms/execution/utils.py:
--------------------------------------------------------------------------------
  1 | import re
  2 | import json
  3 | import shutil
  4 | import pickle
  5 | from PIL import Image
  6 | from .tool_api import *
  7 | from .config import *
  8 | from ..constants import *
  9 | from copy import deepcopy
 10 | 
 11 | def find_node_patterns(text):
 12 |     # The regular expression pattern to find the <node-x>.xxxx in the args
 13 |     pattern = r"<node-\d+>\.[a-zA-Z]+"
 14 |     # Find all occurrences of the pattern in the text
 15 |     matches = re.findall(pattern, text)
 16 |     return matches
 17 | 
 18 | def save_output(task_idx, node_idx, output_dict, result_folder):
 19 |     new_dict = deepcopy(output_dict)
 20 |     full_result_path = os.path.join(result_folder, str(task_idx))
 21 |     if not os.path.exists(full_result_path):
 22 |         os.makedirs(full_result_path)
 23 |     for key, output in output_dict.items():
 24 |         # Determine the type of output and set the appropriate file extension
 25 |         if isinstance(output, Image.Image):
 26 |             file_path = os.path.join(full_result_path, f"node-{node_idx}.jpg")
 27 |             output.save(file_path)
 28 |             new_dict[key] = file_path
 29 | 
 30 |     # print("Output dict to be saved:", new_dict)
 31 |     output_dict_path_json = os.path.join(full_result_path, f"node-{node_idx}.json")
 32 |     output_dict_path_pickle = os.path.join(full_result_path, f"node-{node_idx}.pkl")
 33 |     try:
 34 |         # Try saving the output dictionary in a json file
 35 |         with open(output_dict_path_json, "w") as file:
 36 |             json.dump(new_dict, file)
 37 |         return output_dict_path_json
 38 |     except TypeError as e:
 39 |         # If saving into json failes, remove the json file and save it as a pickle file
 40 |         if os.path.exists(output_dict_path_json):
 41 |             os.remove(output_dict_path_json)
 42 |         with open(output_dict_path_pickle, "wb") as file:
 43 |             pickle.dump(new_dict, file)
 44 |         return output_dict_path_pickle
 45 | 
 46 | 
 47 | def save_result(idx, output, result_folder):
 48 |     # Create the result directory if it doesn't exist
 49 |     if not os.path.exists(result_folder):
 50 |         os.makedirs(result_folder)
 51 | 
 52 |     # Create the subfolder inside the result folder
 53 |     subfolder_path = os.path.join(result_folder, str(idx))
 54 |     if not os.path.exists(subfolder_path):
 55 |         os.makedirs(subfolder_path)
 56 | 
 57 |     # Move each file from the output to the subfolder
 58 |     for file_path in output.values():
 59 |         if os.path.exists(file_path):
 60 |             # Extract the filename from the path
 61 |             file_name = os.path.basename(file_path)
 62 |             # Define the new path for the file
 63 |             new_path = os.path.join(subfolder_path, file_name)
 64 |             # Move the file
 65 |             shutil.move(file_path, new_path)
 66 |         else:
 67 |             print(f"File not found: {file_path}")
 68 |     
 69 |     output_file_path = os.path.join(subfolder_path, 'result.json')
 70 |     with open(output_file_path, 'w') as outfile:
 71 |         json.dump(output, outfile, indent=4)
 72 | 
 73 | def execute_node(node):
 74 |     if node['name'] in TOOL_METADATA:
 75 |         func_name = node['name'].replace(' ', '_')
 76 |         # Call the corresponding tool function and store the value in result
 77 |         func = globals()[func_name]
 78 |         result = func(**node['args'])
 79 |     else:
 80 |         raise ValueError(f"Unknown tool: {node['name']}")
 81 |     return result
 82 | 
 83 | def pipeline(task_id, task_nodes, result_folder=RESULT_PATH):
 84 |     '''
 85 |     Execute a list of tool nodes, and save the tool outputs.
 86 |     '''
 87 |     outputs = {}
 88 |     outputs_path = {}
 89 |     full_result_path = os.path.join(result_folder, str(task_id))
 90 |     for node in task_nodes:
 91 |         # Update task arguments with outputs from previous tasks
 92 |         processed_node = deepcopy(node)
 93 |         for arg_key, arg_value in node['args'].items():
 94 |             if isinstance(arg_value, str) and arg_value.find('<node-') > -1:
 95 |                 # This arg value comes from last node's output
 96 |                 patterns = find_node_patterns(arg_value)
 97 |                 for pattern in patterns:
 98 |                     # Extract node id from the string
 99 |                     node_id = pattern[6]  
100 |                     start = pattern.find('.')
101 |                     last_node_out_arg_name = pattern[start+1:]
102 | 
103 |                     last_node_out_path_json = os.path.join(full_result_path, f"node-{node_id}.json")
104 |                     last_node_out_path_pickle = os.path.join(full_result_path, f"node-{node_id}.pkl")
105 |                     
106 |                     if os.path.exists(last_node_out_path_json): 
107 |                         # Find saved intermediate output
108 |                         print(f'finding last output at {last_node_out_path_json}')
109 |                         last_node_out = json.load(open(last_node_out_path_json, "r"))
110 |                         # Only replace <node-i>.text with last output's text and directly use last output otherwise
111 |                         new_arg_value = processed_node['args'][arg_key].replace(pattern, last_node_out[last_node_out_arg_name]) if last_node_out_arg_name == "text" else  last_node_out[last_node_out_arg_name]
112 |                         processed_node['args'][arg_key] = new_arg_value 
113 |                     elif os.path.exists(last_node_out_path_pickle):
114 |                         print(f'finding last output at {last_node_out_path_pickle}')
115 |                         last_node_out = pickle.load(open(last_node_out_path_pickle, "rb"))
116 |                         new_arg_value = processed_node['args'][arg_key].replace(pattern, last_node_out[last_node_out_arg_name]) if last_node_out_arg_name == "text" else  last_node_out[last_node_out_arg_name]
117 |                         processed_node['args'][arg_key] = new_arg_value 
118 |                     else:
119 |                         raise ValueError(f"Output from node {node_id} not found for task {task_id}")
120 |             elif isinstance(arg_value, str) and arg_value.endswith(('.jpg','.png', '.jpeg', '.flac')):
121 |                 # If the input isn't generated by the tool, but it's a file path, then we need to get the file path from the data folder
122 |                 processed_node['args'][arg_key] = get_full_path_data(arg_value)  
123 |             # print(processed_node['args'][arg_key])
124 |         # Execute the task
125 |         result = execute_node(processed_node)
126 |         # Save the output to result folder and return the path
127 |         output_path = save_output(task_id, node['id'], result, result_folder=result_folder) 
128 |         outputs[str(node['id'])] = result
129 |         outputs_path[str(node['id'])] = output_path
130 | 
131 |     return {"value": outputs, "path": outputs_path}


--------------------------------------------------------------------------------
/mnms/parser.py:
--------------------------------------------------------------------------------
 1 | import json
 2 | 
 3 | class Parser:
 4 |     def __init__(self, plan_mode):
 5 |         self.plan_mode = plan_mode
 6 | 
 7 |     def parse(self, response):
 8 |         if isinstance(response, dict) and 'content' in response:
 9 |             response = response['content']
10 |        
11 |         oring_content = response.replace("\n", "")
12 |         oring_content = oring_content.replace("\_", "_")
13 |         content = oring_content.replace("\\", "")
14 |         
15 |         try:
16 |             start_pos = content.find("RESULT #:")
17 |             if start_pos != -1:
18 |                 content = content[start_pos+len("RESULT #:"):]
19 | 
20 |             start_pos = content.find("{")
21 |             if start_pos != -1:
22 |                 content = content[start_pos:]
23 | 
24 |             end_pos = content.rfind("}")
25 |             if end_pos != -1:
26 |                 content = content[:end_pos+1]
27 | 
28 |             content = json.loads(content)
29 |             return {'status': True, 'content': content['nodes'] if (self.plan_mode == 'multi-step' and 'nodes' in content) else content, 'message': 'Parsing succeeded.', 'error_code': ''}
30 |         except json.JSONDecodeError as err:
31 |             return {'status': False, 'content': content, 'message': f"{type(err)}: {err}.", 'error_code': 'json'} 
32 |         except Exception as err:
33 |             return {'status': False, 'content': content, 'message': f"Unexpected {type(err)}: {err}.", 'error_code': 'unknown'}
34 | 
35 | 
36 | 
37 | class CodeParser(Parser):
38 |     def __init__(self, plan_mode):
39 |         super().__init__(plan_mode)
40 | 
41 |     def parse(self, response):
42 |         if isinstance(response, dict) and 'content' in response:
43 |             response = response['content']
44 |         oring_content = response.replace("\_", "_")
45 |         content = oring_content.replace("\\", "")
46 |         
47 |         try:
48 |             start_pos = content.find("RESULT #:")
49 |             if start_pos != -1:
50 |                 content = content[start_pos+len("RESULT #:"):]
51 |             
52 |             start_pos = content.find("```python")
53 |             if start_pos != -1:
54 |                 content = content[start_pos+len("```python"):]
55 | 
56 |             end_pos = content.find("```")
57 |             if end_pos != -1:
58 |                 content = content[:end_pos]
59 |             
60 |             if start_pos == -1 or end_pos == -1:
61 |                 return {'status': False, 'content': content, 'message': 'Program is NOT enclosed in ```python``` properly.', 'error_code': 'unknown'}
62 |             if len(content) > 0:
63 |                 compile(content, "prog.py", "exec")
64 |                 return {'status': True, 'content': content, 'message': 'Parsing succeeded.', 'error_code': ''}
65 |             else:
66 |                 return {'status': False, 'content': content, 'message': "The content is empty, or it failed to parse the content correctly.", 'error_code': 'unknown'}
67 |         except Exception as err:
68 |             return {'status': False, 'content': content, 'message': f"Unexpected {type(err)}: {err}.", 'error_code': 'unknown'}
69 | 
70 | 
71 | def main():
72 |     parser = CodeParser()
73 |     program = """RESULT #:```python\n"""
74 |     program += """def solve():\n"""
75 |     program += """    output0 = text_generation(prompt="Would you rather have an Apple Watch - or a BABY?")\n"""  
76 |     program += """    output1 = text_summarization(text=output0["text"])\n"""
77 |     program += """    return output1\n""" 
78 |     program += """```"""
79 |     print(program)
80 |     results = parser.parse(program)
81 |     print(results)
82 | 
83 |     parser = Parser()
84 |     # msg = """# RESULT #:\n\nTHOUGHT 0: First, I need to search for the movie 'The Shape of Water' from 2017 to gather information.\nACTION 0: {\"id\": 0, \"name\": \"search movie\", \"args\": {\"movie_title\": \"The Shape of Water\", \"movie_year\": 2017}} \n\n# Now you can execute the above action to proceed."""
85 |     msg = """"# RESULT #:\n\n{\n  \"nodes\": [\n    {\n      \"id\": 0,\n      \"name\": \"get date fact\",\n      \"args\": {\n        \"date\": \"May 7\"\n      }\n    }\n  ]\n}"""
86 |     print(msg)
87 |     results = parser.parse(msg)
88 |     print(results)
89 | 
90 | if __name__ == '__main__':
91 |     main()


--------------------------------------------------------------------------------
/mnms/prompt.py:
--------------------------------------------------------------------------------
  1 | import json
  2 | from typing import Dict, List, Optional, Union
  3 | from mnms.constants import TOOL_METADATA, DEMO_EXAMPLES, REACT_DEMO_EXAMPLES
  4 | 
  5 | class PlanPrompt:
  6 |     def __init__(self, instruction, tool_metadata, demos, requirements):
  7 |         """Generate a planning prompt that consists of instruction, tool metadat, demos and requirements.
  8 | 
  9 |         Args:
 10 |             query: the query of the user
 11 |         Returns:
 12 |             str: the generated prompt.
 13 |         """
 14 |         self.instruction = instruction
 15 |         self.tool_metadata = tool_metadata
 16 |         self.demos = demos
 17 |         self.requirements = requirements
 18 | 
 19 |     def get_prompt_for_curr_query(self, query):
 20 |         """(Abstract method) Generate a prompt based on the received query.
 21 | 
 22 |         Args:
 23 |             query: the query of the user
 24 |         Returns:
 25 |             str: the generated prompt.
 26 |         """
 27 |         pass
 28 | 
 29 | class JsonGenPrompt(PlanPrompt):
 30 |     def __init__(self):
 31 |         tool_metadata = "# TOOL LIST #:\n"
 32 |         for tool, tool_meta in TOOL_METADATA.items():
 33 |             input_list = ', '.join([mod for mod in tool_meta['input']['arg_name']])
 34 |             output_list = ', '.join([mod for mod in tool_meta['output']['arg_name']])
 35 |             tool_metadata += tool + f": {tool_meta['description']} "
 36 |             tool_metadata += "Its input includes " + str(input_list) + ", and output includes " + str(output_list) + ".\n"
 37 |         instruction = """\n# GOAL #: Based on the above tools, I want you to generate the tool nodes to solve the # USER REQUEST #. """
 38 |         instruction += """Each tool node specifies the name and argument of a tool to call and must be in a strict JSON format, like: 
 39 |             {
 40 |                 "nodes": [{
 41 |                     "id": an integer id of the tool, starting from 0, 
 42 |                     "name": "tool name must be from # TOOL LIST #", 
 43 |                     "args": { a dictionary of arguments for the tool. Either original text, or user-mentioned filename, or tag '<node-j>.text' (start from 0) to refer to the text output of the j-th node. }
 44 |                     }]
 45 |             } """ 
 46 |        
 47 |         demo_examples = DEMO_EXAMPLES
 48 |         demos = ""
 49 |         for demo in demo_examples:
 50 |             demo["result"] = {
 51 |                 "nodes": demo["nodes"]
 52 |             }
 53 |             demos += f"""\n# EXAMPLE #:\n# USER REQUEST #: {demo["user_request"]}\n# RESULT #: {json.dumps(demo["result"])}"""
 54 |         requirements = """\n\n# REQUIREMENTS #:"""
 55 |         requirements += """\n1. The generated tool nodes can resolve the given user request # USER REQUEST # perfectly. Tool name must be selected from # TOOL LIST #;"""
 56 |         requirements += """\n2. The arguments of a tool must be the same number, modality, and format specified in # TOOL LIST #;"""
 57 |         requirements += """\n3. Use as few tools as possible.""" 
 58 |         super().__init__(instruction, tool_metadata, demos, requirements)
 59 | 
 60 |     def get_prompt_for_curr_query(self, query):
 61 |         request = f"""\n\n# USER REQUEST #: {query}\nNow please generate your result in a strict JSON format:\n# RESULT #:"""
 62 |         return  self.tool_metadata + self.instruction + self.requirements + self.demos + request
 63 |     
 64 | 
 65 | 
 66 | class ReACTPrompt(PlanPrompt):
 67 |     def __init__(self):
 68 |         tool_metadata = "# TOOL LIST #:\n"
 69 |         for tool, tool_meta in TOOL_METADATA.items():
 70 |             input_list = ', '.join([mod for mod in tool_meta['input']['arg_name']])
 71 |             output_list = ', '.join([mod for mod in tool_meta['output']['arg_name']])
 72 |             tool_metadata += tool + f": {tool_meta['description']} "
 73 |             tool_metadata += "Its input includes " + str(input_list) + ", and output includes " + str(output_list) + ".\n"
 74 |         instruction = """\n# GOAL #: Based on the above tools, I want you to reason about how to solve the # USER REQUEST # and generate the actions step by step. """
 75 |         instruction += """Each action specifies the name and arguments of a tool to call and must be in a strict JSON format, like: 
 76 |             {
 77 |                 "id": an integer id of the tool, starting from 0, 
 78 |                 "name": "tool name must be from # TOOL LIST #", 
 79 |                 "args": { a dictionary of arguments for the tool. Either original text, or user-mentioned filename, or tag '<node-j>.text' (start from 0) to refer to the text output of the j-th node.}
 80 |             } """ 
 81 | 
 82 |         demo_examples = REACT_DEMO_EXAMPLES
 83 |         demos = ""
 84 |         for demo in demo_examples:
 85 |             demo["result"] = {
 86 |                 "nodes": demo["nodes"]
 87 |             }
 88 |             result = ""
 89 |             for i, node in enumerate(demo['nodes']):
 90 |                 result += f"""THOUGHT {i}: {"First" if i == 0 else f"Based on the user query and OBSERVATION {i-1}, then"}, I need to perform {node["name"]}.\n"""
 91 |                 no_output_node = {"id": node['id'], "name": node['name'], "args": node['args']}
 92 |                 # call the tool '{node["name"]}' with the arguments {json.dumps(node["args"])}
 93 |                 result += f"""ACTION {i}: {json.dumps(no_output_node)}\n""" 
 94 |                 if i < len(demo['nodes']) - 1:
 95 |                     obs_str = f"{node['output']}"
 96 |                     result += f"""OBSERVATION {i}: {obs_str}\n"""
 97 |             demos += f"""\n# EXAMPLE #:\n# USER REQUEST #: {demo["user_request"]}\n# RESULT #: \n{result}""" 
 98 |         requirements = """\n\n# REQUIREMENTS #:"""
 99 |         requirements += """\n1. The generated actions can resolve the given user request # USER REQUEST # perfectly. Tool name must be selected from # TOOL LIST #;"""
100 |         requirements += """\n2. The arguments of a tool must be the same number, modality, and format specified in # TOOL LIST #;"""
101 |         requirements += """\n3. Use as few tools as possible.""" 
102 |         super().__init__(instruction, tool_metadata, demos, requirements)
103 | 
104 |     def get_prompt_for_curr_query(self, query):
105 |         request = f"""\n# USER REQUEST #: {query}\nNow please generate only THOUGHT 0 and ACTION 0 in RESULT:\n# RESULT #:\n"""
106 |         return  self.tool_metadata + self.instruction + self.requirements + self.demos + request
107 | 
108 | 
109 | 
110 | def format_python_str(input_str):
111 |     """
112 |     Turns "<node-0>.text" into "{output0['text']}" for proper formatting in python code
113 |     """
114 |     import re
115 |     pattern = r"<node-\d+>\.[a-zA-Z]+"
116 |    
117 |     # Find all occurrences of the pattern in the text
118 |     matches = re.findall(pattern, input_str)
119 |     res_str = input_str
120 |     for match in matches:
121 |         # Extract node id from the string
122 |         node_id = match[6]  
123 |         start = match.find('.')
124 |         last_node_out_arg_name = match[start+1:]
125 |         
126 |         res_str = res_str.replace(match, "{" + f"output{node_id}['{last_node_out_arg_name}']" + "}") 
127 |     return res_str
128 |     
129 | def nodes_to_program(nodes):
130 |     """
131 |     Turns the json-format nodes into a python program named solve()
132 |     Example:
133 |     Input:
134 |     [{"id": 0, "name": "image captioning", "args": {"image": "2327921.jpg"}}, 
135 |     {"id": 1, "name": "text summarization", "args": {"text": output0["text"]}}]
136 |     Output:
137 |     def solve():
138 |         output0 = image_captioning(image="2327921.jpg")
139 |         output1 = text_summarization(text=output0['text'])
140 |         result = {0: output0, 1: output1}
141 |         return result
142 |     """
143 |     code = "def solve():\n"
144 |     outputs = []
145 |     for node in nodes:
146 |         node_name = "output{}".format(node["id"])
147 |         tool_name = node["name"].replace(" ", "_")
148 |         args = []
149 |         for k, v in node["args"].items():
150 |             print(node["args"])
151 |             arg_name = k
152 |             v = v.replace('"', "'")
153 |             
154 |             if v.find("<node-") > -1:
155 |                 if k == "text":
156 |                     arg_value = 'f"' + format_python_str(v) + '"'
157 |                 else:
158 |                     input_node_id = v[6]
159 |                     input_key = v[v.find('.') + 1:]
160 |                     arg_value = f"output{input_node_id}['{input_key}']"
161 |             else:
162 |                 arg_value = f'"{v}"'
163 | 
164 |             args.append(f"{arg_name}={arg_value}")
165 | 
166 |         arg_str = ", ".join(args)
167 |         code += f"    {node_name} = {tool_name}({arg_str})\n"
168 |         outputs.append(f'{node["id"]}: output{node["id"]}')
169 | 
170 |     final_output_str = 'result = {' + ', '.join(outputs) + '}'
171 |     code += f'    {final_output_str}\n'
172 |     code += '    return result'
173 |     return code.strip()
174 | 
175 | class CodeGenPrompt(PlanPrompt):
176 |     def __init__(self):
177 |         tool_metadata = "# TOOL LIST #:\n"
178 |         for tool, tool_meta in TOOL_METADATA.items():
179 |             input_list = ', '.join([mod for mod in tool_meta['input']['arg_name']])
180 |             output_list = ', '.join([mod for mod in tool_meta['output']['arg_name']])
181 |             tool_func_str = tool.replace(' ', '_') + f'({input_list}) -> {output_list}: {tool_meta["description"]}\n'
182 |             tool_metadata += tool_func_str
183 | 
184 |         instruction = """\n# GOAL #: Based on the above tools, I want you to generate a python program to solve the # USER REQUEST #."""
185 |        
186 |         demo_examples = DEMO_EXAMPLES
187 |         demos = ""
188 |         for demo in demo_examples:
189 |             program = """```python\n"""
190 |             program += nodes_to_program(demo['nodes'])
191 |             program += """\n```"""
192 |             demos += f"""\n# EXAMPLE #:\n# USER REQUEST #: {demo["user_request"]}\n# RESULT #: {program}"""
193 |         requirements = """\n\n# REQUIREMENTS #:"""
194 |         requirements += """\n1. The generated program can resolve the given user request # USER REQUEST # perfectly. The functions must be selected from # TOOL LIST #; """
195 |         requirements += """\n2. The arguments of a function must be the same number, modality, and format specified in # TOOL LIST #;"""
196 |         requirements += """\n3. Use as few tools as possible."""
197 |         super().__init__(instruction, tool_metadata, demos, requirements)
198 | 
199 |     def get_prompt_for_curr_query(self, query):
200 |         request = f"""\n\n# USER REQUEST #: {query}\nNow please generate your program enclosed in ```python ```:\n# RESULT #:"""
201 |         return  self.tool_metadata + self.instruction + self.requirements + self.demos + request
202 | 
203 | 
204 | class FeedbackPrompt:
205 |     def __init__(self, plan_mode, action_format):
206 |         self.plan_mode = plan_mode
207 |         self.action_format = action_format
208 |         if self.plan_mode == "step-by-step":
209 |             self.keyword = "ACTION" 
210 |         else: 
211 |             if self.action_format == "code":
212 |                 self.keyword = "program enclosed in ```python ``` in # RESULT #"
213 |             else:
214 |                 self.keyword = "# RESULT #"
215 |         
216 |         self.default_req_msg = f"\nPlease try generating the {self.keyword} again to fix the error. Or, reply with TERMINATE only if you believe this error is not fixable."
217 |         self.msg_prefix = "OBSERVATION: " if self.plan_mode == "step-by-step" else ""
218 | 
219 |     def get_prompt(self, stage, error_status, error_msg, error_code='unknown'):
220 |         if error_status:
221 |             if self.plan_mode == "step-by-step":
222 |                 return self.msg_prefix + error_msg + "\nIf the request has been fulfilled, please reply with TERMINATE, otherwise please generate the next THOUGHT and ACTION."
223 |             elif self.plan_mode == "multi-step":
224 |                 return self.msg_prefix + error_msg
225 |             else:
226 |                 raise NotImplementedError(f"{self.plan_mode} has not been implemented.") 
227 |         else:
228 |             if stage == "parsing":
229 |                 if error_code == "json":
230 |                     req_msg = f"\nPlease format the {self.keyword} to a strict JSON format " + "that starts with left curly bracket and ends with right curly bracket."
231 |                     req_msg += """\nRequirements:\n1. Do not change the information in nodes;\n2. Consider changing double quotes to single quotes or vice versa where applicable;\n3. Consider removing extra curly brackets if any;\n4. Don't tolerate any possible irregular formatting to ensure that the generated content can be converted by json.loads()."""
232 |                 else: # unknown, or any other error codes
233 |                     req_msg = self.default_req_msg
234 |                 return self.msg_prefix + error_msg + req_msg
235 |             elif stage == "verification":
236 |                 if error_code == 'unknown':
237 |                     return self.msg_prefix + error_msg + self.default_req_msg
238 |                 else:
239 |                     return self.msg_prefix + error_msg + f"\nPlease try again and fix the {error_code} in the {self.keyword} while keeping other parts the same."
240 |             elif stage == "execution":
241 |                 return self.msg_prefix + error_msg + self.default_req_msg
242 |            
243 |                 
244 | 
245 | 


--------------------------------------------------------------------------------
/mnms/run_plan_agent.py:
--------------------------------------------------------------------------------
  1 | import json
  2 | import os
  3 | import argparse
  4 | from typing import Dict, Optional, Union
  5 | from autogen.agentchat import AssistantAgent
  6 | from datasets import load_dataset
  7 | from mnms.constants import *
  8 | from mnms.agent import MnmsUserAgent, MnmsUserAgentLocal
  9 | from mnms.prompt import FeedbackPrompt, JsonGenPrompt, ReACTPrompt, CodeGenPrompt
 10 | from mnms.parser import Parser, CodeParser
 11 | from mnms.verifier import Verifier, CodeVerifier
 12 | from mnms.execution.executor import Executor, CodeExecutor
 13 | from mnms.execution.config import *
 14 | 
 15 | 
 16 | 
 17 | def get_args():
 18 |     parser = argparse.ArgumentParser()
 19 |     parser.add_argument('--plan-mode', default='multi-step', type=str, help="use global or local planning mode", choices=['multi-step', 'step-by-step'])
 20 |     parser.add_argument('--action-format', default='json', type=str, help="use dict or code as the format of the action", choices=['json', 'code'])
 21 |     parser.add_argument('--model', default='gpt-4-1106-preview', type=str, help="a string representing a unique model.")
 22 |     parser.add_argument('--verify', action='store_true', help="whether to use feedback from verification or not.")
 23 |     parser.add_argument('--execute', action='store_true', help="whether to use feedback from execution or not.")
 24 |     parser.add_argument('--max-reply', default=0, type=int, help="the maximum number of replies after the first attempt.")
 25 |     parser.add_argument('--seed', default=42, type=int, help="the random seed used in llm prediction.")
 26 |     parser.add_argument('--input-file', default=None, type=str, help="the file to import the input from.")
 27 |     parser.add_argument('--exp-id', default=None, type=str, help="a unique string for identifying the current experiment.")
 28 |     parser.add_argument('--simulate', action='store_true', help="only print the initial planning prompt instead of actually running planning agent.")
 29 |     parser.add_argument('--output-dir', default='prediction', type=str, help="the directory to save outputs to.")
 30 |     args = parser.parse_args()
 31 |     return args
 32 | 
 33 | def read_data_from_file(input_file):
 34 |     data = []
 35 |     rf = open(input_file, "r")
 36 |     for line in rf:
 37 |         example = json.loads(line)
 38 |         data.append(example)
 39 |     rf.close()
 40 |     return data
 41 | 
 42 | def checks_terminate_message(msg):
 43 |     if isinstance(msg, str):
 44 |         return msg.find("TERMINATE") > -1
 45 |     elif isinstance(msg, dict) and 'content' in msg:
 46 |         return msg['content'].find("TERMINATE") > -1
 47 |     else:
 48 |         print(type(msg), msg)
 49 |         raise NotImplementedError
 50 |     
 51 | def main():
 52 |     args = get_args()
 53 |     
 54 |     if args.input_file:
 55 |         input_examples = read_data_from_file(args.input_file)
 56 |     else:
 57 |         input_examples = load_dataset("zixianma/mnms", split="test_human_verified_filtered")
 58 | 
 59 |     if args.model.find('gpt') > -1:
 60 |         config_list = [{
 61 |             'model': args.model,
 62 |             'api_key': os.getenv("OPENAI_API_KEY"),
 63 |         }]
 64 |     elif args.model in ['meta-llama/Llama-2-70b-chat-hf', "google/gemma-7b-it", "mistralai/Mixtral-8x7B-Instruct-v0.1"]:
 65 |         config_list = [{
 66 |             "model": args.model,
 67 |             "base_url": "https://api.deepinfra.com/v1/openai",
 68 |             "api_key": os.getenv("DEEPINFRA_API_KEY")
 69 |         }]
 70 |     elif args.model.find('gemini') > -1:
 71 |         config_list = [{
 72 |             "model": args.model,
 73 |             "api_type": "google",
 74 |             "api_key": os.getenv("GOOGLE_API_KEY")
 75 |         }]
 76 |     else:
 77 |         config_list = [{
 78 |             "model": args.model,
 79 |             "base_url": "http://localhost:8000/v1",
 80 |             "api_key": "NULL"
 81 |         }]
 82 |     
 83 |     print(config_list)
 84 |     llm_config = {
 85 |         "seed": args.seed,
 86 |         "config_list": config_list,
 87 |     }
 88 | 
 89 |     # Set up output json file 
 90 |     run_id = f"{args.exp_id + '-' if args.exp_id else ''}{args.model}-{args.plan_mode}-{args.action_format}"
 91 |     run_id += f"{'-verify' if args.verify else ''}{'-execute' if args.execute else ''}-max-reply-{str(args.max_reply)}-seed-{str(args.seed)}"
 92 |     run_id = run_id.replace('/', '-')
 93 |     wf_name = f"{args.output_dir}/{run_id}.json"
 94 |     if not os.path.exists(args.output_dir):
 95 |         os.makedirs(args.output_dir, exist_ok=True)
 96 |     
 97 |     # Record ids of examples which we have run inference on (to skip these)
 98 |     has_inferenced = []
 99 |     if os.path.exists(wf_name):
100 |         rf = open(wf_name, "r")
101 |         for line in rf:
102 |             data = json.loads(line)
103 |             has_inferenced.append(data["id"])
104 |         rf.close()
105 |     wf = open(wf_name, "a")
106 | 
107 |     # Set up the planning agent with the correct prompt template and parser/verifier/executor modules
108 |     feedback_generator = FeedbackPrompt(plan_mode=args.plan_mode, action_format=args.action_format)
109 |     if args.plan_mode == "multi-step":
110 |         if args.action_format == "json":
111 |             prompt_generator = JsonGenPrompt() 
112 |             parser =  Parser(plan_mode=args.plan_mode)
113 |             verifier =  Verifier(tool_metadata=TOOL_METADATA) if args.verify else None 
114 |             executor =  Executor(result_folder=os.path.join(RESULT_PATH, run_id)) if args.execute else None 
115 |         elif args.action_format == "code":
116 |             prompt_generator = CodeGenPrompt() 
117 |             parser =  CodeParser(plan_mode=args.plan_mode)
118 |             verifier =  CodeVerifier(tool_metadata=TOOL_METADATA) if args.verify else None 
119 |             executor =  CodeExecutor(result_folder=os.path.join(RESULT_PATH, run_id)) if args.execute else None 
120 |         else:
121 |             raise NotImplementedError
122 |         
123 |         user = MnmsUserAgent(
124 |             name="user_agent",
125 |             human_input_mode='NEVER',
126 |             max_consecutive_auto_reply=args.max_reply,
127 |             is_termination_msg=checks_terminate_message,
128 |             prompt_generator=prompt_generator,
129 |             feedback_generator=feedback_generator,
130 |             parser=parser,
131 |             verifier=verifier,
132 |             executor=executor
133 |         )
134 |     elif args.plan_mode == "step-by-step":
135 |         if args.action_format == "json":
136 |             prompt_generator = ReACTPrompt()
137 |             feedback_generator = FeedbackPrompt(plan_mode=args.plan_mode, action_format="json")
138 |             verifier = Verifier(tool_metadata=TOOL_METADATA) if args.verify else None
139 |             parser = Parser(plan_mode=args.plan_mode)
140 |             executor = Executor(log_obs=True, result_folder=os.path.join(RESULT_PATH, run_id)) if args.execute else None 
141 |         else:
142 |             # This implementation does not support step-by-step code generation
143 |             raise NotImplementedError
144 | 
145 |         user = MnmsUserAgentLocal(
146 |             name="user_agent",
147 |             human_input_mode='NEVER',
148 |             max_consecutive_auto_reply=args.max_reply,
149 |             is_termination_msg=checks_terminate_message,
150 |             prompt_generator=prompt_generator,
151 |             feedback_generator=feedback_generator,
152 |             parser=parser,
153 |             verifier=verifier,
154 |             executor=executor
155 |         )
156 |     else:
157 |         raise NotImplementedError
158 | 
159 |     # Run the planning experiment
160 |     all_messages = {}
161 |     for idx, example in enumerate(input_examples):
162 |         if example['id'] in has_inferenced:
163 |             continue
164 |         else:
165 |             # Re-initialize the planner agent for each example to keep track of token usage for each run
166 |             planner = AssistantAgent(
167 |                 name="planner",
168 |                 llm_config=llm_config,
169 |                 # The default system message of the AssistantAgent is overwritten here
170 |                 # system_message=system_message
171 |             )
172 |             query = example['user_request']
173 |             print(query)
174 |             
175 |             try:
176 |                 user.initiate_chat(
177 |                     planner,
178 |                     message=query,
179 |                     task_id=example['id'],
180 |                     log_prompt_only=args.simulate
181 |                 )
182 |                 all_messages = planner.chat_messages
183 |             except Exception as e:
184 |                 print(e)
185 |                 print(f"skipping {example['id']}..")
186 |                 all_messages = {'error': e.message if hasattr(e, 'message') else f"{e}"}
187 | 
188 |             if not args.simulate and args.output_dir:
189 |                 if 'error' in all_messages:
190 |                     example['all_messages'] = all_messages
191 |                 else:
192 |                     messages = {agent.name: msg for agent, msg in all_messages.items()}
193 |                 
194 |                     # Exclude the initial prompt when saving the messages as it is very long
195 |                     messages['user_agent'] = messages['user_agent'][1:]
196 |                     example['all_messages'] = messages
197 |        
198 |                 example['prediction'] = user.current_plan 
199 |                 example['feedback_types'] = user.feedback_types 
200 |                 example['usage_summary'] = {'total': planner.client.total_usage_summary, 'actual': planner.client.actual_usage_summary}
201 |                 
202 |                 save_keys = ['id', 'user_request', 'prediction', 'all_messages', 'usage_summary', 'feedback_types']
203 |                 output_dict = {k: example[k] for k in save_keys}
204 |                 wf.write(json.dumps(output_dict) + "\n")
205 |                 wf.flush()
206 |             # print("FINAL usage:")
207 |             # planner.client.print_usage_summary() 
208 | 
209 |             user.reset()
210 |             planner.reset()
211 |                     
212 | if __name__ == '__main__':
213 |     main()


--------------------------------------------------------------------------------
/mnms/tool_graph.pkl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/RAIVNLab/mnms/89037953d58b14b126ad8679b1a5ce68e4fba21f/mnms/tool_graph.pkl


--------------------------------------------------------------------------------
/mnms/verifier.py:
--------------------------------------------------------------------------------
  1 | import networkx as nx
  2 | import pickle
  3 | import ast
  4 | import re
  5 | from mnms.constants import *
  6 | 
  7 | # the regular expression pattern to find the <node-x>.xxxx in the args
  8 | def find_node_patterns(text):
  9 |     # Regular expression pattern to find <node-x>.xxxx
 10 |     pattern = r"<node-\d+>\.[a-zA-Z]+"
 11 | 
 12 |     # Find all occurrences of the pattern in the text
 13 |     matches = re.findall(pattern, text)
 14 |     return matches
 15 | 
 16 | class Verifier:
 17 |     def __init__(self, tool_metadata):
 18 |         self.tools_dict = tool_metadata
 19 |         file = open("mnms/tool_graph.pkl",'rb')
 20 |         self.tool_graph = pickle.load(file)
 21 | 
 22 |     def create_graph_from_nodes(self, tool_nodes):
 23 |         G = nx.DiGraph()
 24 | 
 25 |         # Add nodes and edges based on the 'id' and 'args'
 26 |         for node in tool_nodes:
 27 |             node_id = node['id']
 28 |             G.add_node(node_id, name=node['name'], args=node['args'])
 29 | 
 30 |         for node in tool_nodes:
 31 |             # Check if there are any references to previous nodes in the args
 32 |             this_id = node['id']
 33 |             for arg_key, arg_value in node['args'].items():
 34 |                 
 35 |                 if isinstance(arg_value, str) and arg_value.startswith('<node-'):
 36 |                     # Extract the referenced node id
 37 |                     referenced_node_id = int(arg_value[6])
 38 |                     # Add an edge from the referenced node to the current node
 39 |                     G.add_edge(referenced_node_id, this_id)
 40 |         return G
 41 | 
 42 |     def verify_tool_name(self, node_name):
 43 |         status = True
 44 |         msgs = []
 45 |         err_codes = []
 46 |         if node_name not in self.tool_graph.nodes():
 47 |             status = False
 48 |             msg = f"{node_name} does not exist in the tool list."
 49 |             msgs.append(msg)
 50 |             err_codes.append("tools")
 51 |         output = {"status": status, "message": msgs, "error_code": err_codes}
 52 |         return output
 53 | 
 54 |     def verify_tool_args(self, node_name, node_args):
 55 |         status = True
 56 |         msgs = []
 57 |         err_codes = []
 58 |         # check arguments number
 59 |         input_args = self.tools_dict[node_name]['input']['arg_name']
 60 |         if len(input_args) != len(node_args):
 61 |             status = False
 62 |             msg = f"for {node_name}, {len(input_args)} arguments are expected, but there are {len(node_args)}."
 63 |             msgs.append(msg)
 64 |             err_codes.append(f"{node_name}'s arguments")
 65 |         # check for any missing arguments
 66 |         for arg in input_args:
 67 |             if arg not in node_args:
 68 |                 status = False
 69 |                 msg = f"'{arg}' is missing in the arguments of {node_name}."
 70 |                 msgs.append(msg)
 71 |                 err_codes.append(f"{node_name}'s arguments")
 72 |         # check for extra arguments
 73 |         for arg_name, arg_value in node_args.items():
 74 |             if arg_name not in input_args:
 75 |                 status = False
 76 |                 msg = f"'{arg_name}' should not appear in the arguments of {node_name}."
 77 |                 msgs.append(msg)
 78 |                 err_codes.append(f"{node_name}'s arguments")
 79 |         output = {"status": status, "message": msgs, "error_code": err_codes}
 80 |         return output
 81 | 
 82 |     def verify_tool_pair(self, prev_tool, curr_tool):
 83 |         status = True
 84 |         msgs = []
 85 |         err_codes = []
 86 |         # check edge existence
 87 |         if (prev_tool, curr_tool) not in self.tool_graph.edges():
 88 |             status = False
 89 |             msg = f"{curr_tool} should not follow {prev_tool}, " 
 90 |             reason = f"because it does not make sense to feed {self.tools_dict[prev_tool]['output']['desc']} into {curr_tool}, which expects the input to be {self.tools_dict[curr_tool]['input']['desc']}."
 91 |             msgs.append(msg + reason)
 92 |             err_codes.append("tools")
 93 |         output = {"status": status, "message": msgs, "error_code": err_codes}
 94 |         return output
 95 | 
 96 |     def verify_single_node(self, node):
 97 |         '''
 98 |         Verifies a single tool node by checking if the tool exists & if the tool's arguments follow the format correctly
 99 |         '''
100 |         statuses = []
101 |         msgs = []
102 |         err_codes = []
103 |         try:
104 |             node_name = node['name']
105 |             # check tool existence 
106 |             output = self.verify_tool_name(node_name)
107 |             status = output['status']
108 |             statuses.append(status)
109 |             if status:
110 |                 node_args = node['args']
111 |                 # check tool arguments
112 |                 output = self.verify_tool_args(node_name, node_args)
113 |                 statuses.append(output['status'])
114 |                 err_codes += output['error_code']
115 |                 msgs += output['message']
116 |             else:
117 |                 err_codes += output['error_code']
118 |                 msgs += output['message']
119 | 
120 |             unique_err_codes = set(err_codes)
121 |             unique_msgs = set(msgs)
122 |             final_status = False not in statuses
123 |             return {'status': final_status, 'message': 'Verification succeeded.' if final_status else  ' '.join(unique_msgs), 'error_code': ', '.join(unique_err_codes)}
124 |         except Exception as err:
125 |             return {'status': False, 'message': f"Unexpected {type(err)}: {err}.", 'error_code': 'unknown'}
126 | 
127 |     def verify(self, content):
128 |         '''
129 |         Verifies the content (i.e. a list of nodes) by checking a set of rules for each of the tool nodes and connections between nodes
130 |         Example:
131 |         [{'id': 0, 'name': 'visual question answering', 'args': {'image': '2415263.jpg', 'question': 'Who wears a shirt?'}}, 
132 |         {'id': 1, 'name': 'text generation', 'args': {'text': 'a possible storyline following the scene of <node-0>.text'}}, 
133 |         {'id': 2, 'name': 'text summarization', 'args': {'text': '<node-1>.text'}}]
134 |         '''
135 |         statuses = []
136 |         msgs = []
137 |         err_codes = []
138 |         try: 
139 |             graph = self.create_graph_from_nodes(content)
140 |             for node in graph.nodes():
141 |                 node_name = graph.nodes[node]['name']
142 |                 # check tool existence 
143 |                 output = self.verify_tool_name(node_name)
144 |                 status = output['status']
145 |                 statuses.append(status)
146 | 
147 |                 if status:
148 |                     node_args = graph.nodes[node]['args']
149 |                     # check tool args validity
150 |                     output = self.verify_tool_args(node_name, node_args)
151 |                     statuses.append(output['status'])
152 |                     err_codes += output['error_code']
153 |                     msgs += output['message']
154 | 
155 |                     # check for output format & reference inconsistency
156 |                     for arg_name, arg_value in node_args.items():
157 |                         if isinstance(arg_value, str) and arg_value.find('<node-') > -1:
158 |                             patterns = find_node_patterns(arg_value)
159 |                             if len(patterns) == 0:
160 |                                 statuses.append(False)
161 |                                 msg = f"{arg_value} has the wrong format. It should refer to a specific output of the last node i by <node-i>.key."
162 |                                 msgs.append(msg)
163 |                                 err_codes.append(f"{node_name}'s arguments")
164 |                             for pattern in patterns:
165 |                                 last_node_id = int(pattern[6])
166 |                                 last_node_name = graph.nodes[last_node_id]['name']
167 |                                 output_mods = self.tools_dict[last_node_name]['output']['arg_name']
168 |                                 
169 |                                 start = pattern.find('.')
170 |                                 last_node_output_mod = pattern[start+1:]
171 |                                 if last_node_output_mod not in output_mods:
172 |                                     statuses.append(False)
173 |                                     msg = f"{last_node_output_mod} is not in the outputs of {last_node_name}, which outputs {str(output_mods)}."
174 |                                     msgs.append(msg)
175 |                                     err_codes.append(f"{node_name}'s arguments")             
176 |                 else:
177 |                     err_codes += output['error_code']
178 |                     msgs += output['message']
179 |             # check edge existence
180 |             node_names = nx.get_node_attributes(graph, 'name')
181 |             for edge in graph.edges():
182 |                 src_name = node_names[edge[0]]
183 |                 tgt_name = node_names[edge[1]]
184 | 
185 |                 if src_name in self.tool_graph.nodes() and tgt_name in self.tool_graph.nodes():
186 |                     output = self.verify_tool_pair(src_name, tgt_name)
187 |                     status = output['status']
188 |                     statuses.append(status)
189 |                     err_codes += output['error_code']
190 |                     msgs += output['message']
191 |                     
192 |             unique_err_codes = set(err_codes)
193 |             unique_msgs = set(msgs)
194 |             final_status = False not in statuses
195 |             return {'status': final_status, 'message': 'Verification succeeded.' if final_status else  ' '.join(unique_msgs), 'error_code': ', '.join(unique_err_codes)}
196 |         except Exception as err:
197 |             return {'status': False, 'message': f"Unexpected {type(err)}: {err}.", 'error_code': 'unknown'}
198 | 
199 | 
200 | class CodeVerifier(Verifier):
201 |     def __init__(self, tool_metadata):
202 |         super().__init__(tool_metadata=tool_metadata)
203 | 
204 |     def get_tool_name_from_func_node(self, node):
205 |         if hasattr(node.func, 'id'): 
206 |             func_name = node.func.id
207 |             tool_name = func_name.replace('_', ' ')
208 |             return tool_name
209 |         return None
210 | 
211 |     def verify(self, content):
212 |         '''
213 |         Example input:
214 |         def solve():
215 |             output0 = image_classification(image="10084.jpg")
216 |             output1 = image_generation(text=f"a new, more detailed or stylized image of {output0['text']}")
217 |             result = {0: output0, 1: output1}
218 |             return result
219 |         '''
220 |         statuses = []
221 |         msgs = []
222 |         err_codes = []
223 |         parsed_code = ast.parse(content)
224 |         var_values = {}
225 |         valid_tool_names_and_args = {}
226 | 
227 |         try:
228 |             for node in ast.walk(parsed_code):
229 |                 # get var name to value mapping
230 |                 if isinstance(node, ast.Assign):
231 |                     for target in node.targets:
232 |                         if isinstance(target, ast.Name) and target.id :
233 |                             var_values[target.id] = node.value
234 |                 # Verify the name and arguments of each tool
235 |                 if isinstance(node, ast.Call):
236 |                     tool_name = self.get_tool_name_from_func_node(node)
237 |                     output = self.verify_tool_name(tool_name)
238 |                     status = output['status']
239 |                     statuses.append(status)
240 | 
241 |                     if status:
242 |                         tool_args = {}
243 |                         # assuming the tool takes in a list of keyword arguments
244 |                         for kwarg in node.keywords: 
245 |                             var_name = kwarg.arg
246 |                             if isinstance(kwarg.value, ast.Constant): 
247 |                                 # extract .value only if it's a constant
248 |                                 tool_args[var_name] = kwarg.value.value
249 |                             else:
250 |                                 # otherwise keep the orginal ast object
251 |                                 tool_args[var_name] = kwarg.value
252 |                                 
253 |                         # print(tool_args)
254 |                         valid_tool_names_and_args[tool_name] = tool_args
255 |                         # verify this tool's arguments
256 |                         output = self.verify_tool_args(tool_name, tool_args)
257 |                         statuses.append(output['status'])
258 |                         err_codes += output['error_code']
259 |                         msgs += output['message']
260 |                     else:
261 |                         err_codes += output['error_code']
262 |                         msgs += output['message']
263 |             
264 |             print(valid_tool_names_and_args)
265 |             # Verify edges and output reference
266 |             # valid_tool_names_and_args = {'image classification': {'image': '10084.jpg'}, 'image generation': {'text': <ast.JoinedStr object at 0x7f602cb370a0>}}
267 |             for tool_name, tool_args in valid_tool_names_and_args.items():
268 |                 for k, v in tool_args.items():
269 |                     last_tool_names = []
270 |                     last_node_out_keys = []
271 | 
272 |                     # We check for the following three common cases where a tool's input refers to last tool's output
273 |                     if isinstance(v, ast.Name): # v = output0 (missing reference, will fail verification)
274 |                         var_name = v.id
275 |                         last_node = var_values[var_name] 
276 |                         if isinstance(last_node, ast.Call):
277 |                             last_tool_names = [self.get_tool_name_from_func_node(last_node)]
278 |                             last_node_out_keys = [None] 
279 |                     elif isinstance(v, ast.Subscript): # v = output0["text"]
280 |                         var_name = v.value.id #  output0["text"] -> output0
281 |                         last_node = var_values[var_name] 
282 |                         if isinstance(last_node, ast.Call):
283 |                             last_tool_names = [self.get_tool_name_from_func_node(last_node)]
284 |                             last_node_out_keys = [v.slice.value] # output0["text"] -> ["text"]
285 |                     elif isinstance(v, ast.JoinedStr): # v = f"a new, more detailed or stylized image of {output0['text']}"
286 |                         subscripts = [val.value for val in v.values if isinstance(val, ast.FormattedValue)] # [<ast.Subscript object at 0x7f966d411b20>] i.e. output0['text']
287 |                         last_outs = [val.value.id for val in subscripts] # ["output0"]
288 |                         last_nodes = [var_values[out_name] for out_name in last_outs] # [<ast.Call object at 0x7f1b14580cd0>]
289 |                         last_tool_names = [self.get_tool_name_from_func_node(last_node) if isinstance(last_node, ast.Call) else None for last_node in last_nodes] # ["image classification"]
290 |                         last_node_out_keys = [subscript.slice.value for subscript in subscripts] # ["text"]
291 |                     
292 |                     for i, last_tool_name in enumerate(last_tool_names):
293 |                         if last_tool_name:
294 |                             if last_tool_name in self.tool_graph.nodes() and tool_name in self.tool_graph.nodes():
295 |                                 # check for edge existence between last tool and this current tool
296 |                                 output = self.verify_tool_pair(last_tool_name, tool_name)
297 |                                 statuses.append(output['status'])
298 |                                 err_codes += output['error_code']
299 |                                 msgs += output['message']
300 | 
301 |                                 output_keys = self.tools_dict[last_tool_name]['output']['arg_name']
302 |                                 last_node_out_key = last_node_out_keys[i]
303 |                                 if last_node_out_key:
304 |                                     if last_node_out_key not in output_keys:
305 |                                         status = False
306 |                                         statuses.append(status)
307 |                                         msg = f"{last_node_out_key} is not in the outputs of {last_tool_name}, which outputs {str(output_keys)}."
308 |                                         msgs.append(msg)
309 |                                         err_codes.append(f"{tool_name}'s arguments")
310 |                                 else:
311 |                                     statuses.append(False)
312 |                                     msg = f"{tool_name}'s argument has the wrong format. It should refer to a specific output of the last node by output['key'] where key can be one of {str(output_keys)}."
313 |                                     msgs.append(msg)
314 |                                     err_codes.append(f"{tool_name}'s arguments")                          
315 |             # print(statuses)
316 |             final_status = False not in statuses
317 |             unique_err_codes = set(err_codes)
318 |             unique_msgs = set(msgs)
319 |             return {'status': final_status, 'message': 'Verification succeeded.' if final_status else  ' '.join(unique_msgs), 'error_code': ', '.join(unique_err_codes)}
320 |         except Exception as err:
321 |             return {'status': False, 'message': f"Unexpected {type(err)}: {err}.", 'error_code': 'unknown'}
322 |         
323 | # def main():
324 |     
325 | #     verifier = Verifier(tool_metadata=TOOL_METADATA)
326 | #     nodes = [{'id': 0, 'name': 'visual question answering', 'args': {'image': '2415263.jpg', 'question': 'Who wears a shirt?'}}, {'id': 1, 'name': 'text generation', 'args': {'text': 'a possible storyline following the scene of <node-0>.text'}}, {'id': 2, 'name': 'text summarization', 'args': {'text': '<node-1>.text'}}]
327 | #     results = verifier.verify(nodes)
328 | #     print(nodes)
329 | 
330 | #     verifier = CodeVerifier(tool_metadata=TOOL_METADATA)
331 | #     program = """def solve():\n"""
332 | #     program += """    output0 = image_classification(image="10084.jpg")\n"""
333 | #     program += """    output1 = image_generation(text=f"a new, more detailed or stylized image that matches the classification of {output0['text']}")\n"""
334 | #     # program += """    output1 = text_generation(text="generate a prefix")\n"""
335 | #     # program += """    output2 = image_generation(text=f"{output1['text']} of {output0['text']}")\n"""
336 | #     program += """    result = {0: output0, 1: output1}\n"""
337 | #     program += """    return result""" 
338 | #     print(program)
339 | #     results = verifier.verify(program)
340 | 
341 | #     print(results)
342 | 
343 | 
344 | # if __name__ == '__main__':
345 | #     main()


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
 1 | augly==1.0.0
 2 | autogen==1.0.16
 3 | datasets==2.15.0
 4 | diffusers==0.24.0
 5 | easyocr==1.7.1
 6 | librosa==0.10.1
 7 | networkx==3.1
 8 | numpy==1.26.4
 9 | openai==1.14.3
10 | opencv_python_headless==4.9.0.80
11 | pandas==2.2.1
12 | Pillow==10.2.0
13 | pyautogen==0.2.11
14 | python_dateutil==2.8.2
15 | python_Levenshtein==0.25.0
16 | Requests==2.31.0
17 | scikit_learn==1.3.1
18 | textract==1.6.5
19 | torch==2.1.2
20 | transformers==4.36.0
21 | 
22 | 


--------------------------------------------------------------------------------