├── .gitignore ├── .gitmodules ├── LICENSE ├── README.md ├── assets └── images │ ├── AgentNetBench.png │ ├── agn_tool_fig.png │ ├── domain_distribution.png │ └── main_fig.png ├── data ├── README.md ├── cot-generate │ ├── .gitignore │ ├── README.md │ ├── gen_cot.py │ ├── gen_cot_example │ │ ├── images │ │ │ ├── 0.png │ │ │ ├── 1.png │ │ │ ├── 10.png │ │ │ ├── 11.png │ │ │ ├── 12.png │ │ │ ├── 13.png │ │ │ ├── 14.png │ │ │ ├── 2.png │ │ │ ├── 3.png │ │ │ ├── 4.png │ │ │ ├── 5.png │ │ │ ├── 6.png │ │ │ ├── 7.png │ │ │ ├── 8.png │ │ │ └── 9.png │ │ ├── raw_example.jsonl │ │ └── visualization │ │ │ ├── app.py │ │ │ └── templates │ │ │ └── index.html │ ├── merge_json.py │ ├── merge_to_jsonl.py │ ├── module │ │ ├── __init__.py │ │ ├── evaluator.py │ │ ├── generator.py │ │ ├── reflector.py │ │ └── reflector_with_prior_judge.py │ └── utils.py └── data-process │ ├── .gitignore │ ├── README.md │ ├── datasets │ ├── raw │ │ └── 20241106114028_e0599885@gmail.com_2c05b837-faae-4b35-8e89-71d4bc2cab66 │ │ │ ├── 2024-11-05 18-56-31.mp4 │ │ │ ├── a11y.jsonl │ │ │ ├── element.jsonl │ │ │ ├── event_buffer.jsonl │ │ │ ├── events.jsonl │ │ │ ├── html.jsonl │ │ │ ├── metadata.json │ │ │ ├── reduced_events_complete.jsonl │ │ │ ├── reduced_events_vis.jsonl │ │ │ ├── task_name.json │ │ │ └── top_window.jsonl │ ├── raw_trajs │ │ └── 20241106114028_e0599885@gmail.com_2c05b837-faae-4b35-8e89-71d4bc2cab66.json │ └── standardized │ │ └── 20241106114028_e0599885@gmail.com_2c05b837-faae-4b35-8e89-71d4bc2cab66.json │ ├── requirements.txt │ ├── scripts │ ├── extract_raw.sh │ └── raw_to_standardized.sh │ └── src │ ├── __init__.py │ ├── extract_raw.py │ ├── raw_to_standardized.py │ ├── schema │ ├── __init__.py │ ├── action.py │ └── trajectory.py │ └── utils │ ├── __init__.py │ └── image.py ├── evaluation └── agentnetbench │ ├── .gitignore │ ├── README.md │ ├── __init__.py │ ├── agent │ ├── aguvis.py │ ├── base_agent.py │ ├── opencua.py │ └── qwen25vl.py │ ├── eval.py │ ├── reeval.py │ ├── run.py │ ├── sample_data │ ├── images │ │ ├── s_5473959e0f6e21f7_1.png │ │ ├── s_5473959e0f6e21f7_10.png │ │ ├── s_5473959e0f6e21f7_11.png │ │ ├── s_5473959e0f6e21f7_2.png │ │ ├── s_5473959e0f6e21f7_3.png │ │ ├── s_5473959e0f6e21f7_5.png │ │ ├── s_5473959e0f6e21f7_6.png │ │ ├── s_5473959e0f6e21f7_7.png │ │ ├── s_5473959e0f6e21f7_8.png │ │ ├── s_5473959e0f6e21f7_9.png │ │ ├── s_7f27a11115e596eb_0.png │ │ ├── s_7f27a11115e596eb_1.png │ │ ├── s_7f27a11115e596eb_10.png │ │ ├── s_7f27a11115e596eb_12.png │ │ ├── s_7f27a11115e596eb_13.png │ │ ├── s_7f27a11115e596eb_3.png │ │ ├── s_7f27a11115e596eb_4.png │ │ ├── s_7f27a11115e596eb_5.png │ │ ├── s_7f27a11115e596eb_6.png │ │ ├── s_7f27a11115e596eb_7.png │ │ ├── s_7f27a11115e596eb_8.png │ │ ├── s_a96285eb665bef92_10.png │ │ ├── s_a96285eb665bef92_11.png │ │ ├── s_a96285eb665bef92_2.png │ │ ├── s_a96285eb665bef92_3.png │ │ ├── s_a96285eb665bef92_4.png │ │ ├── s_a96285eb665bef92_5.png │ │ ├── s_a96285eb665bef92_6.png │ │ ├── s_a96285eb665bef92_7.png │ │ ├── s_a96285eb665bef92_8.png │ │ ├── s_a96285eb665bef92_9.png │ │ ├── s_c53b113bf3e7d362_1.png │ │ ├── s_c53b113bf3e7d362_11.png │ │ ├── s_c53b113bf3e7d362_14.png │ │ ├── s_c53b113bf3e7d362_15.png │ │ ├── s_c53b113bf3e7d362_18.png │ │ ├── s_c53b113bf3e7d362_19.png │ │ ├── s_c53b113bf3e7d362_2.png │ │ ├── s_c53b113bf3e7d362_20.png │ │ ├── s_c53b113bf3e7d362_21.png │ │ ├── s_c53b113bf3e7d362_22.png │ │ ├── s_c53b113bf3e7d362_3.png │ │ ├── s_c53b113bf3e7d362_5.png │ │ ├── s_c53b113bf3e7d362_6.png │ │ ├── s_c53b113bf3e7d362_7.png │ │ ├── s_c53b113bf3e7d362_8.png │ │ ├── s_c53b113bf3e7d362_9.png │ │ ├── s_df0fd37049f470c2_10.png │ │ ├── s_df0fd37049f470c2_2.png │ │ ├── s_df0fd37049f470c2_3.png │ │ ├── s_df0fd37049f470c2_4.png │ │ ├── s_df0fd37049f470c2_6.png │ │ ├── s_df0fd37049f470c2_7.png │ │ └── s_df0fd37049f470c2_9.png │ ├── s_5473959e0f6e21f7.json │ ├── s_7f27a11115e596eb.json │ ├── s_a96285eb665bef92.json │ ├── s_c53b113bf3e7d362.json │ └── s_df0fd37049f470c2.json │ └── utils │ └── qwen_vl_utils.py └── model ├── README.md ├── inference ├── grounding_examples │ ├── 07JVNIBSB9.png │ ├── 0FOB4CLBT2.png │ ├── 6GNDSETVY9.png │ ├── OH23HT37NP.png │ ├── T5l1sFEhSy.png │ ├── test0.json │ ├── test1.json │ ├── test2.json │ ├── test3.json │ └── test4.json └── huggingface_inference.py └── requirement.txt /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/.gitignore -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/.gitmodules -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/README.md -------------------------------------------------------------------------------- /assets/images/AgentNetBench.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/assets/images/AgentNetBench.png -------------------------------------------------------------------------------- /assets/images/agn_tool_fig.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/assets/images/agn_tool_fig.png -------------------------------------------------------------------------------- /assets/images/domain_distribution.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/assets/images/domain_distribution.png -------------------------------------------------------------------------------- /assets/images/main_fig.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/assets/images/main_fig.png -------------------------------------------------------------------------------- /data/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/README.md -------------------------------------------------------------------------------- /data/cot-generate/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/cot-generate/.gitignore -------------------------------------------------------------------------------- /data/cot-generate/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/cot-generate/README.md -------------------------------------------------------------------------------- /data/cot-generate/gen_cot.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/cot-generate/gen_cot.py -------------------------------------------------------------------------------- /data/cot-generate/gen_cot_example/images/0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/cot-generate/gen_cot_example/images/0.png -------------------------------------------------------------------------------- /data/cot-generate/gen_cot_example/images/1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/cot-generate/gen_cot_example/images/1.png -------------------------------------------------------------------------------- /data/cot-generate/gen_cot_example/images/10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/cot-generate/gen_cot_example/images/10.png -------------------------------------------------------------------------------- /data/cot-generate/gen_cot_example/images/11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/cot-generate/gen_cot_example/images/11.png -------------------------------------------------------------------------------- /data/cot-generate/gen_cot_example/images/12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/cot-generate/gen_cot_example/images/12.png -------------------------------------------------------------------------------- /data/cot-generate/gen_cot_example/images/13.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/cot-generate/gen_cot_example/images/13.png -------------------------------------------------------------------------------- /data/cot-generate/gen_cot_example/images/14.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/cot-generate/gen_cot_example/images/14.png -------------------------------------------------------------------------------- /data/cot-generate/gen_cot_example/images/2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/cot-generate/gen_cot_example/images/2.png -------------------------------------------------------------------------------- /data/cot-generate/gen_cot_example/images/3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/cot-generate/gen_cot_example/images/3.png -------------------------------------------------------------------------------- /data/cot-generate/gen_cot_example/images/4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/cot-generate/gen_cot_example/images/4.png -------------------------------------------------------------------------------- /data/cot-generate/gen_cot_example/images/5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/cot-generate/gen_cot_example/images/5.png -------------------------------------------------------------------------------- /data/cot-generate/gen_cot_example/images/6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/cot-generate/gen_cot_example/images/6.png -------------------------------------------------------------------------------- /data/cot-generate/gen_cot_example/images/7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/cot-generate/gen_cot_example/images/7.png -------------------------------------------------------------------------------- /data/cot-generate/gen_cot_example/images/8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/cot-generate/gen_cot_example/images/8.png -------------------------------------------------------------------------------- /data/cot-generate/gen_cot_example/images/9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/cot-generate/gen_cot_example/images/9.png -------------------------------------------------------------------------------- /data/cot-generate/gen_cot_example/raw_example.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/cot-generate/gen_cot_example/raw_example.jsonl -------------------------------------------------------------------------------- /data/cot-generate/gen_cot_example/visualization/app.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/cot-generate/gen_cot_example/visualization/app.py -------------------------------------------------------------------------------- /data/cot-generate/gen_cot_example/visualization/templates/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/cot-generate/gen_cot_example/visualization/templates/index.html -------------------------------------------------------------------------------- /data/cot-generate/merge_json.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/cot-generate/merge_json.py -------------------------------------------------------------------------------- /data/cot-generate/merge_to_jsonl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/cot-generate/merge_to_jsonl.py -------------------------------------------------------------------------------- /data/cot-generate/module/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/cot-generate/module/evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/cot-generate/module/evaluator.py -------------------------------------------------------------------------------- /data/cot-generate/module/generator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/cot-generate/module/generator.py -------------------------------------------------------------------------------- /data/cot-generate/module/reflector.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/cot-generate/module/reflector.py -------------------------------------------------------------------------------- /data/cot-generate/module/reflector_with_prior_judge.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/cot-generate/module/reflector_with_prior_judge.py -------------------------------------------------------------------------------- /data/cot-generate/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/cot-generate/utils.py -------------------------------------------------------------------------------- /data/data-process/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/data-process/.gitignore -------------------------------------------------------------------------------- /data/data-process/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/data-process/README.md -------------------------------------------------------------------------------- /data/data-process/datasets/raw/20241106114028_e0599885@gmail.com_2c05b837-faae-4b35-8e89-71d4bc2cab66/2024-11-05 18-56-31.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/data-process/datasets/raw/20241106114028_e0599885@gmail.com_2c05b837-faae-4b35-8e89-71d4bc2cab66/2024-11-05 18-56-31.mp4 -------------------------------------------------------------------------------- /data/data-process/datasets/raw/20241106114028_e0599885@gmail.com_2c05b837-faae-4b35-8e89-71d4bc2cab66/a11y.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/data-process/datasets/raw/20241106114028_e0599885@gmail.com_2c05b837-faae-4b35-8e89-71d4bc2cab66/a11y.jsonl -------------------------------------------------------------------------------- /data/data-process/datasets/raw/20241106114028_e0599885@gmail.com_2c05b837-faae-4b35-8e89-71d4bc2cab66/element.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/data-process/datasets/raw/20241106114028_e0599885@gmail.com_2c05b837-faae-4b35-8e89-71d4bc2cab66/element.jsonl -------------------------------------------------------------------------------- /data/data-process/datasets/raw/20241106114028_e0599885@gmail.com_2c05b837-faae-4b35-8e89-71d4bc2cab66/event_buffer.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/data-process/datasets/raw/20241106114028_e0599885@gmail.com_2c05b837-faae-4b35-8e89-71d4bc2cab66/event_buffer.jsonl -------------------------------------------------------------------------------- /data/data-process/datasets/raw/20241106114028_e0599885@gmail.com_2c05b837-faae-4b35-8e89-71d4bc2cab66/events.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/data-process/datasets/raw/20241106114028_e0599885@gmail.com_2c05b837-faae-4b35-8e89-71d4bc2cab66/events.jsonl -------------------------------------------------------------------------------- /data/data-process/datasets/raw/20241106114028_e0599885@gmail.com_2c05b837-faae-4b35-8e89-71d4bc2cab66/html.jsonl: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/data-process/datasets/raw/20241106114028_e0599885@gmail.com_2c05b837-faae-4b35-8e89-71d4bc2cab66/metadata.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/data-process/datasets/raw/20241106114028_e0599885@gmail.com_2c05b837-faae-4b35-8e89-71d4bc2cab66/metadata.json -------------------------------------------------------------------------------- /data/data-process/datasets/raw/20241106114028_e0599885@gmail.com_2c05b837-faae-4b35-8e89-71d4bc2cab66/reduced_events_complete.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/data-process/datasets/raw/20241106114028_e0599885@gmail.com_2c05b837-faae-4b35-8e89-71d4bc2cab66/reduced_events_complete.jsonl -------------------------------------------------------------------------------- /data/data-process/datasets/raw/20241106114028_e0599885@gmail.com_2c05b837-faae-4b35-8e89-71d4bc2cab66/reduced_events_vis.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/data-process/datasets/raw/20241106114028_e0599885@gmail.com_2c05b837-faae-4b35-8e89-71d4bc2cab66/reduced_events_vis.jsonl -------------------------------------------------------------------------------- /data/data-process/datasets/raw/20241106114028_e0599885@gmail.com_2c05b837-faae-4b35-8e89-71d4bc2cab66/task_name.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/data-process/datasets/raw/20241106114028_e0599885@gmail.com_2c05b837-faae-4b35-8e89-71d4bc2cab66/task_name.json -------------------------------------------------------------------------------- /data/data-process/datasets/raw/20241106114028_e0599885@gmail.com_2c05b837-faae-4b35-8e89-71d4bc2cab66/top_window.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/data-process/datasets/raw/20241106114028_e0599885@gmail.com_2c05b837-faae-4b35-8e89-71d4bc2cab66/top_window.jsonl -------------------------------------------------------------------------------- /data/data-process/datasets/raw_trajs/20241106114028_e0599885@gmail.com_2c05b837-faae-4b35-8e89-71d4bc2cab66.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/data-process/datasets/raw_trajs/20241106114028_e0599885@gmail.com_2c05b837-faae-4b35-8e89-71d4bc2cab66.json -------------------------------------------------------------------------------- /data/data-process/datasets/standardized/20241106114028_e0599885@gmail.com_2c05b837-faae-4b35-8e89-71d4bc2cab66.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/data-process/datasets/standardized/20241106114028_e0599885@gmail.com_2c05b837-faae-4b35-8e89-71d4bc2cab66.json -------------------------------------------------------------------------------- /data/data-process/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/data-process/requirements.txt -------------------------------------------------------------------------------- /data/data-process/scripts/extract_raw.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/data-process/scripts/extract_raw.sh -------------------------------------------------------------------------------- /data/data-process/scripts/raw_to_standardized.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/data-process/scripts/raw_to_standardized.sh -------------------------------------------------------------------------------- /data/data-process/src/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/data-process/src/__init__.py -------------------------------------------------------------------------------- /data/data-process/src/extract_raw.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/data-process/src/extract_raw.py -------------------------------------------------------------------------------- /data/data-process/src/raw_to_standardized.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/data-process/src/raw_to_standardized.py -------------------------------------------------------------------------------- /data/data-process/src/schema/__init__.py: -------------------------------------------------------------------------------- 1 | # Makes `schema` a package for local imports 2 | 3 | -------------------------------------------------------------------------------- /data/data-process/src/schema/action.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/data-process/src/schema/action.py -------------------------------------------------------------------------------- /data/data-process/src/schema/trajectory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/data-process/src/schema/trajectory.py -------------------------------------------------------------------------------- /data/data-process/src/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Makes `utils` a package for local imports 2 | 3 | -------------------------------------------------------------------------------- /data/data-process/src/utils/image.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/data/data-process/src/utils/image.py -------------------------------------------------------------------------------- /evaluation/agentnetbench/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/.gitignore -------------------------------------------------------------------------------- /evaluation/agentnetbench/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/README.md -------------------------------------------------------------------------------- /evaluation/agentnetbench/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /evaluation/agentnetbench/agent/aguvis.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/agent/aguvis.py -------------------------------------------------------------------------------- /evaluation/agentnetbench/agent/base_agent.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/agent/base_agent.py -------------------------------------------------------------------------------- /evaluation/agentnetbench/agent/opencua.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/agent/opencua.py -------------------------------------------------------------------------------- /evaluation/agentnetbench/agent/qwen25vl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/agent/qwen25vl.py -------------------------------------------------------------------------------- /evaluation/agentnetbench/eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/eval.py -------------------------------------------------------------------------------- /evaluation/agentnetbench/reeval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/reeval.py -------------------------------------------------------------------------------- /evaluation/agentnetbench/run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/run.py -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_5473959e0f6e21f7_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_5473959e0f6e21f7_1.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_5473959e0f6e21f7_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_5473959e0f6e21f7_10.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_5473959e0f6e21f7_11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_5473959e0f6e21f7_11.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_5473959e0f6e21f7_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_5473959e0f6e21f7_2.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_5473959e0f6e21f7_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_5473959e0f6e21f7_3.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_5473959e0f6e21f7_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_5473959e0f6e21f7_5.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_5473959e0f6e21f7_6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_5473959e0f6e21f7_6.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_5473959e0f6e21f7_7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_5473959e0f6e21f7_7.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_5473959e0f6e21f7_8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_5473959e0f6e21f7_8.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_5473959e0f6e21f7_9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_5473959e0f6e21f7_9.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_7f27a11115e596eb_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_7f27a11115e596eb_0.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_7f27a11115e596eb_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_7f27a11115e596eb_1.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_7f27a11115e596eb_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_7f27a11115e596eb_10.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_7f27a11115e596eb_12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_7f27a11115e596eb_12.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_7f27a11115e596eb_13.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_7f27a11115e596eb_13.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_7f27a11115e596eb_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_7f27a11115e596eb_3.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_7f27a11115e596eb_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_7f27a11115e596eb_4.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_7f27a11115e596eb_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_7f27a11115e596eb_5.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_7f27a11115e596eb_6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_7f27a11115e596eb_6.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_7f27a11115e596eb_7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_7f27a11115e596eb_7.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_7f27a11115e596eb_8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_7f27a11115e596eb_8.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_a96285eb665bef92_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_a96285eb665bef92_10.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_a96285eb665bef92_11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_a96285eb665bef92_11.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_a96285eb665bef92_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_a96285eb665bef92_2.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_a96285eb665bef92_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_a96285eb665bef92_3.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_a96285eb665bef92_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_a96285eb665bef92_4.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_a96285eb665bef92_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_a96285eb665bef92_5.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_a96285eb665bef92_6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_a96285eb665bef92_6.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_a96285eb665bef92_7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_a96285eb665bef92_7.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_a96285eb665bef92_8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_a96285eb665bef92_8.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_a96285eb665bef92_9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_a96285eb665bef92_9.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_c53b113bf3e7d362_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_c53b113bf3e7d362_1.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_c53b113bf3e7d362_11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_c53b113bf3e7d362_11.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_c53b113bf3e7d362_14.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_c53b113bf3e7d362_14.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_c53b113bf3e7d362_15.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_c53b113bf3e7d362_15.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_c53b113bf3e7d362_18.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_c53b113bf3e7d362_18.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_c53b113bf3e7d362_19.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_c53b113bf3e7d362_19.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_c53b113bf3e7d362_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_c53b113bf3e7d362_2.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_c53b113bf3e7d362_20.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_c53b113bf3e7d362_20.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_c53b113bf3e7d362_21.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_c53b113bf3e7d362_21.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_c53b113bf3e7d362_22.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_c53b113bf3e7d362_22.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_c53b113bf3e7d362_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_c53b113bf3e7d362_3.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_c53b113bf3e7d362_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_c53b113bf3e7d362_5.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_c53b113bf3e7d362_6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_c53b113bf3e7d362_6.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_c53b113bf3e7d362_7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_c53b113bf3e7d362_7.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_c53b113bf3e7d362_8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_c53b113bf3e7d362_8.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_c53b113bf3e7d362_9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_c53b113bf3e7d362_9.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_df0fd37049f470c2_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_df0fd37049f470c2_10.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_df0fd37049f470c2_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_df0fd37049f470c2_2.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_df0fd37049f470c2_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_df0fd37049f470c2_3.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_df0fd37049f470c2_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_df0fd37049f470c2_4.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_df0fd37049f470c2_6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_df0fd37049f470c2_6.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_df0fd37049f470c2_7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_df0fd37049f470c2_7.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/images/s_df0fd37049f470c2_9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/images/s_df0fd37049f470c2_9.png -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/s_5473959e0f6e21f7.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/s_5473959e0f6e21f7.json -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/s_7f27a11115e596eb.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/s_7f27a11115e596eb.json -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/s_a96285eb665bef92.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/s_a96285eb665bef92.json -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/s_c53b113bf3e7d362.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/s_c53b113bf3e7d362.json -------------------------------------------------------------------------------- /evaluation/agentnetbench/sample_data/s_df0fd37049f470c2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/sample_data/s_df0fd37049f470c2.json -------------------------------------------------------------------------------- /evaluation/agentnetbench/utils/qwen_vl_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/evaluation/agentnetbench/utils/qwen_vl_utils.py -------------------------------------------------------------------------------- /model/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/model/README.md -------------------------------------------------------------------------------- /model/inference/grounding_examples/07JVNIBSB9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/model/inference/grounding_examples/07JVNIBSB9.png -------------------------------------------------------------------------------- /model/inference/grounding_examples/0FOB4CLBT2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/model/inference/grounding_examples/0FOB4CLBT2.png -------------------------------------------------------------------------------- /model/inference/grounding_examples/6GNDSETVY9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/model/inference/grounding_examples/6GNDSETVY9.png -------------------------------------------------------------------------------- /model/inference/grounding_examples/OH23HT37NP.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/model/inference/grounding_examples/OH23HT37NP.png -------------------------------------------------------------------------------- /model/inference/grounding_examples/T5l1sFEhSy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/model/inference/grounding_examples/T5l1sFEhSy.png -------------------------------------------------------------------------------- /model/inference/grounding_examples/test0.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/model/inference/grounding_examples/test0.json -------------------------------------------------------------------------------- /model/inference/grounding_examples/test1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/model/inference/grounding_examples/test1.json -------------------------------------------------------------------------------- /model/inference/grounding_examples/test2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/model/inference/grounding_examples/test2.json -------------------------------------------------------------------------------- /model/inference/grounding_examples/test3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/model/inference/grounding_examples/test3.json -------------------------------------------------------------------------------- /model/inference/grounding_examples/test4.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/model/inference/grounding_examples/test4.json -------------------------------------------------------------------------------- /model/inference/huggingface_inference.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/model/inference/huggingface_inference.py -------------------------------------------------------------------------------- /model/requirement.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xlang-ai/OpenCUA/HEAD/model/requirement.txt --------------------------------------------------------------------------------