├── w.txt ├── env ├── __init__.py ├── data │ ├── room_mapping.json │ ├── semantic_map_tags.json │ ├── room_sampling_points.json │ ├── map_room_data.json │ ├── npc_data.json │ └── map_receptacle_data.json ├── map_process.py └── socket_server.py ├── unity └── start.sh ├── robot ├── example.jpg ├── llm_process.py ├── object_detection.py ├── PRS_Robot.urdf └── baseline.py ├── clean_port.sh ├── prs_requirements.txt ├── task_evaluation.py ├── prs_demo.py ├── test_demo.ipynb ├── README.md ├── task └── eval.py └── document └── api.md /w.txt: -------------------------------------------------------------------------------- 1 | sdad -------------------------------------------------------------------------------- /env/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) PRS. All Rights Reserved -------------------------------------------------------------------------------- /unity/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Start the Unity .x86_64 executable 4 | ./PRS.x86_64 5 | -------------------------------------------------------------------------------- /robot/example.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PRS-Organization/prs-delivery/HEAD/robot/example.jpg -------------------------------------------------------------------------------- /clean_port.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Find processes with port 8000 and kill them 4 | netstat -ap | grep 8000 | awk '{print $7}' | awk -F '/' '{print $1}' | while read -r pid; do 5 | echo "Killing process with PID: $pid" 6 | kill -9 "$pid" 7 | done 8 | -------------------------------------------------------------------------------- /prs_requirements.txt: -------------------------------------------------------------------------------- 1 | ansitable==0.9.10 2 | asyncio==3.4.3 3 | cfgv==3.4.0 4 | colored==1.4.4 5 | contourpy==1.2.0 6 | cycler==0.12.1 7 | DateTime==5.5 8 | distlib==0.3.8 9 | filelock==3.13.3 10 | fonttools==4.50.0 11 | identify==2.5.35 12 | kiwisolver==1.4.5 13 | matplotlib==3.8.3 14 | nodeenv==1.8.0 15 | numpy==1.26.4 16 | openai==1.30.5 17 | opencv-python==4.9.0.80 18 | packaging==24.0 19 | pgraph-python==0.6.2 20 | pillow==10.2.0 21 | platformdirs==4.2.0 22 | pre-commit==3.7.0 23 | progress==1.6 24 | pygame==2.5.2 25 | pyparsing==3.1.2 26 | python-dateutil==2.9.0.post0 27 | pytz==2024.1 28 | PyYAML==6.0.1 29 | roboticstoolbox-python==1.1.0 30 | rtb-data==1.0.1 31 | scikit-fmm==2023.4.2 32 | scipy==1.11.1 33 | six==1.16.0 34 | spatialgeometry==1.1.0 35 | spatialmath-python==1.1.9 36 | swift-sim==1.1.0 37 | transformers==4.40.2 38 | typing_extensions==4.10.0 39 | virtualenv==20.25.1 40 | websockets==12.0 41 | zope.interface==6.2 42 | -------------------------------------------------------------------------------- /task_evaluation.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from env.socket_server import * 4 | import os 5 | from robot.baseline import * 6 | 7 | # Environment initialization 8 | prs = PrsEnv(is_print=1, rendering=1, start_up_mode=1) 9 | 10 | with open(os.path.join('task', 'dataset', 'deliver_task_test_set.json'), 'r') as file: 11 | task_data = json.load(file) 12 | tasks = list(task_data.keys()) 13 | print('task:', len(tasks)) 14 | for task_i, task_id in enumerate(tasks): 15 | task = task_data[task_id] 16 | task_npc_id = task['npc_id'] 17 | start_time = time.time() 18 | instruction, npc_information, data = prs.delivery_task_import(task) 19 | # ------------ robot execution method ----------------- 20 | # delivery_execution(prs, instruction, npc_information) 21 | 22 | # ------------ robot execution method ----------------- 23 | time.sleep(0.5) 24 | result = prs.delivery_task_evaluate(task, score=1, save=0) 25 | print(result['task_score']) 26 | 27 | # ====== if you want to save the result as json file to submission on Eval AI ========== 28 | # result_save = prs.delivery_task_evaluate(task, score=0, save=1) 29 | # task_results[task_id] = result_save 30 | # with open('task/result/deliver_task_result.json', 'w') as file: 31 | # json.dump(task_results, file, indent=4) 32 | # --------------------------------------------------------------------------------------- 33 | time.sleep(1) 34 | prs.finish_env() -------------------------------------------------------------------------------- /env/data/room_mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "0": { 3 | "room0": "storage room 1", 4 | "room1": "energy room", 5 | "room2": "storage room 2", 6 | "room3": "conveyor room" 7 | }, 8 | "1": { 9 | "room0": "hall room 2", 10 | "room1": "hall room 1", 11 | "room2": "warm room", 12 | "room3": "lab room 2", 13 | "room4": "lab room 1", 14 | "room5": "medical room 3", 15 | "room6": "server room", 16 | "room7": "storage room", 17 | "room8": "medical room 2", 18 | "room9": "lab room 3", 19 | "room10": "hall room 3", 20 | "room11": "medical room 1", 21 | "room12": "rest room" 22 | }, 23 | "2": { 24 | "room0": "bed room 4", 25 | "room1": "bed room 10", 26 | "room2": "bed room 8", 27 | "room3": "bed room 2", 28 | "room4": "bed room 11", 29 | "room5": "bed room 5", 30 | "room6": "bed room 6", 31 | "room7": "kitchen room", 32 | "room8": "bed room 7", 33 | "room9": "bed room 9", 34 | "room10": "bed room 3", 35 | "room11": "bed room 1", 36 | "room12": "bed room 0", 37 | "room13": "hall room 1", 38 | "room14": "gym room", 39 | "room15": "office room 1", 40 | "room16": "hall room 3", 41 | "room17": "office room 2", 42 | "room18": "rest room", 43 | "room19": "office space", 44 | "room20": "storage room", 45 | "room21": "hall room 2", 46 | "room22": "living room", 47 | "room23": "conference room" 48 | } 49 | } -------------------------------------------------------------------------------- /prs_demo.py: -------------------------------------------------------------------------------- 1 | from env.socket_server import * 2 | import matplotlib.pyplot as plt 3 | # from env.socket_server import server 4 | # import sys 5 | 6 | if __name__ == '__main__': 7 | # Environment initialization 8 | prs = PrsEnv(is_print=1, rendering=1, start_up_mode=1) 9 | prs.npc_start(5) 10 | # How many NPCs are there 11 | prs.agent.initial_pose() 12 | prs.agent.go_to_destination(tar_location='kitchen') 13 | prs.agent.rotate_right(degree=-30) 14 | # prs.agent.direction_adjust(position=(0, 0, 0)) 15 | # print(prs.agent.query_near_objects()) 16 | # print(len(prs.agent.get_room_area('kitchen room'))) 17 | # print(prs.agent.get_receptacles_within_room(room_name='kitchen room')) 18 | # prs.agent.goto_receptacle(room='kitchen room', recepacle=0, random=0) 19 | # print(prs.agent.object_information_query(obj_id=0)) 20 | 21 | # rgb = prs.agent.head_camera_look_at() 22 | # rgb = prs.agent.observation(degree=0, camera=0) 23 | # view = prs.agent.site_view(pos=(0, 0, 0)) 24 | # depth = prs.agent.get_depth() 25 | arr, tags = prs.agent.get_segmentation(camera_type=0, decode=1) 26 | print(tags) 27 | plt.imshow(arr) 28 | plt.show() 29 | 30 | marked_arr = np.zeros_like(arr) 31 | for tag in tags: 32 | if 'apple' in tags[tag].lower(): 33 | aaa = tag 34 | break 35 | marked_arr[arr == aaa] = 1 36 | prs.agent.object_interaction(marked_arr, manipulaton=1) 37 | prs.agent.goto_target_goal((2, 130, 120), position_mode=1) 38 | # robot navigate to floor3 point (130, 120) 39 | map_room = prs.server.maps.floor3 40 | 41 | time.sleep(20) 42 | # ------------ end ------------ 43 | prs.finish_env() -------------------------------------------------------------------------------- /test_demo.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "417071ec", 7 | "metadata": {}, 8 | "outputs": [ 9 | { 10 | "name": "stdout", 11 | "output_type": "stream", 12 | "text": [ 13 | "pygame 2.5.2 (SDL 2.28.3, Python 3.11.5)\n", 14 | "Hello from the pygame community. https://www.pygame.org/contribute.html\n", 15 | "PRS environment beta is starting without interaction\n", 16 | "Please open the Unity program (start.sh)\n", 17 | "PRS challenge task and benchmark come soon!\n" 18 | ] 19 | } 20 | ], 21 | "source": [ 22 | "from socket_server import *\n", 23 | "# Environment initialization\n", 24 | "prs = PrsEnv(is_print=0, not_test_mode=0)" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": null, 30 | "id": "19a9dd9f", 31 | "metadata": {}, 32 | "outputs": [], 33 | "source": [ 34 | "# npc behavior\n", 35 | "prs.npc_start(5)" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": null, 41 | "id": "7d358a1a", 42 | "metadata": {}, 43 | "outputs": [], 44 | "source": [ 45 | "# action sequence of robot\n" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": null, 51 | "id": "d2f268f3", 52 | "metadata": {}, 53 | "outputs": [], 54 | "source": [ 55 | "prs.finish_env()" 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": null, 61 | "id": "bed19f72", 62 | "metadata": {}, 63 | "outputs": [], 64 | "source": [] 65 | } 66 | ], 67 | "metadata": { 68 | "kernelspec": { 69 | "display_name": "Python 3 (ipykernel)", 70 | "language": "python", 71 | "name": "python3" 72 | }, 73 | "language_info": { 74 | "codemirror_mode": { 75 | "name": "ipython", 76 | "version": 3 77 | }, 78 | "file_extension": ".py", 79 | "mimetype": "text/x-python", 80 | "name": "python", 81 | "nbconvert_exporter": "python", 82 | "pygments_lexer": "ipython3", 83 | "version": "3.11.5" 84 | } 85 | }, 86 | "nbformat": 4, 87 | "nbformat_minor": 5 88 | } 89 | -------------------------------------------------------------------------------- /robot/llm_process.py: -------------------------------------------------------------------------------- 1 | # import openai 2 | from matplotlib import pyplot as plt 3 | from openai import OpenAI 4 | # from zhipuai import ZhipuAI 5 | import cv2 6 | import base64 7 | import io 8 | from PIL import Image, ImageDraw 9 | from robot.object_detection import * 10 | 11 | 12 | client = OpenAI( 13 | # api_key=os.environ.get("OPENAI_API_KEY") 14 | ) 15 | grounding_dino = GroundingDino() 16 | 17 | 18 | def object_detect_module(image, text='the human'): 19 | if text[-1] != '.': 20 | text = text + '.' 21 | result = grounding_dino.predict(image, text) 22 | return result 23 | 24 | 25 | def lmm_interaction(content, image): 26 | image = Image.fromarray(image) 27 | image_file = io.BytesIO() 28 | image.save(image_file, format='PNG') 29 | encoded_string = base64.b64encode(image_file.getvalue()).decode() 30 | response = client.chat.completions.create( 31 | model="gpt-4o", # Fill in the name of the model that needs to be called 32 | messages=[ 33 | { 34 | "role": "user", 35 | "content": [ 36 | { 37 | "type": "text", 38 | "text": content 39 | }, 40 | { 41 | "type": "image_url", 42 | "image_url": { 43 | "url": f"data:image/jpeg;base64,{encoded_string}" 44 | } 45 | } 46 | ] 47 | } 48 | ], 49 | max_tokens=300 50 | ) 51 | res = response.choices[0].message.content 52 | return res 53 | 54 | 55 | def llm_interaction(content='Hello World!', temperature=0.9): 56 | response = client.chat.completions.create( 57 | model="gpt-4-turbo", # gpt-3.5-turbo-0125 58 | messages=[ 59 | {"role": "user", "content": content} 60 | ], 61 | ) 62 | res = response.choices[0].message.content 63 | return res 64 | 65 | 66 | if __name__ == '__main__': 67 | im = cv2.imread('example.jpg') 68 | # print(im.shape) 69 | mat = object_detect_module(im, 'the water bottle.') 70 | plt.imshow(mat) 71 | plt.show() -------------------------------------------------------------------------------- /robot/object_detection.py: -------------------------------------------------------------------------------- 1 | from matplotlib import pyplot as plt 2 | from transformers import AutoProcessor, GroundingDinoForObjectDetection 3 | from PIL import Image 4 | import requests 5 | import torch 6 | import numpy as np 7 | import cv2 8 | 9 | 10 | class GroundingDino(object): 11 | def __init__(self): 12 | self.processor = AutoProcessor.from_pretrained("IDEA-Research/grounding-dino-tiny") 13 | self.model = GroundingDinoForObjectDetection.from_pretrained("IDEA-Research/grounding-dino-tiny") 14 | 15 | def predict(self, rgb_image, text): 16 | image = Image.fromarray(rgb_image) 17 | inputs = self.processor(images=image, text=text, return_tensors="pt") 18 | outputs = self.model(**inputs) 19 | # convert outputs (bounding boxes and class logits) to COCO API 20 | target_sizes = torch.tensor([image.size[::-1]]) 21 | results = self.processor.image_processor.post_process_object_detection( 22 | outputs, threshold=0.35, target_sizes=target_sizes 23 | )[0] 24 | item, score_max = None, 0 25 | for score, label, box in zip(results["scores"], results["labels"], results["boxes"]): 26 | box = [round(i, 2) for i in box.tolist()] 27 | # print(f"Detected {label.item()} with confidence " f"{round(score.item(), 3)} at location {box}") 28 | if round(score.item(), 3) > score_max: 29 | score_max = round(score.item(), 3) 30 | item = box 31 | if item is None: 32 | return None 33 | h, w = rgb_image.shape[0], rgb_image.shape[1] 34 | box = item 35 | top_left = (round(box[0]), round(box[1])) 36 | bottom_right = (round(box[2]), round(box[3])) 37 | # print(bottom_right, top_left, h, w) 38 | cv2.rectangle(rgb_image, top_left, bottom_right, (255, 0, 0), 2) 39 | mat = np.zeros((h, w)) 40 | mat[top_left[1]:bottom_right[1] + 1, top_left[0]:bottom_right[0] + 1] = 1 41 | return mat 42 | 43 | 44 | if __name__ == "__main__": 45 | grounding_dino = GroundingDino() 46 | url = "http://images.cocodataset.org/val2017/000000039769.jpg" 47 | pil_image = Image.open(requests.get(url, stream=True).raw) 48 | image = np.array(pil_image) 49 | res = grounding_dino.predict(image, 'cat.') 50 | plt.imshow(res) 51 | plt.show() 52 | print(np.sum(res) / (480 * 640)) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [//]: # (# PRS-Test) 2 | # Human-centered In-building Embodied Delivery Benchmark 3 | ## [PRS Challenge](https://prsorg.github.io/challenge) hosted on [CVPR 2024 Embodied AI Workshop](https://embodied-ai.org/) 4 | 5 | ## Quick Start & Installation 6 | 7 | Follow these steps to quickly set up and run the PRS delivery task version: 8 | 9 | 1. Clone the PRS delivery repository: 10 | ``` 11 | git clone https://github.com/PRS-Organization/prs-delivery.git 12 | ``` 13 | 2. Ensure you have a Python virtual environment (Python version >= 3.9) activated. 14 | 15 | ``` 16 | conda create -n prs python=3.9 17 | conda activate mvp 18 | ``` 19 | 20 | 3. Install the required Python packages: 21 | ``` 22 | pip install -r prs_requirements.txt 23 | ``` 24 | 4. Download the Unity executable file (for Ubuntu, Windows, and Mac) from [PRS executable program](https://docs.google.com/forms/d/e/1FAIpQLScrk25iSnnmOH8cj4eqD8lcALcj1Cx1bSiiTsw9q9DzvWnCig/viewform?usp=sf_link). If you are using the macOS or Windows version, you need to modify some of the environment data paths in ```StreamingAssets``` folder and executable application paths. 25 | 26 | 5. Extract the `PRS_Ubuntu_x.x.x.zip` file into the `unity` folder: 27 | ``` 28 | cd unity 29 | unzip PRS_Ubuntu_0.3.0.zip 30 | ``` 31 | Note that the contents after unzipping should be placed in `unity` folder, and give `unity` folder file permissions: 32 | ``` 33 | sudo chmod 777 -R unity 34 | ``` 35 | 6. Start running the demo: 36 | ``` 37 | python prs_demo.py 38 | ``` 39 | or start with only unity application: 40 | ``` 41 | bash ./unity/start.sh 42 | ``` 43 | 7. If you encounter a port occupation error, clean up occupied ports: 44 | ``` 45 | bash clean_port.sh 46 | ``` 47 | 8. Manual start: class initialization parameter is ```PrsEnv(start_up_mode=0)```, after running the Python script, you can open another terminal and execute ```unity/start.sh``` or directly run `unity/PRS.x86_64`. 48 | 49 | 9. Runing on the headless server without rendering initialization ```PrsEnv(rendering=0)```. 50 | 51 | 10. Wait a few seconds for Unity to render the graphics. In Unity, you can control the camera movement using the keyboard keys W, A, S, D, Q, and E. Robot control using the keyboard keys I R, F J, O P, K L, G H, N M, Z X, V B. Switch perspectives using C, accelerate time using numeric keypad 123456789. 52 | 53 | 11. To close the demo, first close the Unity program (or press Esc), then stop the Python program (Ctrl+C or Ctrl+Z), and finally run: 54 | ``` 55 | bash clean_port.sh 56 | ``` 57 | Note: Or use ```prs.finish_env()``` to end PRS environment in Python script. 58 | 59 | 12. To get started with the Delivery Task Dataset, simply run the following command in your terminal: 60 | 61 | ``` 62 | python task_evaluation.py 63 | ``` 64 | This will initiate the evaluation process using delivery task dataset. 65 | 66 | ## Baseline Method 67 | 68 | If you want to run baseline method, please install ```transformers==4.40.2 ```, ```torch==2.0.1```, ```openai==1.30.5```. And fill in the API-key in the ```robot\llm_process.py```. 69 | We utilize [Grounding DINO](https://github.com/IDEA-Research/GroundingDINO) to achieve zero-shot object detection with text prompt, you can replace it with others, e.g. [Grounded SAM](https://github.com/IDEA-Research/Grounded-Segment-Anything). 70 | 71 | Make```delivery_execution(prs, instruction, npc_information)``` available in ```task_evaluation.py``` on line 20. 72 | 73 | The main process of the baseline method is in ```./robot/baseline.py```, ```./robot/object_detection.py``` is visual detection, and ```./robot/llm_process.py``` is the LMM and LLM application. 74 | 75 | Save the result and submit the ```result.json``` to [Eval AI leaderboard](https://eval.ai/web/challenges/challenge-page/2313/overview). 76 | 77 | ## More API Guidance 78 | [PRS Platform API](document/api.md) 79 | 80 | 81 | 82 | [//]: # (input your API key for LLM service) 83 | 84 | [//]: # (download vision model for object detect) 85 | 86 | [//]: # (python task evaluation py) 87 | 88 | [//]: # (save the result (save=1)) 89 | 90 | [//]: # (submit the json to Eval AI leaderboard) 91 | 92 | [//]: # (cite us contact us project homepage) 93 | 94 | [//]: # (long term leaderboard for delivery) 95 | -------------------------------------------------------------------------------- /robot/PRS_Robot.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | -------------------------------------------------------------------------------- /task/eval.py: -------------------------------------------------------------------------------- 1 | import random 2 | import datetime 3 | import numpy as np 4 | import json 5 | import math 6 | 7 | 8 | def calculate_distance(point1, point2): 9 | # NumPy array 10 | try: 11 | point1_array = np.array([point1['x'], point1['y'], point1['z']], dtype=float) 12 | except: 13 | point1_array = np.array([point1[0], point1[1], point1[2]], dtype=float) 14 | try: 15 | point2_array = np.array([point2['x'], point2['y'], point2['z']], dtype=float) 16 | except: 17 | point2_array = np.array([point2[0], point2[1], point2[2]], dtype=float) 18 | try: 19 | # distance = np.linalg.norm(point2_array - point1_array) 20 | distance = math.sqrt(sum((a - b) ** 2 for a, b in zip(point1_array, point2_array))) 21 | except: 22 | distance = 99999 23 | return distance 24 | 25 | 26 | def delivery_task_score(result_data, task_id=0): 27 | task_res, res_grasp, res_find, human_find, res_deliver = 0, 0, 0, 0, 0 28 | start_t = datetime.datetime.fromisoformat(result_data['start_time']) 29 | end_t = datetime.datetime.fromisoformat(result_data['end_time']) 30 | time_cost = end_t - start_t 31 | seconds_cost = (end_t - start_t).total_seconds() 32 | minutes_cost = time_cost.total_seconds() / 60 33 | # 1. target find, 2. grasp target, 3. target object dis, 4. deliver 34 | if result_data['agent_object_name'] is not None: 35 | if result_data['tar_object_name'] == result_data['agent_object_name']: 36 | res_grasp = 1 37 | else: 38 | if result_data['target_object_type'] in result_data['agent_object_name']: 39 | res_grasp = 0.5 40 | hr_dis = calculate_distance(result_data['npc_position'], result_data['agent_position']) 41 | if hr_dis < 3: 42 | human_find = 1 43 | elif hr_dis < 5: 44 | human_find = 0.5 45 | res_deliver = human_find * res_grasp 46 | task_res = res_grasp + res_find + res_deliver + human_find 47 | task_result = {'sub_result': {'grasp': res_grasp, 'object_find': res_find, 48 | 'deliver': res_deliver, 'human_find': human_find}, 49 | 'result': task_res, 'time': minutes_cost} 50 | return task_result 51 | 52 | 53 | def evaluate(test_annotation_file, user_submission_file, phase_codename, **kwargs): 54 | print("Starting Evaluation.....") 55 | """ 56 | Evaluates the submission for a particular challenge phase and returns score 57 | Arguments: 58 | 59 | `test_annotations_file`: Path to test_annotation_file on the server 60 | `user_submission_file`: Path to file submitted by the user 61 | `phase_codename`: Phase to which submission is made 62 | 63 | `**kwargs`: keyword arguments that contains additional submission 64 | metadata that challenge hosts can use to send slack notification. 65 | You can access the submission metadata 66 | with kwargs['submission_metadata'] 67 | 68 | Example: A sample submission metadata can be accessed like this: 69 | >>> print(kwargs['submission_metadata']) 70 | { 71 | 'status': u'running', 72 | 'when_made_public': None, 73 | 'participant_team': 5, 74 | 'input_file': 'https://abc.xyz/path/to/submission/file.json', 75 | 'execution_time': u'123', 76 | 'publication_url': u'ABC', 77 | 'challenge_phase': 1, 78 | 'created_by': u'ABC', 79 | 'stdout_file': 'https://abc.xyz/path/to/stdout/file.json', 80 | 'method_name': u'Test', 81 | 'stderr_file': 'https://abc.xyz/path/to/stderr/file.json', 82 | 'participant_team_name': u'Test Team', 83 | 'project_url': u'http://foo.bar', 84 | 'method_description': u'ABC', 85 | 'is_public': False, 86 | 'submission_result_file': 'https://abc.xyz/path/result/file.json', 87 | 'id': 123, 88 | 'submitted_at': u'2017-03-20T19:22:03.880652Z' 89 | } 90 | """ 91 | with open(test_annotation_file, 'r') as file: 92 | task_data = json.load(file) 93 | with open(user_submission_file, 'r') as file: 94 | task_submission = json.load(file) 95 | task_r, grasp_r, human_r, time_used = 0, 0, 0, 0 96 | number, tasks = len(list(task_data)), [] 97 | for task_id, task in task_submission.items(): 98 | print(task_id) 99 | if task_id in tasks: 100 | continue 101 | else: 102 | tasks.append(task) 103 | task_result = delivery_task_score(task) 104 | # task_result = {'sub_result': {'grasp': res_grasp, 'object_find': res_find, 105 | # 'deliver': res_deliver, 'human_find': human_find}, 106 | # 'result': task_res, 'time': minutes_cost} 107 | if task_result['time'] > 10: 108 | continue 109 | task_r += int(task_result['result']/4) 110 | grasp_r += task_result['sub_result']['grasp'] 111 | human_r += task_result['sub_result']['human_find'] 112 | time_used += task_result['time'] 113 | print(number, len(tasks)) 114 | task_r = task_r / float(number) 115 | grasp_r = grasp_r / float(number) 116 | human_r = human_r / float(number) 117 | time_used = time_used / float(len(tasks)) 118 | 119 | output = {} 120 | if phase_codename == "dev": 121 | # print("Evaluating for Dev Phase") 122 | # ["Task SR", "Object Manipulation SR", "Human Search SR", "Time Used"] 123 | output["result"] = [ 124 | { 125 | "Delivery_dataset": { 126 | "Task SR": task_r, 127 | "Object Manipulation SR": grasp_r, 128 | "Human Search SR": human_r, 129 | "Time Used": time_used 130 | } 131 | } 132 | ] 133 | # To display the results in the result file 134 | output["submission_result"] = output["result"][0]["Delivery_dataset"] 135 | print("Completed evaluation for Dev Phase") 136 | elif phase_codename == "test": 137 | # print("Evaluating for Test Phase") 138 | output["result"] = [ 139 | { 140 | "Delivery_dataset": { 141 | "Task SR": task_r, 142 | "Object Manipulation SR": grasp_r, 143 | "Human Search SR": human_r, 144 | "Time Used": time_used 145 | } 146 | } 147 | ] 148 | # To display the results in the result file 149 | output["submission_result"] = output["result"][0]["Delivery_dataset"] 150 | print("Completed evaluation for Test Phase") 151 | return output 152 | 153 | 154 | if __name__ == "__main__": 155 | res = evaluate('dataset/deliver_task_test_set.json', 156 | 'D:\\code\\prs_develop\\prs_system_git_push_version\\task\\result\\deliver_task_5_19_result.json' 157 | , 'test') 158 | print(res) -------------------------------------------------------------------------------- /document/api.md: -------------------------------------------------------------------------------- 1 | # PRS API 2 | We provide **API** that includes various functions: 3 | - [Environment API](#environment-api) 4 | - [Task API](#task-api) 5 | - [Robot API](#robot-api) 6 | - [Map API](#map-and-position-api) 7 | - [NPC API](#npc-api) 8 | 9 | ## Environment API 10 | Operations: Accelerate, Terminate, Initiate 11 | 12 | Instantiate environment class 13 | ``` 14 | prs = PrsEnv() 15 | ``` 16 | Start autonomous activities for NPCs, specifying the number of activated NPCs 17 | ``` 18 | prs.npc_start(3) 19 | ``` 20 | Accelerate the scene simulation by specifying the speed multiplier 21 | ``` 22 | prs.sim_speed(2) 23 | ``` 24 | set environment time 25 | ``` 26 | prs.env_time.set_time(2024, 6, 1, 9, 0) 27 | ``` 28 | Terminate the environment and end the process 29 | ``` 30 | prs.finish_env() 31 | ``` 32 | 33 | ## Task API 34 | 35 | Initialize the task program, "task" stands for JSON content of a delivery task 36 | ``` 37 | prs.delivery_task_import(task) 38 | ``` 39 | Evaluate the task result: score=1 represents result calculation, and save = 1 will return the result dict for saving the JSON file 40 | ``` 41 | prs.delivery_task_evaluate(task, score=1, save=0) 42 | ``` 43 | ## Robot API 44 | 45 | Retrieve vision data, specifying the camera type (0 for head, 1 for hand) 46 | ``` 47 | prs.agent.observation_camera(camera_type=0) 48 | ``` 49 | Move the robot to a target location, specifying the position mode (0 for world, 1 for grid map) 50 | ``` 51 | prs.agent.goto_target_goal((2, 130, 120), position_mode=1) 52 | ``` 53 | Retrieve robot position information, returning position (pos) and detailed information (info) 54 | ``` 55 | prs.agent.pos_query() 56 | ``` 57 | Rotate the robot, specifying the degree of rotation (positive for right, negative for left) 58 | ``` 59 | prs.agent.rotate_right(degree=30) 60 | ``` 61 | Control robot joints, specifying the joint ID and target 62 | ``` 63 | prs.agent.joint_control(joint_id=1, target=20) 64 | ``` 65 | Adjust robot direction alignment based on input coordinates 66 | ``` 67 | prs.agent.direction_adjust(position=(22.38, 0.1, -0.17)) 68 | ``` 69 | Perform inverse kinematics (IK) for the robot, specifying relative coordinates 70 | ``` 71 | prs.agent.ik_process(x=0, y=1, z=0.1) 72 | ``` 73 | Calculate 6 Degrees of Freedom (6-DoF) IK for the robot, specifying the target coordinates and rotation matrix (yaw, pitch, roll) 74 | ``` 75 | prs.agent.input_pos(prs.agent.robot, x=0.2, y=0, z=0.75, phi=0, theta=1.25, psi=0, plot=0) 76 | ``` 77 | Move the robot forward by specifying the distance 78 | ``` 79 | prs.agent.move_forward(dis=1.0) 80 | ``` 81 | Navigate to a specific destination, specifying the location name 82 | ``` 83 | prs.agent.go_to_destination('kitchen') 84 | ``` 85 | Retrieve the latest map 86 | ``` 87 | prs.agent.get_all_map() 88 | ``` 89 | Calculate IK for relative coordinates and target values, specifying world coordinates 90 | ``` 91 | prs.agent.ik_calculation((-10.1, 0.1, 1.6)) 92 | ``` 93 | Calculate IK target, specifying relative coordinates 94 | ``` 95 | prs.agent.ik_process(0.25, 0.1, 0.79) 96 | ``` 97 | Control robot arms, specifying joint target values 98 | ``` 99 | prs.agent.arm_control([0, 0, 0, 0, 0]) 100 | ``` 101 | Grab an object, specifying the object's ID 102 | ``` 103 | prs.agent.grasp_object(17) 104 | ``` 105 | Move towards the vicinity of an object, specifying the object ID or functional feature 106 | ``` 107 | prs.agent.go_to_target_object(feature='Seat') 108 | ``` 109 | Walk towards and grab the target object 110 | ``` 111 | prs.agent.goto_and_grasp('apple') 112 | ``` 113 | Release the held object 114 | ``` 115 | depth_m = prs.agent.get_depth(0) 116 | ``` 117 | Retrieve depth information, specifying mode (0 for head camera, 1 for hand camera) 118 | ``` 119 | prs.agent.get_depth() 120 | ``` 121 | Retrieve camera semantic segmentation, specifying mode (0 for head camera, 1 for hand camera), tags contain object information 122 | ``` 123 | seg, tags = prs.agent.get_segmentation(0) 124 | ``` 125 | Head camera twisted downwards 126 | ``` 127 | prs.agent.joint_control(joint_id=4, target=20) 128 | ``` 129 | Using the camera to observe the surroundings with the given angle 130 | ``` 131 | prs.agent.observation(degree=0, camera=0, up_down=30) 132 | ``` 133 | 134 | "Request Visual Interaction" 135 | 136 | ``` 137 | prs.agent.object_interaction(input_matrix=array, manipulaton=1, type=0) 138 | ``` 139 | [This function requests an interaction with the visual system. 140 | Input: A two-dimensional matrix marking the target and the operation type: 141 | 0: recognize 142 | 1: grasp 143 | 2: approach target 144 | 3, etc.] 145 | 146 | [//]: # (prs.agent.request_interaction()) 147 | [//]: # (prs.agent.interaction()) 148 | 149 | Robot pose adjustment 150 | ``` 151 | prs.agent.initial_pose() 152 | ``` 153 | 154 | Go to receptacle of the room 155 | ``` 156 | prs.agent.goto_receptacle(room='office', recepacle='table', random=1) 157 | ``` 158 | 159 | Look at somewhere (head camera) 160 | ``` 161 | prs.agent.head_camera_look_at((1, 122, 54)) 162 | ``` 163 | 164 | Estimating distance through mask 165 | ``` 166 | tar_distance, view_angle = prs.agent.depth_estimation(target_mask, depth_m, field_of_view=90) 167 | ``` 168 | 169 | Calculate accessible locations based on target distance and observation angle 170 | ``` 171 | position = prs.agent.target_direction(degree=camera_degree, distance=tar_distance, target_degree_view=view_angle) 172 | ``` 173 | 174 | [//]: # (```) 175 | 176 | [//]: # (prs.agent.head_camera_look_at) 177 | 178 | [//]: # (```) 179 | ## Map and Position API 180 | 181 | Determine the area based on world coordinates, returning the room or None 182 | ``` 183 | prs.objs_data.point_determine((15.7, -5.1, -19.5)) 184 | ``` 185 | Retrieve information from the 2D grid map, specifying world coordinates [-10.1, 0.1, -6.1], output includes floor, map_i, map_j, and obstacle status 186 | ``` 187 | prs.server.maps.get_point_info({'x': -10.1, 'y': 0.1, 'z': -6.1}) 188 | ``` 189 | Retrieve world position based on floor, map_i, and map_j coordinates 190 | ``` 191 | prs.server.maps.get_world_position(1, 89, 108) 192 | ``` 193 | Retrieve 2D grid maps for floors 1, 2, and 3 194 | ``` 195 | map1, map2, map3 = prs.server.maps.floor1, prs.server.maps.floor2, prs.server.maps.floor3 196 | ``` 197 | Retrieve the latest map 198 | ``` 199 | prs.agent.get_all_map() 200 | ``` 201 | Parse object information based on instructions, querying a specific object 202 | ``` 203 | obj_list = prs.objs_data.object_parsing(instruction, ['apple']) 204 | ``` 205 | Global query for an object's ID, specifying the object's name 206 | ``` 207 | obj_list = prs.objs_data.object_query(['apple']) 208 | ``` 209 | Query object information based on ID 210 | ``` 211 | prs.object_query(obj_id=100) 212 | ``` 213 | 214 | ## NPC API 215 | 216 | Execute actions for NPCs based on action IDs, including looping actions, interactive actions, and performance actions 217 | ``` 218 | prs.npcs[0].npc_action(tar_action='stand') 219 | ``` 220 | Retrieve photos around the NPC, from the camera perspective above the head 221 | ``` 222 | prs.npcs[0].observation_surrounding() 223 | ``` 224 | Retrieve the current NPC's position, returning position and detailed NPC information 225 | ``` 226 | prs.npcs[0].where_npc() 227 | ``` 228 | NPC move to a specific place based on preset location lists 229 | ``` 230 | prs.npcs[0].go_to_place() 231 | ``` 232 | Retrieve detailed information about the NPC, including position and other information 233 | ``` 234 | pos, npc_info = prs.npcs[0].query_information() 235 | ``` 236 | Move towards the vicinity of an object, specifying the distance range to the target 237 | ``` 238 | res, obj = prs.npcs[0].go_to_object('Seat') 239 | ``` -------------------------------------------------------------------------------- /env/data/semantic_map_tags.json: -------------------------------------------------------------------------------- 1 | { 2 | "0": { 3 | "0": "obstacles", 4 | "1": "accessible area", 5 | "4": [ 6 | "room0" 7 | ], 8 | "5": [ 9 | "room1" 10 | ], 11 | "6": [ 12 | "room2" 13 | ], 14 | "7": [ 15 | "room3" 16 | ] 17 | }, 18 | "1": { 19 | "0": "obstacles", 20 | "1": "accessible area", 21 | "4": [ 22 | "room10", 23 | "reception counter" 24 | ], 25 | "5": [ 26 | "room10", 27 | "table" 28 | ], 29 | "6": [ 30 | "room10", 31 | "table" 32 | ], 33 | "7": [ 34 | "room10", 35 | "table" 36 | ], 37 | "8": [ 38 | "room10", 39 | "table" 40 | ], 41 | "9": [ 42 | "room0", 43 | "table" 44 | ], 45 | "10": [ 46 | "room0", 47 | "table" 48 | ], 49 | "11": [ 50 | "room1", 51 | "polygon table" 52 | ], 53 | "12": [ 54 | "room5", 55 | "desk" 56 | ], 57 | "13": [ 58 | "room0" 59 | ], 60 | "14": [ 61 | "room1" 62 | ], 63 | "15": [ 64 | "room2" 65 | ], 66 | "16": [ 67 | "room3" 68 | ], 69 | "17": [ 70 | "room4" 71 | ], 72 | "18": [ 73 | "room5" 74 | ], 75 | "19": [ 76 | "room6" 77 | ], 78 | "20": [ 79 | "room7" 80 | ], 81 | "21": [ 82 | "room8" 83 | ], 84 | "22": [ 85 | "room9" 86 | ], 87 | "23": [ 88 | "room10" 89 | ], 90 | "24": [ 91 | "room11" 92 | ], 93 | "25": [ 94 | "room12" 95 | ] 96 | }, 97 | "2": { 98 | "0": "obstacles", 99 | "1": "accessible area", 100 | "4": [ 101 | "room0", 102 | "table" 103 | ], 104 | "5": [ 105 | "room0", 106 | "bed" 107 | ], 108 | "6": [ 109 | "room1", 110 | "table" 111 | ], 112 | "7": [ 113 | "room1", 114 | "bed" 115 | ], 116 | "8": [ 117 | "room2", 118 | "table" 119 | ], 120 | "9": [ 121 | "room2", 122 | "bed" 123 | ], 124 | "10": [ 125 | "room3", 126 | "table" 127 | ], 128 | "11": [ 129 | "room3", 130 | "bed" 131 | ], 132 | "12": [ 133 | "room15", 134 | "desk" 135 | ], 136 | "13": [ 137 | "room16", 138 | "table" 139 | ], 140 | "14": [ 141 | "room16", 142 | "table" 143 | ], 144 | "15": [ 145 | "room16", 146 | "table" 147 | ], 148 | "16": [ 149 | "room16", 150 | "reception counter" 151 | ], 152 | "17": [ 153 | "room16", 154 | "table" 155 | ], 156 | "18": [ 157 | "room19", 158 | "side table" 159 | ], 160 | "19": [ 161 | "room19", 162 | "desk" 163 | ], 164 | "20": [ 165 | "room19", 166 | "desk" 167 | ], 168 | "21": [ 169 | "room19", 170 | "desk" 171 | ], 172 | "22": [ 173 | "room19", 174 | "desk" 175 | ], 176 | "23": [ 177 | "room19", 178 | "desk" 179 | ], 180 | "24": [ 181 | "room19", 182 | "desk" 183 | ], 184 | "25": [ 185 | "room19", 186 | "desk" 187 | ], 188 | "26": [ 189 | "room19", 190 | "desk" 191 | ], 192 | "27": [ 193 | "room22", 194 | "dinner counter" 195 | ], 196 | "28": [ 197 | "room22", 198 | "coffee table" 199 | ], 200 | "29": [ 201 | "room4", 202 | "table" 203 | ], 204 | "30": [ 205 | "room4", 206 | "bed" 207 | ], 208 | "31": [ 209 | "room5", 210 | "table" 211 | ], 212 | "32": [ 213 | "room5", 214 | "bed" 215 | ], 216 | "33": [ 217 | "room17", 218 | "desk" 219 | ], 220 | "34": [ 221 | "room6", 222 | "table" 223 | ], 224 | "35": [ 225 | "room6", 226 | "bed" 227 | ], 228 | "36": [ 229 | "room7", 230 | "small table" 231 | ], 232 | "37": [ 233 | "room7", 234 | "table" 235 | ], 236 | "38": [ 237 | "room7", 238 | "kitchen counter" 239 | ], 240 | "39": [ 241 | "room7", 242 | "kitchen wardrobe" 243 | ], 244 | "40": [ 245 | "room7", 246 | "dinner counter" 247 | ], 248 | "41": [ 249 | "room21", 250 | "table" 251 | ], 252 | "42": [ 253 | "room21", 254 | "table" 255 | ], 256 | "43": [ 257 | "room8", 258 | "table" 259 | ], 260 | "44": [ 261 | "room8", 262 | "bed" 263 | ], 264 | "45": [ 265 | "room23", 266 | "meeting table large" 267 | ], 268 | "46": [ 269 | "room9", 270 | "table" 271 | ], 272 | "47": [ 273 | "room9", 274 | "bed" 275 | ], 276 | "48": [ 277 | "room14", 278 | "furnituregymdesk" 279 | ], 280 | "49": [ 281 | "room10", 282 | "table" 283 | ], 284 | "50": [ 285 | "room10", 286 | "bed" 287 | ], 288 | "51": [ 289 | "room11", 290 | "table" 291 | ], 292 | "52": [ 293 | "room11", 294 | "bed" 295 | ], 296 | "53": [ 297 | "room13", 298 | "polygon table" 299 | ], 300 | "54": [ 301 | "room12", 302 | "table" 303 | ], 304 | "55": [ 305 | "room12", 306 | "bed" 307 | ], 308 | "56": [ 309 | "room0" 310 | ], 311 | "57": [ 312 | "room1" 313 | ], 314 | "58": [ 315 | "room2" 316 | ], 317 | "59": [ 318 | "room3" 319 | ], 320 | "60": [ 321 | "room4" 322 | ], 323 | "61": [ 324 | "room5" 325 | ], 326 | "62": [ 327 | "room6" 328 | ], 329 | "63": [ 330 | "room7" 331 | ], 332 | "64": [ 333 | "room8" 334 | ], 335 | "65": [ 336 | "room9" 337 | ], 338 | "66": [ 339 | "room10" 340 | ], 341 | "67": [ 342 | "room11" 343 | ], 344 | "68": [ 345 | "room12" 346 | ], 347 | "69": [ 348 | "room13" 349 | ], 350 | "70": [ 351 | "room14" 352 | ], 353 | "71": [ 354 | "room15" 355 | ], 356 | "72": [ 357 | "room16" 358 | ], 359 | "73": [ 360 | "room17" 361 | ], 362 | "74": [ 363 | "room18" 364 | ], 365 | "75": [ 366 | "room19" 367 | ], 368 | "76": [ 369 | "room20" 370 | ], 371 | "77": [ 372 | "room21" 373 | ], 374 | "78": [ 375 | "room22" 376 | ], 377 | "79": [ 378 | "room23" 379 | ] 380 | } 381 | } -------------------------------------------------------------------------------- /env/data/room_sampling_points.json: -------------------------------------------------------------------------------- 1 | { 2 | "0_storage room 1": [ 3 | [ 4 | 0, 5 | 5, 6 | 69 7 | ], 8 | [ 9 | 0, 10 | 42, 11 | 36 12 | ] 13 | ], 14 | "0_energy room": [ 15 | [ 16 | 0, 17 | 0, 18 | 0 19 | ], 20 | [ 21 | 0, 22 | 138, 23 | 72 24 | ] 25 | ], 26 | "0_storage room 2": [ 27 | [ 28 | 0, 29 | 5, 30 | 140 31 | ], 32 | [ 33 | 0, 34 | 42, 35 | 108 36 | ] 37 | ], 38 | "0_conveyor room": [ 39 | [ 40 | 0, 41 | 67, 42 | 201 43 | ], 44 | [ 45 | 0, 46 | 80, 47 | 208 48 | ] 49 | ], 50 | "1_hall room 2": [ 51 | [ 52 | 1, 53 | 141, 54 | 187 55 | ], 56 | [ 57 | 1, 58 | 152, 59 | 178 60 | ] 61 | ], 62 | "1_hall room 1": [ 63 | [ 64 | 1, 65 | 104, 66 | 190 67 | ], 68 | [ 69 | 1, 70 | 111, 71 | 179 72 | ] 73 | ], 74 | "1_warm room": [ 75 | [ 76 | 1, 77 | 122, 78 | 134 79 | ], 80 | [ 81 | 1, 82 | 112, 83 | 132 84 | ] 85 | ], 86 | "1_lab room 2": [ 87 | [ 88 | 1, 89 | 82, 90 | 207 91 | ], 92 | [ 93 | 1, 94 | 83, 95 | 217 96 | ] 97 | ], 98 | "1_lab room 1": [ 99 | [ 100 | 1, 101 | 114, 102 | 207 103 | ], 104 | [ 105 | 1, 106 | 116, 107 | 217 108 | ] 109 | ], 110 | "1_medical room 3": [ 111 | [ 112 | 1, 113 | 26, 114 | 189 115 | ], 116 | [ 117 | 1, 118 | 25, 119 | 178 120 | ] 121 | ], 122 | "1_server room": [ 123 | [ 124 | 1, 125 | 122, 126 | 159 127 | ], 128 | [ 129 | 1, 130 | 112, 131 | 158 132 | ] 133 | ], 134 | "1_storage room": [ 135 | [ 136 | 1, 137 | 0, 138 | 0 139 | ], 140 | [ 141 | 1, 142 | 29, 143 | 219 144 | ] 145 | ], 146 | "1_medical room 2": [ 147 | [ 148 | 1, 149 | 51, 150 | 189 151 | ], 152 | [ 153 | 1, 154 | 50, 155 | 178 156 | ] 157 | ], 158 | "1_lab room 3": [ 159 | [ 160 | 1, 161 | 51, 162 | 207 163 | ], 164 | [ 165 | 1, 166 | 50, 167 | 217 168 | ] 169 | ], 170 | "1_hall room 3": [ 171 | [ 172 | 1, 173 | 141, 174 | 208 175 | ], 176 | [ 177 | 1, 178 | 148, 179 | 218 180 | ] 181 | ], 182 | "1_medical room 1": [ 183 | [ 184 | 1, 185 | 82, 186 | 189 187 | ], 188 | [ 189 | 1, 190 | 84, 191 | 178 192 | ] 193 | ], 194 | "1_rest room": [ 195 | [ 196 | 1, 197 | 0, 198 | 0 199 | ], 200 | [ 201 | 1, 202 | 12, 203 | 222 204 | ] 205 | ], 206 | "2_bed room 4": [ 207 | [ 208 | 2, 209 | 141, 210 | 108 211 | ], 212 | [ 213 | 2, 214 | 152, 215 | 103 216 | ] 217 | ], 218 | "2_bed room 10": [ 219 | [ 220 | 2, 221 | 141, 222 | 31 223 | ], 224 | [ 225 | 2, 226 | 152, 227 | 26 228 | ] 229 | ], 230 | "2_bed room 8": [ 231 | [ 232 | 2, 233 | 123, 234 | 46 235 | ], 236 | [ 237 | 2, 238 | 112, 239 | 52 240 | ] 241 | ], 242 | "2_bed room 2": [ 243 | [ 244 | 2, 245 | 123, 246 | 72 247 | ], 248 | [ 249 | 2, 250 | 112, 251 | 77 252 | ] 253 | ], 254 | "2_bed room 11": [ 255 | [ 256 | 2, 257 | 141, 258 | 83 259 | ], 260 | [ 261 | 2, 262 | 152, 263 | 77 264 | ] 265 | ], 266 | "2_bed room 5": [ 267 | [ 268 | 2, 269 | 123, 270 | 98 271 | ], 272 | [ 273 | 2, 274 | 112, 275 | 102 276 | ] 277 | ], 278 | "2_bed room 6": [ 279 | [ 280 | 2, 281 | 123, 282 | 149 283 | ], 284 | [ 285 | 2, 286 | 112, 287 | 154 288 | ] 289 | ], 290 | "2_kitchen room": [ 291 | [ 292 | 2, 293 | 19, 294 | 190 295 | ], 296 | [ 297 | 2, 298 | 16, 299 | 180 300 | ] 301 | ], 302 | "2_bed room 7": [ 303 | [ 304 | 2, 305 | 141, 306 | 57 307 | ], 308 | [ 309 | 2, 310 | 152, 311 | 52 312 | ] 313 | ], 314 | "2_bed room 9": [ 315 | [ 316 | 2, 317 | 141, 318 | 134 319 | ], 320 | [ 321 | 2, 322 | 152, 323 | 128 324 | ] 325 | ], 326 | "2_bed room 3": [ 327 | [ 328 | 2, 329 | 141, 330 | 160 331 | ], 332 | [ 333 | 2, 334 | 152, 335 | 154 336 | ] 337 | ], 338 | "2_bed room 1": [ 339 | [ 340 | 2, 341 | 123, 342 | 123 343 | ], 344 | [ 345 | 2, 346 | 112, 347 | 128 348 | ] 349 | ], 350 | "2_bed room 0": [ 351 | [ 352 | 2, 353 | 123, 354 | 21 355 | ], 356 | [ 357 | 2, 358 | 112, 359 | 26 360 | ] 361 | ], 362 | "2_hall room 1": [ 363 | [ 364 | 2, 365 | 104, 366 | 190 367 | ], 368 | [ 369 | 2, 370 | 111, 371 | 179 372 | ] 373 | ], 374 | "2_gym room": [ 375 | [ 376 | 2, 377 | 114, 378 | 207 379 | ], 380 | [ 381 | 2, 382 | 116, 383 | 217 384 | ] 385 | ], 386 | "2_office room 1": [ 387 | [ 388 | 2, 389 | 58, 390 | 190 391 | ], 392 | [ 393 | 2, 394 | 58, 395 | 179 396 | ] 397 | ], 398 | "2_hall room 3": [ 399 | [ 400 | 2, 401 | 141, 402 | 208 403 | ], 404 | [ 405 | 2, 406 | 148, 407 | 218 408 | ] 409 | ], 410 | "2_office room 2": [ 411 | [ 412 | 2, 413 | 43, 414 | 190 415 | ], 416 | [ 417 | 2, 418 | 42, 419 | 179 420 | ] 421 | ], 422 | "2_rest room": [ 423 | [ 424 | 2, 425 | 0, 426 | 0 427 | ], 428 | [ 429 | 2, 430 | 12, 431 | 222 432 | ] 433 | ], 434 | "2_office space": [ 435 | [ 436 | 2, 437 | 82, 438 | 207 439 | ], 440 | [ 441 | 2, 442 | 83, 443 | 217 444 | ] 445 | ], 446 | "2_storage room": [ 447 | [ 448 | 2, 449 | 0, 450 | 0 451 | ], 452 | [ 453 | 2, 454 | 29, 455 | 219 456 | ] 457 | ], 458 | "2_hall room 2": [ 459 | [ 460 | 2, 461 | 141, 462 | 187 463 | ], 464 | [ 465 | 2, 466 | 152, 467 | 178 468 | ] 469 | ], 470 | "2_living room": [ 471 | [ 472 | 2, 473 | 51, 474 | 207 475 | ], 476 | [ 477 | 2, 478 | 50, 479 | 217 480 | ] 481 | ], 482 | "2_conference room": [ 483 | [ 484 | 2, 485 | 82, 486 | 189 487 | ], 488 | [ 489 | 2, 490 | 83, 491 | 178 492 | ] 493 | ] 494 | } -------------------------------------------------------------------------------- /env/data/map_room_data.json: -------------------------------------------------------------------------------- 1 | { 2 | "0": { 3 | "room0": { 4 | "x": [ 5 | 2, 6 | 83 7 | ], 8 | "y": [ 9 | 4, 10 | 69 11 | ] 12 | }, 13 | "room1": { 14 | "x": [ 15 | 118, 16 | 159 17 | ], 18 | "y": [ 19 | 39, 20 | 105 21 | ] 22 | }, 23 | "room2": { 24 | "x": [ 25 | 2, 26 | 83 27 | ], 28 | "y": [ 29 | 75, 30 | 140 31 | ] 32 | }, 33 | "room3": { 34 | "x": [ 35 | 62, 36 | 98 37 | ], 38 | "y": [ 39 | 188, 40 | 228 41 | ] 42 | } 43 | }, 44 | "1": { 45 | "room0": { 46 | "x": [ 47 | 141, 48 | 163 49 | ], 50 | "y": [ 51 | 168, 52 | 187 53 | ] 54 | }, 55 | "room1": { 56 | "x": [ 57 | 100, 58 | 122 59 | ], 60 | "y": [ 61 | 168, 62 | 190 63 | ] 64 | }, 65 | "room2": { 66 | "x": [ 67 | 101, 68 | 122 69 | ], 70 | "y": [ 71 | 116, 72 | 148 73 | ] 74 | }, 75 | "room3": { 76 | "x": [ 77 | 68, 78 | 98 79 | ], 80 | "y": [ 81 | 207, 82 | 227 83 | ] 84 | }, 85 | "room4": { 86 | "x": [ 87 | 100, 88 | 131 89 | ], 90 | "y": [ 91 | 207, 92 | 227 93 | ] 94 | }, 95 | "room5": { 96 | "x": [ 97 | 17, 98 | 33 99 | ], 100 | "y": [ 101 | 168, 102 | 189 103 | ] 104 | }, 105 | "room6": { 106 | "x": [ 107 | 101, 108 | 122 109 | ], 110 | "y": [ 111 | 150, 112 | 166 113 | ] 114 | }, 115 | "room7": { 116 | "x": [ 117 | 25, 118 | 33 119 | ], 120 | "y": [ 121 | 211, 122 | 227 123 | ] 124 | }, 125 | "room8": { 126 | "x": [ 127 | 33, 128 | 67 129 | ], 130 | "y": [ 131 | 168, 132 | 189 133 | ] 134 | }, 135 | "room9": { 136 | "x": [ 137 | 35, 138 | 65 139 | ], 140 | "y": [ 141 | 207, 142 | 227 143 | ] 144 | }, 145 | "room10": { 146 | "x": [ 147 | 133, 148 | 162 149 | ], 150 | "y": [ 151 | 208, 152 | 227 153 | ] 154 | }, 155 | "room11": { 156 | "x": [ 157 | 68, 158 | 100 159 | ], 160 | "y": [ 161 | 168, 162 | 189 163 | ] 164 | }, 165 | "room12": { 166 | "x": [ 167 | 0, 168 | 24 169 | ], 170 | "y": [ 171 | 216, 172 | 227 173 | ] 174 | } 175 | }, 176 | "2": { 177 | "room0": { 178 | "x": [ 179 | 141, 180 | 163 181 | ], 182 | "y": [ 183 | 91, 184 | 115 185 | ] 186 | }, 187 | "room1": { 188 | "x": [ 189 | 141, 190 | 163 191 | ], 192 | "y": [ 193 | 14, 194 | 38 195 | ] 196 | }, 197 | "room2": { 198 | "x": [ 199 | 101, 200 | 123 201 | ], 202 | "y": [ 203 | 40, 204 | 63 205 | ] 206 | }, 207 | "room3": { 208 | "x": [ 209 | 101, 210 | 123 211 | ], 212 | "y": [ 213 | 65, 214 | 89 215 | ] 216 | }, 217 | "room4": { 218 | "x": [ 219 | 141, 220 | 163 221 | ], 222 | "y": [ 223 | 65, 224 | 89 225 | ] 226 | }, 227 | "room5": { 228 | "x": [ 229 | 101, 230 | 123 231 | ], 232 | "y": [ 233 | 91, 234 | 114 235 | ] 236 | }, 237 | "room6": { 238 | "x": [ 239 | 101, 240 | 123 241 | ], 242 | "y": [ 243 | 142, 244 | 166 245 | ] 246 | }, 247 | "room7": { 248 | "x": [ 249 | 0, 250 | 33 251 | ], 252 | "y": [ 253 | 169, 254 | 190 255 | ] 256 | }, 257 | "room8": { 258 | "x": [ 259 | 141, 260 | 163 261 | ], 262 | "y": [ 263 | 40, 264 | 63 265 | ] 266 | }, 267 | "room9": { 268 | "x": [ 269 | 141, 270 | 163 271 | ], 272 | "y": [ 273 | 117, 274 | 140 275 | ] 276 | }, 277 | "room10": { 278 | "x": [ 279 | 141, 280 | 163 281 | ], 282 | "y": [ 283 | 142, 284 | 166 285 | ] 286 | }, 287 | "room11": { 288 | "x": [ 289 | 101, 290 | 123 291 | ], 292 | "y": [ 293 | 117, 294 | 140 295 | ] 296 | }, 297 | "room12": { 298 | "x": [ 299 | 101, 300 | 123 301 | ], 302 | "y": [ 303 | 14, 304 | 38 305 | ] 306 | }, 307 | "room13": { 308 | "x": [ 309 | 100, 310 | 122 311 | ], 312 | "y": [ 313 | 168, 314 | 190 315 | ] 316 | }, 317 | "room14": { 318 | "x": [ 319 | 101, 320 | 131 321 | ], 322 | "y": [ 323 | 207, 324 | 227 325 | ] 326 | }, 327 | "room15": { 328 | "x": [ 329 | 50, 330 | 65 331 | ], 332 | "y": [ 333 | 168, 334 | 190 335 | ] 336 | }, 337 | "room16": { 338 | "x": [ 339 | 133, 340 | 162 341 | ], 342 | "y": [ 343 | 208, 344 | 227 345 | ] 346 | }, 347 | "room17": { 348 | "x": [ 349 | 35, 350 | 50 351 | ], 352 | "y": [ 353 | 168, 354 | 190 355 | ] 356 | }, 357 | "room18": { 358 | "x": [ 359 | 0, 360 | 24 361 | ], 362 | "y": [ 363 | 216, 364 | 227 365 | ] 366 | }, 367 | "room19": { 368 | "x": [ 369 | 68, 370 | 98 371 | ], 372 | "y": [ 373 | 207, 374 | 227 375 | ] 376 | }, 377 | "room20": { 378 | "x": [ 379 | 25, 380 | 33 381 | ], 382 | "y": [ 383 | 211, 384 | 227 385 | ] 386 | }, 387 | "room21": { 388 | "x": [ 389 | 141, 390 | 163 391 | ], 392 | "y": [ 393 | 168, 394 | 187 395 | ] 396 | }, 397 | "room22": { 398 | "x": [ 399 | 36, 400 | 65 401 | ], 402 | "y": [ 403 | 207, 404 | 227 405 | ] 406 | }, 407 | "room23": { 408 | "x": [ 409 | 68, 410 | 98 411 | ], 412 | "y": [ 413 | 168, 414 | 189 415 | ] 416 | } 417 | } 418 | } -------------------------------------------------------------------------------- /env/map_process.py: -------------------------------------------------------------------------------- 1 | import json 2 | import matplotlib.pyplot as plt 3 | import numpy as np 4 | import copy 5 | import skfmm 6 | 7 | 8 | class RoomMap(object): 9 | 10 | def __init__(self): 11 | self.floor1 = None 12 | self.floor2 = None 13 | self.floor3 = None 14 | # F1, f2, f3 = -16.69344711303711, -5.217403411865234, -0.0499998964369297 15 | self.height_floor1 = -16.693447 16 | self.height_floor2 = -5.2174 17 | self.height_floor3 = -0.0499999 18 | self.floor1_x0, self.floor1_y0, self.floor1_z0 = None, None, None 19 | self.maps_info = [ 20 | {'x0': None, 'y0': None, 'z0': None, 'scale': None, 'width': None, 'height': None, 'grid': None}, 21 | {'x0': None, 'y0': None, 'z0': None, 'scale': None, 'width': None, 'height': None, 'grid': None}, 22 | {'x0': None, 'y0': None, 'z0': None, 'scale': None, 'width': None, 'height': None, 'grid': None} 23 | ] 24 | self.floors = [self.height_floor1, self.height_floor2, self.height_floor3] 25 | 26 | def get_world_position(self, n, i, j): 27 | x0, y0 = self.maps_info[n]['x0'], self.maps_info[n]['z0'] 28 | scale, width, height = self.maps_info[n]['scale'], self.maps_info[n]['width'], self.maps_info[n]['height'] 29 | x, y = x0 + i * scale, y0 + j * scale 30 | return x, self.floors[n], y 31 | 32 | def get_grid_position(self, n, x, y): 33 | x0, y0 = self.maps_info[n]['x0'], self.maps_info[n]['z0'] 34 | scale, width, height = self.maps_info[n]['scale'], self.maps_info[n]['width'], self.maps_info[n]['height'] 35 | # x, y = x0 + i * scale, y0 + j * scale 36 | i, j = (x - x0) / scale, (y - y0) / scale 37 | return round(i), round(j) 38 | 39 | def get_point_info(self, point): 40 | try: 41 | x, y, z = point['x'], point['y'], point['z'] 42 | except: 43 | x, y, z = point[0], point[1], point[2] 44 | 45 | ind, value = min(enumerate(self.floors), key=lambda lis: abs(y - lis[1])) 46 | point_i, point_j = self.get_grid_position(ind, x, z) 47 | try: 48 | return ind, point_i, point_j, self.maps_info[ind]['grid'][point_i][point_j] 49 | except: 50 | i_max, j_max = np.array(self.maps_info[ind]['grid']).shape[0], np.array(self.maps_info[ind]['grid']).shape[ 51 | 1] 52 | p_i, p_j = min(point_i, i_max - 1), min(point_j, j_max - 1) 53 | return ind, p_i, p_j, self.maps_info[ind]['grid'][p_i][p_j] 54 | 55 | def get_an_aligned_world_coordinate_randomly(self, floor, x, y, random=1): 56 | point_i, floor_layer, point_j = self.get_world_position(floor, x, y) 57 | accuracy = self.maps_info[floor]['scale'] 58 | random_x_i = np.random.uniform(point_i, point_i + accuracy) 59 | random_y_j = np.random.uniform(point_j, point_j + accuracy) 60 | return random_x_i, random_y_j 61 | 62 | def get_an_accessible_area(self, x, y, z, radius_meter=1.5, mode=0, sort=1, inflation=0, dis=0): 63 | # mode 0 represent the world position, 1 is the matrix map(x=floor_n, y=map_i, z=map_j) 64 | if not mode: 65 | floor, map_i, map_j, is_obstacle = self.get_point_info([x, y, z]) 66 | else: 67 | floor, map_i, map_j = round(x), round(y), round(z) 68 | is_obstacle = self.maps_info[floor]['grid'][map_i][map_j] 69 | map_array = np.array(self.maps_info[floor]['grid']) 70 | radius = radius_meter / self.maps_info[floor]['scale'] 71 | # Determine the scope of the query domain 72 | min_i, max_i = round(max(0, map_i - radius)), round(min(map_array.shape[0] - 1, map_i + radius)) 73 | min_j, max_j = round(max(0, map_j - radius)), round(min(map_array.shape[1] - 1, map_j + radius)) 74 | # Find feasible points within the specified radius 75 | valid_points = [] 76 | # exclude points with a distance of 1 from obstacles 77 | valid_points = [] 78 | for i in range(min_i, max_i + 1): 79 | for j in range(min_j, max_j + 1): 80 | if map_array[i, j] != 0 and ((i - map_i) ** 2 + (j - map_j) ** 2) <= radius ** 2: 81 | too_close_to_obstacle = False 82 | if inflation: 83 | for ii in range(max(0, i - 1), min(map_array.shape[0], i + 2)): 84 | for jj in range(max(0, j - 1), min(map_array.shape[1], j + 2)): 85 | if map_array[ii, jj] == 0: 86 | too_close_to_obstacle = True 87 | if not too_close_to_obstacle: 88 | valid_points.append((i, j)) 89 | if dis != 0: 90 | distances = [abs(np.sqrt((i - map_i) ** 2 + (j - map_j) ** 2) - dis) for i, j in valid_points] 91 | sorted_valid_points = [point for _, point in sorted(zip(distances, valid_points))] 92 | return floor, sorted_valid_points 93 | if sort: 94 | # Calculate the distance from each feasible point to a given point 95 | distances = [np.sqrt((i - map_i) ** 2 + (j - map_j) ** 2) for i, j in valid_points] 96 | # Sort feasible points in ascending order of distance 97 | sorted_valid_points = [point for _, point in sorted(zip(distances, valid_points))] 98 | # print('here: ', len(sorted_valid_points), ' in radius ', radius_meter, ', scale', self.maps_info[floor]['scale']) 99 | return floor, sorted_valid_points 100 | else: 101 | return floor, valid_points 102 | 103 | def add_room(self, json_data): 104 | # parsing map information 105 | map_id = json_data['mapId'] 106 | floor = json_data['mapName'] 107 | width = json_data['width'] 108 | height = json_data['height'] 109 | points = json_data['points'] 110 | scale = json_data['accuracy'] 111 | positions = json_data['minPoint'] 112 | n_length = scale 113 | x, y = 0, 0 114 | # Create a 2D matrix map 115 | # Analyze points and add point information to the map matrix 116 | n = 0 117 | map_data = [] 118 | xx, yy = [], [] 119 | # json_m = {"mapId": 1, "mapName": "F3", "width": 51, "height": 68, "accuracy": 1.0, "points": []} 120 | po = eval(points) 121 | for point in points: 122 | # point_data = json.loads(point) 123 | map_data.append(1) 124 | matrix = [[0 for _ in range(height)] for _ in range(width)] 125 | navMapPoints = [[None for _ in range(height)] for _ in range(width)] 126 | for i in range(width): 127 | for j in range(height): 128 | index = i * height + j 129 | matrix[i][j] = map_data[index] 130 | navMapPoints[i][j] = {"x": x + i * n_length, "y": y + j * n_length, "data": map_data[index]} 131 | flag = None 132 | if floor == 'F1': 133 | self.floor1 = po 134 | flag = 0 135 | elif floor == 'F2': 136 | self.floor2 = po 137 | flag = 1 138 | elif floor == 'F3': 139 | self.floor3 = po 140 | flag = 2 141 | # plt.show() 142 | self.maps_info[flag]['scale'] = scale 143 | self.maps_info[flag]['width'] = width 144 | self.maps_info[flag]['height'] = height 145 | self.maps_info[flag]['grid'] = po 146 | self.maps_info[flag]['x0'] = positions['x'] 147 | self.maps_info[flag]['y0'] = positions['z'] 148 | self.maps_info[flag]['z0'] = positions['z'] 149 | self.floors[flag] = positions['y'] 150 | # matrix_map = np.array(map_data).reshape((width, height)) 151 | # Convert all values uniformly to values greater than or equal to 0 152 | # Create and initialize matrix 153 | # matrix = [[0 for _ in range(max_y)] for _ in range(max_x)] 154 | 155 | def draw(self, i, n, j): 156 | if isinstance(n, int) and isinstance(i, int) and isinstance(j, int): 157 | pass 158 | else: 159 | n, i, j, is_obstacle = self.get_point_info((i, n, j)) 160 | mat = copy.deepcopy(self.maps_info[n]['grid']) 161 | mat[i][j] = 5 162 | plt.imshow(mat, cmap='gray') 163 | plt.title('Floor{}'.format(n)) 164 | plt.show() 165 | 166 | def plot_map(self): 167 | map = self.floor3 168 | plt.imshow(map, cmap='gray') 169 | plt.title('Room Map') 170 | plt.grid(False) 171 | plt.axis('off') 172 | plt.xticks([]) 173 | plt.yticks([]) 174 | plt.show() 175 | 176 | def dis_matrix(self, mappp, target_position): 177 | map = np.array(mappp) 178 | map_copy = copy.deepcopy(map) 179 | # mask = np.logical_and(abs(X) < 0.1, abs(Y) < 0.5) 180 | # phi = np.ma.MaskedArray(phi, mask) 181 | tar_map = np.ones_like(map) 182 | tar_map[target_position[0]][target_position[1]] = 0 183 | obstacle_map = np.ones_like(map) 184 | obstacle_map[map == 1] = 0 185 | bool_map = np.array(obstacle_map, dtype=bool) 186 | bool_map = np.ma.MaskedArray(tar_map, bool_map) 187 | return skfmm.distance(bool_map) 188 | 189 | def search_route(self, map, current_position, target_position): 190 | target_x, target_y = 1, 1 191 | map_show = copy.deepcopy(map) 192 | dis_matrix = self.dis_matrix(map, target_position) 193 | # plt.imshow(map) 194 | # plt.show() 195 | arrive = 0 196 | while not arrive: 197 | map_show[current_position[0]][current_position[1]] = -50 198 | xx, yy = current_position[0], current_position[1] 199 | if dis_matrix[xx][yy] == 0: 200 | break 201 | min_value, next_x, next_y = 100000, 0, 0 202 | for i in range(xx-1, xx+2): 203 | for j in range(yy-1, yy+2): 204 | if min_value > dis_matrix[i][j]: 205 | min_value, next_x, next_y = dis_matrix[i][j], i, j 206 | current_position = (next_x, next_y) 207 | # break 208 | plt.imshow(map_show) 209 | plt.axis('off') 210 | plt.show() 211 | 212 | ''' 213 | points:["{\"viability\":false,\"position\":{\"x\":-19.94000244140625,\"y\":-0.0499998964369297,\"z\":-59.400001525878909}}"] 214 | points:[[0/1,x,y,z],[0/1,-19.94000,-0.04999,-59.4000],[0/1,x,y,z]] 215 | ''' 216 | 217 | if __name__ == '__main__': 218 | # read Map.json 219 | file_name = "map/map4.json" 220 | with open(file_name, 'r') as file: 221 | json_data = json.load(file) 222 | map = RoomMap() 223 | map.add_room(json_data) -------------------------------------------------------------------------------- /robot/baseline.py: -------------------------------------------------------------------------------- 1 | import time 2 | import re 3 | import difflib 4 | import os 5 | import matplotlib.pyplot as plt 6 | # from env.socket_server import * 7 | # from llm_process import * 8 | from robot.llm_process import * 9 | # Notice the relative path of the main script 10 | 11 | 12 | def instruction_parsing_res(prs, instruction, resume): 13 | task_planning = instruction_parsing(instruction, resume) 14 | # task_planning = ['white bagged cake', 'marble dinner counter', 'living room', 'man with white coat', 'bed room'] 15 | object_room, human_room = room_filter(prs, task_planning[2])[0], room_filter(prs, task_planning[4])[0] 16 | possible_receptacles, receptacles_index = approach_landmark(prs, task_planning[1], object_room) 17 | result_data = {'target_object': task_planning[0], 'target_object_receptacle': possible_receptacles, 18 | 'target_object_room': (object_room, task_planning[2]), 'target_human': task_planning[3], 19 | 'target_human_room': (human_room, task_planning[4])} 20 | return result_data 21 | 22 | 23 | def calculate_similarity(a, b): 24 | len_a, len_b = len(a), len(b) 25 | dp = [[0] * (len_b + 1) for _ in range(len_a + 1)] 26 | for i in range(1, len_a + 1): 27 | for j in range(1, len_b + 1): 28 | if a[i - 1] == b[j - 1]: 29 | dp[i][j] = dp[i - 1][j - 1] + 1 30 | else: 31 | dp[i][j] = max(dp[i - 1][j], dp[i][j - 1]) 32 | lcs_length = dp[len_a][len_b] 33 | similarity = 2.0 * lcs_length / (len_a + len_b) 34 | if similarity > 0.5: 35 | return True 36 | else: 37 | return False 38 | 39 | 40 | def delivery_execution(prs, instruction, resume): 41 | ''' 42 | 1. Analyze language instructions to determine the target 43 | 2. Locate/proceed to the room where the target item is located 44 | 3. Find the container based on the semantic map 45 | 4. Identify the target item using camera information (or explore if not found) 46 | 5. Grasp the target item based on segmentation information 47 | 6. After grasping, proceed to the room based on the information from step 1 48 | 7. Locate and recognize the target person 49 | ''' 50 | # prs.agent.initial_pose() 51 | task_planning = instruction_parsing(instruction, resume) 52 | # task_planning = ['white bagged cake', 'marble dinner counter', 'living room', 'man with white coat', 'bed room'] 53 | # print(task_planning) 54 | object_room, human_room = room_filter(prs, task_planning[2])[0], room_filter(prs, task_planning[4])[0] 55 | # print(object_room, human_room) 56 | speed_time, camera_pitch = 3, 20 57 | prs.sim_speed(speed_time) 58 | # Unity rendering and runtime acceleration, notice the possibility of logical errors caused by fast rendering speed 59 | go_to_location(prs, object_room) 60 | prs.sim_speed(1) 61 | camera_rgb, camera_degree = scene_understanding(prs, task_planning[0], task_planning[1], pitch=camera_pitch, mode=1) 62 | possible_receptacles, receptacles_index = approach_landmark(prs, task_planning[1], object_room) 63 | for r_index, rec in enumerate(possible_receptacles): 64 | if camera_rgb is not None: break 65 | for time_i in range(2): 66 | prs.sim_speed(speed_time) 67 | prs.agent.goto_receptacle(room=object_room, recepacle=receptacles_index[r_index], random=1) 68 | prs.sim_speed(1) 69 | camera_rgb, camera_degree = scene_understanding(prs, task_planning[0], task_planning[1], pitch=camera_pitch, mode=1) 70 | if camera_rgb is not None: break 71 | if camera_rgb is None: pass 72 | # exploration of environment 73 | else: 74 | break 75 | if camera_rgb is not None: 76 | # Operate items and identify, determine room 77 | prs.sim_speed(1) 78 | manipulate_target(prs, task_planning[0], camera_degree, pitch=camera_pitch) 79 | time.sleep(1) 80 | prs.sim_speed(speed_time) 81 | go_to_location(prs, human_room) 82 | prs.sim_speed(1) 83 | camera_rgb, camera_degree = scene_understanding(prs, task_planning[3], pitch=camera_pitch, mode=1) 84 | search_result = 0 85 | if camera_rgb is not None: 86 | search_result = approach_target(camera_rgb, camera_degree, task_planning[3], 3) 87 | if not search_result: 88 | for room_i in prs.objs_data.room_area: 89 | if room_i['name'] == human_room: 90 | room_info = room_i 91 | break 92 | if room_info is None: return None 93 | receptacles_info = room_info['receptacles_list'] 94 | # Locate possible receptacles in the room 95 | for r_index, rec in enumerate(receptacles_info): 96 | if camera_rgb is not None: break 97 | prs.sim_speed(speed_time) 98 | prs.agent.goto_receptacle(room=human_room, recepacle=r_index, random=1) 99 | prs.sim_speed(1) 100 | camera_rgb, camera_degree = scene_understanding(prs, task_planning[3], pitch=camera_pitch, 101 | mode=1) 102 | if camera_rgb is None: 103 | pass 104 | else: 105 | search_result = approach_target(camera_rgb, camera_degree, task_planning[3], 3) 106 | break 107 | return 1 108 | 109 | 110 | def target_matching(target, options=['room']): 111 | best_match, matches_with_similarity = None, [] 112 | target_preprocess = re.sub(r'[^a-zA-Z0-9]', '', target).lower() 113 | for location in options: 114 | similarity = difflib.SequenceMatcher(None, target_preprocess, location.lower()).ratio() 115 | matches_with_similarity.append((similarity, location)) 116 | matches_with_similarity.sort(key=lambda x: x[0], reverse=True) 117 | sorted_matches = [match[1] for match in matches_with_similarity] 118 | # Sort by similarity 119 | return sorted_matches 120 | 121 | 122 | def instruction_parsing(task_instruction, description=None): 123 | background = 'Now you are a robotic assistant tasked with assisting in executing a human-centred item delivery mission. Please deduce the necessary execution actions and objectives based on the following task instructions:' 124 | npc_info = 'Introduction to the target person for the transportation mission: ' + description 125 | requirement = 'The following information needs to be extracted from the task: the target item, the container that holds the target item, the room where the target item is located, the visual characteristics of the target person (excluding the name but including gender for identification, such as a woman in a white dress), and the room where the target person is located (including the room number such as bedroom 1, with only the room name information and excluding ambiguous location descriptions, for example, outputting office 1). Only the inference results are required, the format is as follows:' 126 | reference = '' 127 | prompt = background + task_instruction + npc_info + requirement + reference 128 | task_parameter = None 129 | for i in range(3): 130 | reasoning_result = llm_interaction(prompt) 131 | # reasoning_result = '' 132 | # print(reasoning_result) 133 | try: 134 | cleaned_string = reasoning_result.strip('<>').strip() 135 | items = cleaned_string.split(',') 136 | task_parameter = [item.strip() for item in items] 137 | break 138 | except Exception as e: 139 | continue 140 | # decoded_str = s.encode('utf-8').decode('unicode-escape') 141 | return task_parameter 142 | 143 | 144 | def go_to_location(prs, destination='bed room 1'): 145 | # go to the target room through global map 146 | if destination is not None: 147 | # prs.agent.goto_target_goal((-8.4, 0.1, 7.9), radius=1.5, position_mode=1) 148 | position = None 149 | for room in prs.objs_data.room_sampling_points: 150 | if room.split('_')[1] == destination: 151 | position = prs.objs_data.room_sampling_points[room] 152 | if position is None: return 0 153 | target = position[0] 154 | prs.agent.goto_target_goal(target, 1.25, position_mode=1) 155 | # Facing the center position of the room space 156 | prs.agent.head_camera_look_at(position[1], accuracy=0) 157 | 158 | 159 | def room_filter(prs, destination, floor=2): 160 | rooms = prs.objs_data.room_area 161 | locations = [roo['name'] for roo in rooms] 162 | # locations = prs.objs_data.buliding_rooms[floor] 163 | best_match_rooms = target_matching(destination, locations) 164 | return best_match_rooms 165 | 166 | 167 | def approach_landmark(prs, landmark, room): 168 | # Determine which one in the list has a higher similarity to the destination. 169 | # For 'bed room 1', it is desired to output 'bedroom1' instead of 'bedroom' 170 | room_info = None 171 | for room_i in prs.objs_data.room_area: 172 | if room_i['name'] == room: 173 | room_info = room_i 174 | break 175 | if room_info is None: return None 176 | landmarks, original_index = room_info['receptacles_list'], [] 177 | potential_receptacles = target_matching(landmark, landmarks) 178 | for rec in potential_receptacles: 179 | original_index.append(landmarks.index(rec)) 180 | return potential_receptacles, original_index 181 | # Selecting potential containers through semantic matching 182 | 183 | 184 | def scene_understanding(prs, target, surrounding=None, pitch=20, mode=0): 185 | # The angle of horizontal X-axis rotation, equivalent to the pitch angle, whether to segment 186 | observation_views = [0, -60, 60] 187 | for degree in observation_views: 188 | if surrounding is not None: 189 | target = target + ' or ' + surrounding 190 | context1 = 'Please answer yes or no. If there is ' 191 | context2 = ', or there may be similar objects or content. Only answer yes or no!' 192 | detect_prompt = context1 + target + context2 193 | head_camera_rgb = prs.agent.observation(degree=degree, camera=0, up_down=pitch) 194 | if mode: 195 | recognition_result = lmm_interaction(detect_prompt, head_camera_rgb) 196 | if 'yes' in recognition_result.lower(): 197 | return head_camera_rgb, degree 198 | else: 199 | if surrounding is not None: 200 | target = target + '. ' + surrounding 201 | seg = object_detect_module(head_camera_rgb, target) 202 | if seg is not None: 203 | return head_camera_rgb, degree 204 | prs.agent.joint_control(joint_id=14, target=0) 205 | # Adjust the camera to its original degree 0 (look straight ahead) 206 | return None, None 207 | 208 | 209 | def approach_target(ego_view, camera_degree, target, distance=2): 210 | target_mask = object_detect_module(ego_view, target) 211 | res = 0 212 | if target_mask is not None: 213 | depth_m = prs.agent.get_depth(0) 214 | tar_distance, view_angle = prs.agent.depth_estimation(target_mask, depth_m, field_of_view=90) 215 | if tar_distance > distance: 216 | point = prs.agent.target_direction(degree=camera_degree, distance=tar_distance, target_degree_view=view_angle) 217 | res = prs.agent.goto_target_goal(point, radius=2.5, position_mode=1, inflation=1) 218 | prs.agent.direction_adjust(point, pos_input=1) 219 | else: 220 | res = 1 221 | return res 222 | 223 | 224 | def manipulate_target(prs, target_obj, degree, pitch=20): 225 | s_now = prs.agent.observation(degree, up_down=pitch) 226 | seg, result = object_detect_module(s_now, target_obj), None 227 | if seg is not None: 228 | result = prs.agent.object_interaction(input_matrix=seg, manipulaton=1, type=0) 229 | return result 230 | 231 | 232 | if __name__ == "__main__": 233 | # Environment initialization 234 | prs = PrsEnv(is_print=1, rendering=1, start_up_mode=1) 235 | 236 | with open(os.path.join('task', 'dataset', 'deliver_task_test_set.json'), 'r') as file: 237 | task_data = json.load(file) 238 | tasks = list(task_data.keys()) 239 | errors, error_num, task_results = [], 0, dict() 240 | for task_i, task_id in enumerate(tasks): 241 | print('task:', task_id) 242 | task = task_data[task_id] 243 | task_npc_id = task['npc_id'] 244 | start_time = time.time() 245 | instruction, npc_information, data = prs.delivery_task_import(task) 246 | now = prs.env_time.current_date.strftime("%d %H:%M:%S") 247 | try: 248 | # ------------ baseline method ----------------- 249 | time.sleep(0.5) 250 | delivery_execution(prs, instruction, npc_information) 251 | except Exception as e: 252 | print(e, 'error') 253 | break 254 | result = prs.delivery_task_evaluate(task, score=1, save=0) 255 | print(result['task_score']) 256 | # ====== if you want to save the result as json file to submission on Eval AI ========== 257 | # result_save = prs.delivery_task_evaluate(task, score=0, save=1) 258 | # task_results[task_id] = result_save 259 | # with open('task/result/deliver_task_result.json', 'w') as file: 260 | # json.dump(task_results, file, indent=4) 261 | # --------------------------------------------------------------------------------------- 262 | time.sleep(3) 263 | prs.finish_env() -------------------------------------------------------------------------------- /env/data/npc_data.json: -------------------------------------------------------------------------------- 1 | { 2 | "npc": [ 3 | { 4 | "description": "I'm {name} from europe, a computer engineer based in a co-working space. My favorite attire includes light-colored jackets. I often work late into the night and occasionally participate in meetings during the afternoon. Besides working, eating, and attending meetings, I usually stay in Room 0.", 5 | "feature": "I'm {name}, a man with light-colored jackets.", 6 | "character": "a man with light-colored jackets.", 7 | "information": { 8 | "id": 0, 9 | "name": null, 10 | "age": 31, 11 | "gender": 0, 12 | "room": "F3_Bedroom_0", 13 | "office": "F3_OfficeSpaceRoom", 14 | "job": "IT", 15 | "npc_name": "Npc_CasualMale", 16 | "schedule": null 17 | } 18 | }, 19 | { 20 | "description": "I'm {name}, a scientific advisor at a polar research station. My room number is 1 and my office is located in office 1, next to the conference room. I lead a regular life and often spend some time in the hall in the evenings. My fashion preferences include blue shirts and black glasses.", 21 | "feature": "I'm {name}, a woman, and my outfit includes blue shirt and black glasses.", 22 | "character": "a woman with blue shirt and black glasses", 23 | "information": { 24 | "id": 1, 25 | "name": null, 26 | "age": 48, 27 | "gender": 1, 28 | "room": "F3_Bedroom_1", 29 | "office": "F3_OfficeRoom01", 30 | "job": "Advisor", 31 | "npc_name": "Npc_BusinessFemale", 32 | "schedule": null 33 | } 34 | }, 35 | { 36 | "description": " I'm {name}, a supervisor who's often busy with meetings and office work. However, I'm also a fitness enthusiast. My office is in Office 2, and Room 2 is where I stay. If you see a middle-aged man in a white shirt, that's me.", 37 | "feature": "I'm {name}, a middle-aged man in a white shirt.", 38 | "character": "a middle-aged man in a white shirt ", 39 | "information": { 40 | "id": 2, 41 | "name": null, 42 | "age": 52, 43 | "gender": 0, 44 | "room": "F3_Bedroom_2", 45 | "office": "F3_OfficeRoom02", 46 | "job": "Supervisor", 47 | "npc_name": "Npc_BusinessMale", 48 | "schedule": null 49 | } 50 | }, 51 | { 52 | "description": "I'm {name}, an intern from Asia. You can find me in the shared office, and my room number is 3. Besides attending afternoon meetings and evening workouts, I spend most of my time in the office studying and handling data. I have some trendy hats and hoodies.", 53 | "feature": "I'm {name} and have trendy hat and hoody, a young lady.", 54 | "character": "a young lady with trendy hat and hoody", 55 | "information": { 56 | "id": 3, 57 | "name": null, 58 | "age": 24, 59 | "gender": 1, 60 | "room": "F3_Bedroom_3", 61 | "office": "F3_OfficeSpaceRoom", 62 | "job": "Internship", 63 | "npc_name": "Npc_CasualFemale", 64 | "schedule": null 65 | } 66 | }, 67 | { 68 | "description": "I am designer {name}. I reside in Room 4 and work in the shared workspace opposite the conference room. During my lunch break and evenings, you might find me seeking inspiration on the couch in the living room. I'm often spotted in a black jacket.", 69 | "feature": "I am designer {name} and spotted in a black jacket, a gentleman.", 70 | "character": "a gentleman with black jacket", 71 | "information": { 72 | "id": 4, 73 | "name": null, 74 | "age": 37, 75 | "gender": 0, 76 | "room": "F3_Bedroom_4", 77 | "office": "F3_OfficeSpaceRoom", 78 | "job": "Designer", 79 | "npc_name": "Npc_CasualMale", 80 | "schedule": null 81 | } 82 | }, 83 | { 84 | "description": "I'm doctor {name}. I can usually be found in the medical room on the second floor. During mealtimes, I sometimes have a light meal at the tables in the hall or kitchen. I typically wear a white lab coat and reside in Room 5.", 85 | "feature": "I'm doctor {name} and wear a white lab coat, a woman.", 86 | "character": "a woman with white lab coat", 87 | "information": { 88 | "id": 5, 89 | "name": null, 90 | "age": 45, 91 | "gender": 1, 92 | "room": "F3_Bedroom_5", 93 | "office": "F2_MedicalRoom03", 94 | "job": "Doctor", 95 | "npc_name": "Npc_UniformFemale", 96 | "schedule": null 97 | } 98 | }, 99 | { 100 | "description": "I'm {name}, a researcher at a polar research station, specializing in biological experiments. Currently, I'm working in the shared office area to process experimental data. Sometimes, I take a break in the living room for a drink. My room number is 6, and I often wear a white lab coat.", 101 | "feature": "I'm {name} and wear a white lab coat, a lady.", 102 | "character": "a lady with white lab coat", 103 | "information": { 104 | "id": 6, 105 | "name": null, 106 | "age": 29, 107 | "gender": 1, 108 | "room": "F3_Bedroom_6", 109 | "office": "F3_OfficeSpaceRoom", 110 | "job": "Laboratory", 111 | "npc_name": "Npc_UniformFemale", 112 | "schedule": null 113 | } 114 | }, 115 | { 116 | "description": "I'm {name}, a facility maintenance technician. I oversee the equipment on the first floor and reside in Room 7. While I spend most of my time with the equipment, apart from meals in the third-floor dining area, I also take breaks in the living room. You can easily spot me in my signature blue work uniform.", 117 | "feature": "I'm {name} with my blue work uniform, a man.", 118 | "character": "a man with blue work uniform", 119 | "information": { 120 | "id": 7, 121 | "name": null, 122 | "age": 40, 123 | "gender": 0, 124 | "room": "F3_Bedroom_7", 125 | "office": "F1_ConveyorRoom", 126 | "job": "Equipment", 127 | "npc_name": "Npc_UniformMale", 128 | "schedule": null 129 | } 130 | }, 131 | { 132 | "description": " I'm {name}, a worker here. My room is number 8, and I usually wear an orange vest. My daily routine is quite regular. I spend my days handling cargo on the first floor and having meals in the kitchen. In the evenings, I retreat to my room to rest.", 133 | "feature": "I'm {name} and wear an orange vest, a man.", 134 | "character": "a man with orange vest", 135 | "information": { 136 | "id": 8, 137 | "name": null, 138 | "age": 42, 139 | "gender": 0, 140 | "room": "F3_Bedroom_8", 141 | "office": "F1_EnergyRoom", 142 | "job": "Worker", 143 | "npc_name": "Npc_UniformMale", 144 | "schedule": null 145 | } 146 | }, 147 | { 148 | "description": "I'm an experimental researcher {name}. I conduct intricate biological and chemical experiments in the second-floor laboratory. I reside in Room 9 on the third floor, identifiable by my white lab coat and yellow hair. I often cook for myself in the kitchen and enjoy my meals at the table in the main hall.", 149 | "feature": "I'm {name}, a gentleman, identifiable by my white lab coat and yellow hair.", 150 | "character": "a gentleman with white lab coat and yellow hair", 151 | "information": { 152 | "id": 9, 153 | "name": null, 154 | "age": 31, 155 | "gender": 0, 156 | "room": "F3_Bedroom_9", 157 | "office": "F2_LabRoom03", 158 | "job": "Medical", 159 | "npc_name": "Npc_UniformMale", 160 | "schedule": null 161 | } 162 | } 163 | ], 164 | "names": { 165 | "0": [ 166 | "Jack", 167 | "Peter", 168 | "Michael", 169 | "James", 170 | "David", 171 | "Robert", 172 | "John", 173 | "William", 174 | "Richard", 175 | "Charles", 176 | "Joseph", 177 | "Thomas", 178 | "Christopher", 179 | "Daniel", 180 | "Matthew", 181 | "Anthony", 182 | "Andrew", 183 | "Paul", 184 | "Stephen", 185 | "Edward", 186 | "Jonathan", 187 | "Kevin", 188 | "Justin", 189 | "Henry", 190 | "Adam", 191 | "Joshua", 192 | "Nicholas", 193 | "Ryan", 194 | "Brian", 195 | "Samuel", 196 | "Eric", 197 | "Jason", 198 | "Mark", 199 | "George", 200 | "Timothy", 201 | "Jeffrey", 202 | "Alexander", 203 | "Joseph", 204 | "Adam", 205 | "Jacob", 206 | "Noah", 207 | "Ethan", 208 | "William", 209 | "Michael", 210 | "Alexander", 211 | "James", 212 | "Benjamin", 213 | "Oliver", 214 | "Matthew", 215 | "David", 216 | "Daniel", 217 | "Joseph", 218 | "Jackson", 219 | "Lucas", 220 | "Logan", 221 | "Nathan", 222 | "Elijah", 223 | "Christian", 224 | "Gabriel", 225 | "Owen", 226 | "Aiden", 227 | "Isaac", 228 | "Sebastian", 229 | "Lucas", 230 | "Jack", 231 | "Hunter", 232 | "Jayden", 233 | "Andrew", 234 | "Dylan", 235 | "Wyatt", 236 | "Liam", 237 | "Caleb", 238 | "Evan", 239 | "Charles", 240 | "Leo", 241 | "Maxwell", 242 | "Adam", 243 | "Aaron", 244 | "Abraham", 245 | "Adam", 246 | "Adrian", 247 | "Alan", 248 | "Albert", 249 | "Alec", 250 | "Alex", 251 | "Alexander", 252 | "Alfred", 253 | "Ali", 254 | "Allen", 255 | "Alvin", 256 | "Andre", 257 | "Andrew", 258 | "Andy", 259 | "Anthony", 260 | "Antonio", 261 | "Archer", 262 | "Ari", 263 | "Arthur", 264 | "Asher", 265 | "Austin", 266 | "Axel", 267 | "Barry", 268 | "Ben", 269 | "Benjamin", 270 | "Blake", 271 | "Bradley", 272 | "Brandon", 273 | "Brayden", 274 | "Brendan", 275 | "Brian", 276 | "Bruce", 277 | "Bryan", 278 | "Caleb", 279 | "Calvin", 280 | "Cameron", 281 | "Carl", 282 | "Carlos", 283 | "Carter", 284 | "Charles", 285 | "Charlie", 286 | "Chase", 287 | "Chris", 288 | "Christian", 289 | "Christopher", 290 | "Clarence", 291 | "Clark", 292 | "Cody", 293 | "Cole", 294 | "Colin", 295 | "Collin", 296 | "Connor", 297 | "Cooper", 298 | "Craig", 299 | "Curtis", 300 | "Cyrus", 301 | "Dale", 302 | "Dalton", 303 | "Damian", 304 | "Daniel", 305 | "Danny", 306 | "Darius", 307 | "Darren", 308 | "David", 309 | "Dean", 310 | "Dennis", 311 | "Derek", 312 | "Derrick", 313 | "Devin", 314 | "Diego", 315 | "Dominic", 316 | "Donald", 317 | "Douglas", 318 | "Dylan", 319 | "Eddie", 320 | "Edward", 321 | "Edwin", 322 | "Eli", 323 | "Elijah", 324 | "Elliot", 325 | "Elliott", 326 | "Emmanuel", 327 | "Eric", 328 | "Ethan", 329 | "Evan", 330 | "Everett", 331 | "Ezra", 332 | "Felix", 333 | "Fernando", 334 | "Francis", 335 | "Frank", 336 | "Franklin", 337 | "Frederick", 338 | "Gabriel", 339 | "Gavin", 340 | "George", 341 | "Gerald", 342 | "Gilbert", 343 | "Giovanni", 344 | "Graham", 345 | "Grant", 346 | "Gregory", 347 | "Harrison", 348 | "Harry", 349 | "Hayden", 350 | "Henry", 351 | "Hudson", 352 | "Hugh", 353 | "Hunter", 354 | "Ian", 355 | "Isaac", 356 | "Isaiah", 357 | "Ivan", 358 | "Jack", 359 | "Jackson", 360 | "Jacob", 361 | "Jake", 362 | "James", 363 | "Jamie", 364 | "Jared", 365 | "Jason", 366 | "Jaxon", 367 | "Jay", 368 | "Jayden", 369 | "Jeff", 370 | "Jeffrey", 371 | "Jeremiah", 372 | "Jeremy", 373 | "Jerome", 374 | "Jerry", 375 | "Jesse", 376 | "Jesus", 377 | "Jim", 378 | "Jimmy", 379 | "Joe", 380 | "Joel", 381 | "John", 382 | "Johnny", 383 | "Jonah", 384 | "Jonathan", 385 | "Jordan", 386 | "Joseph", 387 | "Joshua", 388 | "Josiah", 389 | "Juan", 390 | "Julian", 391 | "Justin", 392 | "Kai", 393 | "Kaleb", 394 | "Kameron", 395 | "Karl", 396 | "Keith", 397 | "Kenneth", 398 | "Kevin", 399 | "Kieran", 400 | "Kyle", 401 | "Landon", 402 | "Larry", 403 | "Lawrence", 404 | "Lee", 405 | "Leo", 406 | "Leon", 407 | "Leonard", 408 | "Liam", 409 | "Lincoln", 410 | "Logan", 411 | "Louis", 412 | "Lucas", 413 | "Luke", 414 | "Malachi", 415 | "Malcolm", 416 | "Marcus", 417 | "Mario", 418 | "Mark", 419 | "Marshall", 420 | "Martin", 421 | "Mason", 422 | "Mathew", 423 | "Matthew", 424 | "Max", 425 | "Maxwell", 426 | "Miles", 427 | "Mitchell", 428 | "Nathan", 429 | "Nathaniel", 430 | "Neil", 431 | "Nicholas", 432 | "Noah", 433 | "Nolan", 434 | "Oliver", 435 | "Oscar", 436 | "Owen", 437 | "Patrick", 438 | "Paul", 439 | "Peter", 440 | "Philip", 441 | "Preston", 442 | "Quentin", 443 | "Ralph", 444 | "Randall", 445 | "Randy", 446 | "Raymond", 447 | "Reid", 448 | "Richard", 449 | "Rick", 450 | "Riley", 451 | "Robert", 452 | "Robin", 453 | "Rodney", 454 | "Roger", 455 | "Ronald", 456 | "Ross", 457 | "Roy", 458 | "Russell", 459 | "Ryan", 460 | "Sam", 461 | "Samuel", 462 | "Scott", 463 | "Sean", 464 | "Sebastian", 465 | "Seth", 466 | "Shane", 467 | "Shaun", 468 | "Shawn", 469 | "Sidney", 470 | "Simon", 471 | "Spencer", 472 | "Stanley", 473 | "Stephen", 474 | "Steve", 475 | "Steven", 476 | "Stuart", 477 | "Taylor", 478 | "Ted", 479 | "Terrence", 480 | "Terry", 481 | "Theodore", 482 | "Thomas", 483 | "Timothy", 484 | "Todd", 485 | "Tommy", 486 | "Tony", 487 | "Travis", 488 | "Trevor", 489 | "Troy", 490 | "Tyler", 491 | "Victor", 492 | "Vincent", 493 | "Walter", 494 | "Warren", 495 | "Wayne", 496 | "Wesley", 497 | "William", 498 | "Willie", 499 | "Wyatt", 500 | "Xavier", 501 | "Zachary", 502 | "Zane" 503 | ], 504 | "1": [ 505 | "Emily", 506 | "Olivia", 507 | "Sophia", 508 | "Isabella", 509 | "Ava", 510 | "Mia", 511 | "Charlotte", 512 | "Amelia", 513 | "Harper", 514 | "Abigail", 515 | "Elizabeth", 516 | "Sofia", 517 | "Chloe", 518 | "Ella", 519 | "Zoe", 520 | "Samantha", 521 | "Grace", 522 | "Emily", 523 | "Emma", 524 | "Madison", 525 | "Lily", 526 | "Anna", 527 | "Aubrey", 528 | "Hannah", 529 | "Addison", 530 | "Kayla", 531 | "Alyssa", 532 | "Ashley", 533 | "Bella", 534 | "Natalie", 535 | "Victoria", 536 | "Sarah", 537 | "Kaitlyn", 538 | "Layla", 539 | "Alexa", 540 | "Olivia", 541 | "Savannah", 542 | "Sydney", 543 | "Taylor", 544 | "Kimberly", 545 | "Morgan", 546 | "Jasmine", 547 | "Eva", 548 | "Riley", 549 | "Quinn", 550 | "Jasmine", 551 | "Gabriella", 552 | "Aria", 553 | "Elena", 554 | "Isabel", 555 | "Nora", 556 | "Maya", 557 | "Lillian", 558 | "Stella", 559 | "Gabrielle", 560 | "Lauren", 561 | "Skylar", 562 | "Natalia", 563 | "Katherine", 564 | "Alexa", 565 | "Camila", 566 | "Makayla", 567 | "Kylie", 568 | "Brooklyn", 569 | "Audrey", 570 | "Ivy", 571 | "Eleanor", 572 | "Adriana", 573 | "Leah", 574 | "Mia", 575 | "Caroline", 576 | "Kennedy", 577 | "Piper", 578 | "Maria", 579 | "Lilyana", 580 | "Julia", 581 | "Isla", 582 | "Evelyn", 583 | "Genesis", 584 | "Melanie", 585 | "Scarlett", 586 | "Hayley", 587 | "Serena", 588 | "Valentina", 589 | "Angelina", 590 | "Claire", 591 | "Maya", 592 | "Abigail", 593 | "Ada", 594 | "Adela", 595 | "Adriana", 596 | "Agnes", 597 | "Aisha", 598 | "Alana", 599 | "Alejandra", 600 | "Alexa", 601 | "Alexandra", 602 | "Alice", 603 | "Alina", 604 | "Alisa", 605 | "Alisha", 606 | "Alivia", 607 | "Alyssa", 608 | "Amalia", 609 | "Amanda", 610 | "Amara", 611 | "Amaya", 612 | "Amelia", 613 | "Amira", 614 | "Amy", 615 | "Ana", 616 | "Anastasia", 617 | "Andrea", 618 | "Angela", 619 | "Angelica", 620 | "Angelina", 621 | "Angie", 622 | "Anika", 623 | "Anita", 624 | "Anna", 625 | "Annabelle", 626 | "Anne", 627 | "Annie", 628 | "April", 629 | "Aria", 630 | "Ariana", 631 | "Arianna", 632 | "Ariel", 633 | "Arabella", 634 | "Aria", 635 | "Ashley", 636 | "Aspen", 637 | "Athena", 638 | "Aubree", 639 | "Aubrey", 640 | "Audrey", 641 | "Aurora", 642 | "Autumn", 643 | "Ava", 644 | "Avery", 645 | "Ayla", 646 | "Bailey", 647 | "Barbara", 648 | "Beatrice", 649 | "Bella", 650 | "Bernadette", 651 | "Bethany", 652 | "Bianca", 653 | "Blake", 654 | "Brenda", 655 | "Briana", 656 | "Brianna", 657 | "Bridget", 658 | "Brinley", 659 | "Britney", 660 | "Brooke", 661 | "Brynn", 662 | "Caitlin", 663 | "Caitlyn", 664 | "Camila", 665 | "Candice", 666 | "Carla", 667 | "Carmen", 668 | "Caroline", 669 | "Carolyn", 670 | "Casey", 671 | "Cassandra", 672 | "Catherine", 673 | "Cecilia", 674 | "Celeste", 675 | "Celia", 676 | "Charlotte", 677 | "Chelsea", 678 | "Cheyenne", 679 | "Chloe", 680 | "Christina", 681 | "Christine", 682 | "Cindy", 683 | "Claire", 684 | "Clara", 685 | "Clarissa", 686 | "Claudia", 687 | "Colette", 688 | "Colleen", 689 | "Constance", 690 | "Cora", 691 | "Corinne", 692 | "Courtney", 693 | "Crystal", 694 | "Cynthia", 695 | "Daisy", 696 | "Dakota", 697 | "Dana", 698 | "Daniela", 699 | "Danielle", 700 | "Daphne", 701 | "Darlene", 702 | "Delaney", 703 | "Delilah", 704 | "Denise", 705 | "Desiree", 706 | "Destiny", 707 | "Diana", 708 | "Dina", 709 | "Dora", 710 | "Dorothy", 711 | "Eden", 712 | "Eleanor", 713 | "Elena", 714 | "Eliana", 715 | "Eliza", 716 | "Elizabeth", 717 | "Ella", 718 | "Ellie", 719 | "Eloise", 720 | "Elsa", 721 | "Emerson", 722 | "Emery", 723 | "Emilia", 724 | "Emily", 725 | "Emma", 726 | "Emmy", 727 | "Erica", 728 | "Erika", 729 | "Erin", 730 | "Esmeralda", 731 | "Estelle", 732 | "Esther", 733 | "Eva", 734 | "Evangeline", 735 | "Eve", 736 | "Evelyn", 737 | "Faith", 738 | "Fatima", 739 | "Fiona", 740 | "Francesca", 741 | "Gabriela", 742 | "Gabriella", 743 | "Gabrielle", 744 | "Gemma", 745 | "Genesis", 746 | "Genevieve", 747 | "Georgia", 748 | "Gianna", 749 | "Giselle", 750 | "Gloria", 751 | "Grace", 752 | "Gracie", 753 | "Gwendolyn", 754 | "Hadley", 755 | "Hailey", 756 | "Haley", 757 | "Hanna", 758 | "Hannah", 759 | "Harmony", 760 | "Harper", 761 | "Hazel", 762 | "Heidi", 763 | "Helena", 764 | "Hope", 765 | "Imani", 766 | "India", 767 | "Iris", 768 | "Isabel", 769 | "Isabella", 770 | "Isabelle", 771 | "Ivy", 772 | "Jacqueline", 773 | "Jade", 774 | "Jasmine", 775 | "Jayla", 776 | "Jenna", 777 | "Jennifer", 778 | "Jessica", 779 | "Jillian", 780 | "Jocelyn", 781 | "Joelle", 782 | "Josephine", 783 | "Josie", 784 | "Joy", 785 | "Julia", 786 | "Juliana", 787 | "Julianna", 788 | "Julie", 789 | "Juliet", 790 | "June", 791 | "Kaia", 792 | "Kaitlyn", 793 | "Kara", 794 | "Karen", 795 | "Karina", 796 | "Katelyn", 797 | "Katherine", 798 | "Kathleen", 799 | "Katie", 800 | "Kayla", 801 | "Kaylee", 802 | "Keira", 803 | "Kelly", 804 | "Kelsey", 805 | "Kendall", 806 | "Kennedy", 807 | "Khloe", 808 | "Kimberly", 809 | "Kinsley", 810 | "Kira", 811 | "Kylie", 812 | "Laila", 813 | "Lana", 814 | "Lara", 815 | "Laura", 816 | "Lauren", 817 | "Layla", 818 | "Leah", 819 | "Leila", 820 | "Lena", 821 | "Leona", 822 | "Leslie", 823 | "Lila", 824 | "Liliana", 825 | "Lillian", 826 | "Lily", 827 | "Linda", 828 | "London", 829 | "Lorelei", 830 | "Lucia", 831 | "Lucy", 832 | "Luna", 833 | "Lydia", 834 | "Lyla", 835 | "Lyric", 836 | "Mackenzie", 837 | "Macy", 838 | "Madeline", 839 | "Madelyn", 840 | "Madison", 841 | "Maeve", 842 | "Maggie", 843 | "Makayla", 844 | "Makenzie", 845 | "Malaysia", 846 | "Malia", 847 | "Mallory", 848 | "Mara", 849 | "Margaret", 850 | "Maria", 851 | "Mariah", 852 | "Marina", 853 | "Marissa", 854 | "Marjorie", 855 | "Marley", 856 | "Mary", 857 | "Maya", 858 | "Mckenna", 859 | "Megan", 860 | "Melanie", 861 | "Melissa", 862 | "Melody", 863 | "Meredith", 864 | "Mia", 865 | "Michaela", 866 | "Michelle", 867 | "Mikayla", 868 | "Mila", 869 | "Miley", 870 | "Miranda", 871 | "Miriam", 872 | "Molly", 873 | "Monica", 874 | "Morgan", 875 | "Mya", 876 | "Nadia", 877 | "Naomi", 878 | "Natalia", 879 | "Natalie", 880 | "Natasha", 881 | "Naya", 882 | "Nevaeh", 883 | "Nicole", 884 | "Nina", 885 | "Noelle", 886 | "Norah", 887 | "Nora", 888 | "Nova", 889 | "Olivia", 890 | "Ophelia", 891 | "Paige", 892 | "Paisley", 893 | "Payton", 894 | "Penelope", 895 | "Phoebe", 896 | "Piper", 897 | "Presley", 898 | "Priscilla", 899 | "Rachel", 900 | "Raegan", 901 | "Raina", 902 | "Rebecca", 903 | "Reese", 904 | "Regina", 905 | "Riley", 906 | "Rosa", 907 | "Rose", 908 | "Rosie", 909 | "Ruby", 910 | "Ruth", 911 | "Sabrina", 912 | "Sadie", 913 | "Samantha", 914 | "Samara", 915 | "Sandra", 916 | "Sara", 917 | "Sarah", 918 | "Savannah", 919 | "Scarlet", 920 | "Serena", 921 | "Shelby", 922 | "Sienna", 923 | "Sierra", 924 | "Simone", 925 | "Skye", 926 | "Skylar", 927 | "Sofia", 928 | "Sophia", 929 | "Sophie", 930 | "Stella", 931 | "Stephanie", 932 | "Summer", 933 | "Sydney", 934 | "Sylvia", 935 | "Talia", 936 | "Tatiana", 937 | "Teresa", 938 | "Tessa", 939 | "Theresa", 940 | "Tiffany", 941 | "Valentina", 942 | "Valeria", 943 | "Valerie", 944 | "Vanessa", 945 | "Vera", 946 | "Veronica", 947 | "Victoria", 948 | "Violet", 949 | "Vivian", 950 | "Wendy", 951 | "Whitney", 952 | "Willow", 953 | "Winter", 954 | "Yara", 955 | "Zara", 956 | "Zoe", 957 | "Zoey" 958 | ] 959 | } 960 | } -------------------------------------------------------------------------------- /env/data/map_receptacle_data.json: -------------------------------------------------------------------------------- 1 | { 2 | "0": {}, 3 | "1": { 4 | "room10": { 5 | "receptacles": [ 6 | { 7 | "name": "table", 8 | "x": [ 9 | 152, 10 | 154 11 | ], 12 | "y": [ 13 | 213, 14 | 215 15 | ], 16 | "position": [ 17 | 153, 18 | 214 19 | ], 20 | "floor": 1 21 | }, 22 | { 23 | "name": "table", 24 | "x": [ 25 | 160, 26 | 162 27 | ], 28 | "y": [ 29 | 212, 30 | 214 31 | ], 32 | "position": [ 33 | 161, 34 | 213 35 | ], 36 | "floor": 1 37 | }, 38 | { 39 | "name": "reception counter", 40 | "x": [ 41 | 136, 42 | 139 43 | ], 44 | "y": [ 45 | 208, 46 | 225 47 | ], 48 | "position": [ 49 | 138, 50 | 216 51 | ], 52 | "floor": 1 53 | }, 54 | { 55 | "name": "table", 56 | "x": [ 57 | 160, 58 | 162 59 | ], 60 | "y": [ 61 | 220, 62 | 222 63 | ], 64 | "position": [ 65 | 161, 66 | 221 67 | ], 68 | "floor": 1 69 | }, 70 | { 71 | "name": "table", 72 | "x": [ 73 | 156, 74 | 158 75 | ], 76 | "y": [ 77 | 215, 78 | 219 79 | ], 80 | "position": [ 81 | 157, 82 | 217 83 | ], 84 | "floor": 1 85 | } 86 | ], 87 | "receptacle_names": [ 88 | "table", 89 | "table", 90 | "reception counter", 91 | "table", 92 | "table" 93 | ] 94 | }, 95 | "room0": { 96 | "receptacles": [ 97 | { 98 | "name": "table", 99 | "x": [ 100 | 153, 101 | 159 102 | ], 103 | "y": [ 104 | 179, 105 | 182 106 | ], 107 | "position": [ 108 | 156, 109 | 180 110 | ], 111 | "floor": 1 112 | }, 113 | { 114 | "name": "table", 115 | "x": [ 116 | 153, 117 | 159 118 | ], 119 | "y": [ 120 | 171, 121 | 174 122 | ], 123 | "position": [ 124 | 156, 125 | 172 126 | ], 127 | "floor": 1 128 | } 129 | ], 130 | "receptacle_names": [ 131 | "table", 132 | "table" 133 | ] 134 | }, 135 | "room1": { 136 | "receptacles": [ 137 | { 138 | "name": "polygon table", 139 | "x": [ 140 | 107, 141 | 111 142 | ], 143 | "y": [ 144 | 176, 145 | 183 146 | ], 147 | "position": [ 148 | 109, 149 | 180 150 | ], 151 | "floor": 1 152 | } 153 | ], 154 | "receptacle_names": [ 155 | "polygon table" 156 | ] 157 | }, 158 | "room5": { 159 | "receptacles": [ 160 | { 161 | "name": "desk", 162 | "x": [ 163 | 18, 164 | 26 165 | ], 166 | "y": [ 167 | 171, 168 | 179 169 | ], 170 | "position": [ 171 | 22, 172 | 175 173 | ], 174 | "floor": 1 175 | } 176 | ], 177 | "receptacle_names": [ 178 | "desk" 179 | ] 180 | } 181 | }, 182 | "2": { 183 | "room19": { 184 | "receptacles": [ 185 | { 186 | "name": "desk", 187 | "x": [ 188 | 71, 189 | 74 190 | ], 191 | "y": [ 192 | 207, 193 | 214 194 | ], 195 | "position": [ 196 | 72, 197 | 210 198 | ], 199 | "floor": 2 200 | }, 201 | { 202 | "name": "desk", 203 | "x": [ 204 | 88, 205 | 91 206 | ], 207 | "y": [ 208 | 220, 209 | 227 210 | ], 211 | "position": [ 212 | 90, 213 | 224 214 | ], 215 | "floor": 2 216 | }, 217 | { 218 | "name": "desk", 219 | "x": [ 220 | 71, 221 | 74 222 | ], 223 | "y": [ 224 | 220, 225 | 227 226 | ], 227 | "position": [ 228 | 72, 229 | 224 230 | ], 231 | "floor": 2 232 | }, 233 | { 234 | "name": "desk", 235 | "x": [ 236 | 91, 237 | 95 238 | ], 239 | "y": [ 240 | 207, 241 | 214 242 | ], 243 | "position": [ 244 | 93, 245 | 210 246 | ], 247 | "floor": 2 248 | }, 249 | { 250 | "name": "desk", 251 | "x": [ 252 | 74, 253 | 78 254 | ], 255 | "y": [ 256 | 207, 257 | 214 258 | ], 259 | "position": [ 260 | 76, 261 | 210 262 | ], 263 | "floor": 2 264 | }, 265 | { 266 | "name": "desk", 267 | "x": [ 268 | 74, 269 | 78 270 | ], 271 | "y": [ 272 | 220, 273 | 227 274 | ], 275 | "position": [ 276 | 76, 277 | 224 278 | ], 279 | "floor": 2 280 | }, 281 | { 282 | "name": "desk", 283 | "x": [ 284 | 91, 285 | 95 286 | ], 287 | "y": [ 288 | 220, 289 | 227 290 | ], 291 | "position": [ 292 | 93, 293 | 224 294 | ], 295 | "floor": 2 296 | }, 297 | { 298 | "name": "desk", 299 | "x": [ 300 | 88, 301 | 91 302 | ], 303 | "y": [ 304 | 207, 305 | 214 306 | ], 307 | "position": [ 308 | 90, 309 | 210 310 | ], 311 | "floor": 2 312 | }, 313 | { 314 | "name": "side table", 315 | "x": [ 316 | 81, 317 | 84 318 | ], 319 | "y": [ 320 | 212, 321 | 223 322 | ], 323 | "position": [ 324 | 82, 325 | 218 326 | ], 327 | "floor": 2 328 | } 329 | ], 330 | "receptacle_names": [ 331 | "desk", 332 | "desk", 333 | "desk", 334 | "desk", 335 | "desk", 336 | "desk", 337 | "desk", 338 | "desk", 339 | "side table" 340 | ] 341 | }, 342 | "room17": { 343 | "receptacles": [ 344 | { 345 | "name": "desk", 346 | "x": [ 347 | 40, 348 | 50 349 | ], 350 | "y": [ 351 | 172, 352 | 180 353 | ], 354 | "position": [ 355 | 45, 356 | 176 357 | ], 358 | "floor": 2 359 | } 360 | ], 361 | "receptacle_names": [ 362 | "desk" 363 | ] 364 | }, 365 | "room0": { 366 | "receptacles": [ 367 | { 368 | "name": "table", 369 | "x": [ 370 | 103, 371 | 111 372 | ], 373 | "y": [ 374 | 92, 375 | 93 376 | ], 377 | "position": [ 378 | 107, 379 | 92 380 | ], 381 | "floor": 2 382 | }, 383 | { 384 | "name": "bed", 385 | "x": [ 386 | 112, 387 | 119 388 | ], 389 | "y": [ 390 | 100, 391 | 107 392 | ], 393 | "position": [ 394 | 116, 395 | 104 396 | ], 397 | "floor": 2 398 | } 399 | ], 400 | "receptacle_names": [ 401 | "table", 402 | "bed" 403 | ] 404 | }, 405 | "room1": { 406 | "receptacles": [ 407 | { 408 | "name": "table", 409 | "x": [ 410 | 153, 411 | 161 412 | ], 413 | "y": [ 414 | 112, 415 | 114 416 | ], 417 | "position": [ 418 | 157, 419 | 113 420 | ], 421 | "floor": 2 422 | }, 423 | { 424 | "name": "bed", 425 | "x": [ 426 | 145, 427 | 152 428 | ], 429 | "y": [ 430 | 98, 431 | 106 432 | ], 433 | "position": [ 434 | 148, 435 | 102 436 | ], 437 | "floor": 2 438 | } 439 | ], 440 | "receptacle_names": [ 441 | "table", 442 | "bed" 443 | ] 444 | }, 445 | "room2": { 446 | "receptacles": [ 447 | { 448 | "name": "table", 449 | "x": [ 450 | 103, 451 | 111 452 | ], 453 | "y": [ 454 | 66, 455 | 68 456 | ], 457 | "position": [ 458 | 107, 459 | 67 460 | ], 461 | "floor": 2 462 | }, 463 | { 464 | "name": "bed", 465 | "x": [ 466 | 112, 467 | 119 468 | ], 469 | "y": [ 470 | 74, 471 | 82 472 | ], 473 | "position": [ 474 | 116, 475 | 78 476 | ], 477 | "floor": 2 478 | } 479 | ], 480 | "receptacle_names": [ 481 | "table", 482 | "bed" 483 | ] 484 | }, 485 | "room3": { 486 | "receptacles": [ 487 | { 488 | "name": "table", 489 | "x": [ 490 | 153, 491 | 161 492 | ], 493 | "y": [ 494 | 163, 495 | 165 496 | ], 497 | "position": [ 498 | 157, 499 | 164 500 | ], 501 | "floor": 2 502 | }, 503 | { 504 | "name": "bed", 505 | "x": [ 506 | 145, 507 | 152 508 | ], 509 | "y": [ 510 | 149, 511 | 157 512 | ], 513 | "position": [ 514 | 148, 515 | 153 516 | ], 517 | "floor": 2 518 | } 519 | ], 520 | "receptacle_names": [ 521 | "table", 522 | "bed" 523 | ] 524 | }, 525 | "room4": { 526 | "receptacles": [ 527 | { 528 | "name": "table", 529 | "x": [ 530 | 103, 531 | 111 532 | ], 533 | "y": [ 534 | 117, 535 | 119 536 | ], 537 | "position": [ 538 | 107, 539 | 118 540 | ], 541 | "floor": 2 542 | }, 543 | { 544 | "name": "bed", 545 | "x": [ 546 | 112, 547 | 119 548 | ], 549 | "y": [ 550 | 126, 551 | 133 552 | ], 553 | "position": [ 554 | 116, 555 | 130 556 | ], 557 | "floor": 2 558 | } 559 | ], 560 | "receptacle_names": [ 561 | "table", 562 | "bed" 563 | ] 564 | }, 565 | "room16": { 566 | "receptacles": [ 567 | { 568 | "name": "table", 569 | "x": [ 570 | 160, 571 | 162 572 | ], 573 | "y": [ 574 | 212, 575 | 214 576 | ], 577 | "position": [ 578 | 161, 579 | 213 580 | ], 581 | "floor": 2 582 | }, 583 | { 584 | "name": "table", 585 | "x": [ 586 | 152, 587 | 154 588 | ], 589 | "y": [ 590 | 213, 591 | 215 592 | ], 593 | "position": [ 594 | 153, 595 | 214 596 | ], 597 | "floor": 2 598 | }, 599 | { 600 | "name": "reception counter", 601 | "x": [ 602 | 136, 603 | 139 604 | ], 605 | "y": [ 606 | 208, 607 | 225 608 | ], 609 | "position": [ 610 | 138, 611 | 216 612 | ], 613 | "floor": 2 614 | }, 615 | { 616 | "name": "table", 617 | "x": [ 618 | 156, 619 | 158 620 | ], 621 | "y": [ 622 | 215, 623 | 219 624 | ], 625 | "position": [ 626 | 157, 627 | 217 628 | ], 629 | "floor": 2 630 | }, 631 | { 632 | "name": "table", 633 | "x": [ 634 | 160, 635 | 162 636 | ], 637 | "y": [ 638 | 220, 639 | 222 640 | ], 641 | "position": [ 642 | 161, 643 | 221 644 | ], 645 | "floor": 2 646 | } 647 | ], 648 | "receptacle_names": [ 649 | "table", 650 | "table", 651 | "reception counter", 652 | "table", 653 | "table" 654 | ] 655 | }, 656 | "room5": { 657 | "receptacles": [ 658 | { 659 | "name": "table", 660 | "x": [ 661 | 153, 662 | 161 663 | ], 664 | "y": [ 665 | 61, 666 | 63 667 | ], 668 | "position": [ 669 | 157, 670 | 62 671 | ], 672 | "floor": 2 673 | }, 674 | { 675 | "name": "bed", 676 | "x": [ 677 | 145, 678 | 152 679 | ], 680 | "y": [ 681 | 47, 682 | 54 683 | ], 684 | "position": [ 685 | 148, 686 | 50 687 | ], 688 | "floor": 2 689 | } 690 | ], 691 | "receptacle_names": [ 692 | "table", 693 | "bed" 694 | ] 695 | }, 696 | "room15": { 697 | "receptacles": [ 698 | { 699 | "name": "desk", 700 | "x": [ 701 | 50, 702 | 59 703 | ], 704 | "y": [ 705 | 171, 706 | 179 707 | ], 708 | "position": [ 709 | 54, 710 | 175 711 | ], 712 | "floor": 2 713 | } 714 | ], 715 | "receptacle_names": [ 716 | "desk" 717 | ] 718 | }, 719 | "room6": { 720 | "receptacles": [ 721 | { 722 | "name": "table", 723 | "x": [ 724 | 103, 725 | 111 726 | ], 727 | "y": [ 728 | 15, 729 | 17 730 | ], 731 | "position": [ 732 | 107, 733 | 16 734 | ], 735 | "floor": 2 736 | }, 737 | { 738 | "name": "bed", 739 | "x": [ 740 | 112, 741 | 119 742 | ], 743 | "y": [ 744 | 23, 745 | 30 746 | ], 747 | "position": [ 748 | 116, 749 | 26 750 | ], 751 | "floor": 2 752 | } 753 | ], 754 | "receptacle_names": [ 755 | "table", 756 | "bed" 757 | ] 758 | }, 759 | "room7": { 760 | "receptacles": [ 761 | { 762 | "name": "table", 763 | "x": [ 764 | 153, 765 | 161 766 | ], 767 | "y": [ 768 | 86, 769 | 88 770 | ], 771 | "position": [ 772 | 157, 773 | 87 774 | ], 775 | "floor": 2 776 | }, 777 | { 778 | "name": "bed", 779 | "x": [ 780 | 145, 781 | 152 782 | ], 783 | "y": [ 784 | 73, 785 | 80 786 | ], 787 | "position": [ 788 | 148, 789 | 76 790 | ], 791 | "floor": 2 792 | } 793 | ], 794 | "receptacle_names": [ 795 | "table", 796 | "bed" 797 | ] 798 | }, 799 | "room8": { 800 | "receptacles": [ 801 | { 802 | "name": "table", 803 | "x": [ 804 | 153, 805 | 161 806 | ], 807 | "y": [ 808 | 138, 809 | 140 810 | ], 811 | "position": [ 812 | 157, 813 | 139 814 | ], 815 | "floor": 2 816 | }, 817 | { 818 | "name": "bed", 819 | "x": [ 820 | 145, 821 | 152 822 | ], 823 | "y": [ 824 | 124, 825 | 131 826 | ], 827 | "position": [ 828 | 148, 829 | 128 830 | ], 831 | "floor": 2 832 | } 833 | ], 834 | "receptacle_names": [ 835 | "table", 836 | "bed" 837 | ] 838 | }, 839 | "room9": { 840 | "receptacles": [ 841 | { 842 | "name": "table", 843 | "x": [ 844 | 103, 845 | 111 846 | ], 847 | "y": [ 848 | 40, 849 | 42 850 | ], 851 | "position": [ 852 | 107, 853 | 41 854 | ], 855 | "floor": 2 856 | }, 857 | { 858 | "name": "bed", 859 | "x": [ 860 | 112, 861 | 119 862 | ], 863 | "y": [ 864 | 49, 865 | 56 866 | ], 867 | "position": [ 868 | 116, 869 | 52 870 | ], 871 | "floor": 2 872 | } 873 | ], 874 | "receptacle_names": [ 875 | "table", 876 | "bed" 877 | ] 878 | }, 879 | "room22": { 880 | "receptacles": [ 881 | { 882 | "name": "coffee table", 883 | "x": [ 884 | 42, 885 | 45 886 | ], 887 | "y": [ 888 | 216, 889 | 221 890 | ], 891 | "position": [ 892 | 44, 893 | 218 894 | ], 895 | "floor": 2 896 | }, 897 | { 898 | "name": "dinner counter", 899 | "x": [ 900 | 56, 901 | 59 902 | ], 903 | "y": [ 904 | 212, 905 | 223 906 | ], 907 | "position": [ 908 | 58, 909 | 218 910 | ], 911 | "floor": 2 912 | } 913 | ], 914 | "receptacle_names": [ 915 | "coffee table", 916 | "dinner counter" 917 | ] 918 | }, 919 | "room10": { 920 | "receptacles": [ 921 | { 922 | "name": "table", 923 | "x": [ 924 | 153, 925 | 161 926 | ], 927 | "y": [ 928 | 35, 929 | 37 930 | ], 931 | "position": [ 932 | 157, 933 | 36 934 | ], 935 | "floor": 2 936 | }, 937 | { 938 | "name": "bed", 939 | "x": [ 940 | 145, 941 | 152 942 | ], 943 | "y": [ 944 | 21, 945 | 29 946 | ], 947 | "position": [ 948 | 148, 949 | 25 950 | ], 951 | "floor": 2 952 | } 953 | ], 954 | "receptacle_names": [ 955 | "table", 956 | "bed" 957 | ] 958 | }, 959 | "room23": { 960 | "receptacles": [ 961 | { 962 | "name": "meeting table large", 963 | "x": [ 964 | 74, 965 | 92 966 | ], 967 | "y": [ 968 | 176, 969 | 181 970 | ], 971 | "position": [ 972 | 83, 973 | 178 974 | ], 975 | "floor": 2 976 | } 977 | ], 978 | "receptacle_names": [ 979 | "meeting table large" 980 | ] 981 | }, 982 | "room11": { 983 | "receptacles": [ 984 | { 985 | "name": "small table", 986 | "x": [ 987 | 14, 988 | 16 989 | ], 990 | "y": [ 991 | 169, 992 | 171 993 | ], 994 | "position": [ 995 | 15, 996 | 170 997 | ], 998 | "floor": 2 999 | }, 1000 | { 1001 | "name": "table", 1002 | "x": [ 1003 | 9, 1004 | 17 1005 | ], 1006 | "y": [ 1007 | 180, 1008 | 183 1009 | ], 1010 | "position": [ 1011 | 13, 1012 | 182 1013 | ], 1014 | "floor": 2 1015 | }, 1016 | { 1017 | "name": "kitchen counter", 1018 | "x": [ 1019 | 30, 1020 | 33 1021 | ], 1022 | "y": [ 1023 | 177, 1024 | 187 1025 | ], 1026 | "position": [ 1027 | 32, 1028 | 182 1029 | ], 1030 | "floor": 2 1031 | }, 1032 | { 1033 | "name": "kitchen wardrobe", 1034 | "x": [ 1035 | 30, 1036 | 32 1037 | ], 1038 | "y": [ 1039 | 175, 1040 | 177 1041 | ], 1042 | "position": [ 1043 | 31, 1044 | 176 1045 | ], 1046 | "floor": 2 1047 | }, 1048 | { 1049 | "name": "dinner counter", 1050 | "x": [ 1051 | 21, 1052 | 24 1053 | ], 1054 | "y": [ 1055 | 172, 1056 | 183 1057 | ], 1058 | "position": [ 1059 | 22, 1060 | 178 1061 | ], 1062 | "floor": 2 1063 | } 1064 | ], 1065 | "receptacle_names": [ 1066 | "small table", 1067 | "table", 1068 | "kitchen counter", 1069 | "kitchen wardrobe", 1070 | "dinner counter" 1071 | ] 1072 | }, 1073 | "room21": { 1074 | "receptacles": [ 1075 | { 1076 | "name": "table", 1077 | "x": [ 1078 | 153, 1079 | 159 1080 | ], 1081 | "y": [ 1082 | 171, 1083 | 174 1084 | ], 1085 | "position": [ 1086 | 156, 1087 | 172 1088 | ], 1089 | "floor": 2 1090 | }, 1091 | { 1092 | "name": "table", 1093 | "x": [ 1094 | 153, 1095 | 159 1096 | ], 1097 | "y": [ 1098 | 179, 1099 | 182 1100 | ], 1101 | "position": [ 1102 | 156, 1103 | 180 1104 | ], 1105 | "floor": 2 1106 | } 1107 | ], 1108 | "receptacle_names": [ 1109 | "table", 1110 | "table" 1111 | ] 1112 | }, 1113 | "room13": { 1114 | "receptacles": [ 1115 | { 1116 | "name": "polygon table", 1117 | "x": [ 1118 | 107, 1119 | 111 1120 | ], 1121 | "y": [ 1122 | 176, 1123 | 183 1124 | ], 1125 | "position": [ 1126 | 109, 1127 | 180 1128 | ], 1129 | "floor": 2 1130 | } 1131 | ], 1132 | "receptacle_names": [ 1133 | "polygon table" 1134 | ] 1135 | }, 1136 | "room14": { 1137 | "receptacles": [ 1138 | { 1139 | "name": "furnituregymdesk", 1140 | "x": [ 1141 | 100, 1142 | 103 1143 | ], 1144 | "y": [ 1145 | 207, 1146 | 210 1147 | ], 1148 | "position": [ 1149 | 102, 1150 | 208 1151 | ], 1152 | "floor": 2 1153 | } 1154 | ], 1155 | "receptacle_names": [ 1156 | "furnituregymdesk" 1157 | ] 1158 | }, 1159 | "room12": { 1160 | "receptacles": [ 1161 | { 1162 | "name": "table", 1163 | "x": [ 1164 | 103, 1165 | 111 1166 | ], 1167 | "y": [ 1168 | 143, 1169 | 145 1170 | ], 1171 | "position": [ 1172 | 107, 1173 | 144 1174 | ], 1175 | "floor": 2 1176 | }, 1177 | { 1178 | "name": "bed", 1179 | "x": [ 1180 | 112, 1181 | 119 1182 | ], 1183 | "y": [ 1184 | 151, 1185 | 159 1186 | ], 1187 | "position": [ 1188 | 116, 1189 | 155 1190 | ], 1191 | "floor": 2 1192 | } 1193 | ], 1194 | "receptacle_names": [ 1195 | "table", 1196 | "bed" 1197 | ] 1198 | } 1199 | } 1200 | } -------------------------------------------------------------------------------- /env/socket_server.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import sys 3 | import time 4 | import subprocess 5 | # import atexit 6 | from socket import * 7 | import threading 8 | import json 9 | import struct 10 | from enum import Enum 11 | # from game_demo import * 12 | import ast 13 | from env.map_process import RoomMap 14 | from env.npc_control import Npc, Agent 15 | import datetime 16 | import numpy as np 17 | 18 | 19 | class EnvTime(object): 20 | def __init__(self, speed=120, year=2025, month=3, day=12, hour=6, minute=50, second=0, end=2050): 21 | # Define start date. At a rate of speed(120) times 22 | self.start_date = datetime.datetime(year, month, day, hour, minute, second) 23 | # Define time multiplier 24 | self.time_multiplier = speed 25 | self.running = 1 26 | self.end = end 27 | # Simulation time 28 | self.current_date = self.start_date 29 | self.start_time = self.start_date 30 | # self.current_date.isoweekday() 31 | self.week = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday'] 32 | 33 | def set_time(self, year=2025, month=3, day=12, hour=6, minute=50, second=0, end=2050): 34 | self.current_date = datetime.datetime(year, month, day, hour, minute, second) 35 | print(self.current_date) 36 | 37 | def time_simulation(self, stop_event): 38 | while True: 39 | # print(stop_event.is_set()) 40 | if not self.running or stop_event.is_set(): 41 | break 42 | # print("Current Date:", self.current_date) 43 | # Accelerate at 120 times the speed 44 | time_delta = datetime.timedelta(seconds=1) # Add one more day 45 | self.current_date += time_delta * self.time_multiplier 46 | # Control simulation speed 47 | time.sleep(1) # Update every second 48 | # Termination conditions can be added, such as stopping simulation when a specific date is reached 49 | if self.current_date.year > self.end: 50 | break 51 | 52 | def time_difference(self): 53 | time_diff = self.current_date - self.start_time 54 | hours = time_diff.total_seconds() // 3600 55 | # print("The time difference is% d hours" % hours) 56 | return time_diff.days 57 | 58 | def weekday_now(self): 59 | return self.week[self.current_date.weekday()] 60 | 61 | def simulation_start(self): 62 | self.start_time = self.current_date 63 | 64 | 65 | # message define 66 | class MsgCmd(Enum): 67 | # 0 disconnects, 1 server sends behavior instructions, 2 servers send status requests, 3 clients reply with behavior callbacks, 68 | # 4 clients reply with target status, 5 instructions to robots, 6 requests/feedback about robot clients 69 | EXIT = 0 70 | Instruction = 1 71 | Request = 2 72 | Result = 3 73 | State = 4 74 | Control = 5 75 | Information = 6 76 | Server_Update_Config = 7 77 | Server_Device_Create_Request = 8 78 | Server_Device_Inform_Request = 9 79 | Server_Device_Status_Request = 10 80 | Client_Device_Status_Response = 11 81 | Server_Config_Inform = 12 82 | Client_Config_Response = 13 83 | 84 | 85 | class Server(object): 86 | def __init__(self, stop_event): 87 | self.stop_event = stop_event 88 | self.state = 1 89 | self.clients = [] 90 | self.messages = [] 91 | self.information = '' 92 | # 1.Create a socket 93 | self.sock = socket(AF_INET, SOCK_STREAM) 94 | # 2. Prepare to connect to the server and establish a connection 95 | serve_ip = 'localhost' 96 | serve_port = 8000 # search for available port starting from 8000 97 | # tcp_socket.connect((serve_ip,serve_port)) 98 | # Connect to the server, establish a connection, with parameters in tuple form 99 | tcp_address = ('localhost', serve_port) 100 | # Provide a mechanism for checking ports 101 | sock_result = 0 102 | while not sock_result: 103 | try: 104 | self.sock.bind(tcp_address) 105 | sock_result = 1 106 | except: 107 | serve_port += 1 108 | tcp_address = ('localhost', serve_port) 109 | with open('unity/PRS_Data/StreamingAssets/config.json', 'r') as file: 110 | env_data = json.load(file) 111 | env_data["serverConnectionPort"] = serve_port 112 | with open('unity/PRS_Data/StreamingAssets/config.json', 'w') as file: 113 | json.dump(env_data, file) 114 | print('server started: ', str(tcp_address)) 115 | MAX_CONNECTION = 100 116 | # Start listening for connections 117 | self.sock.listen(MAX_CONNECTION) 118 | self.headerSize = 12 119 | self.count = 0 120 | # self.robot = PRS_IK() 121 | # robot ik algorithm 122 | self.maps = RoomMap() 123 | self.notes = {} 124 | self.byte_stream = bytes() 125 | self.header_length = 0 126 | self.sock.settimeout(10) 127 | 128 | def wait_for_connection(self): 129 | while True: 130 | try: 131 | now_client, addr = self.sock.accept() 132 | print('Connected by', now_client) 133 | self.state = 2 134 | now_client.settimeout(300) 135 | self.clients.append([addr, now_client]) 136 | except: pass 137 | for index_client, n_client in enumerate(self.clients): 138 | # result = self.sock.connect_ex(n_client) 139 | try: 140 | result = n_client[1].getsockname() 141 | r = n_client[1].getpeername() 142 | # print('===========perfect connection============') 143 | except Exception as e: 144 | print(e, n_client[0], 'Connected Closed Now') 145 | try: 146 | self.clients.remove(n_client) 147 | if len(self.clients) == 0 and self.state == 2: 148 | self.state = 0 149 | self.stop_event.set() 150 | except: 151 | pass 152 | if not self.state or self.stop_event.is_set(): 153 | print(self.state, 'No waiting for connection') 154 | self.sock.close() 155 | break 156 | if len(self.clients): 157 | time.sleep(0.5) 158 | else: 159 | time.sleep(0.01) 160 | 161 | def check_connection(self): 162 | pass 163 | # for index_client, n_client in enumerate(self.clients): 164 | # # result = self.sock.connect_ex(n_client) 165 | # rrr = n_client[1].recv(1024) 166 | # result = n_client[1].getsockname() 167 | # r = n_client[1].getpeername() 168 | 169 | def handle_data(self, n_client): 170 | # receive message from client -> information process 171 | data = n_client.recv(10240000) 172 | if not data: 173 | return 0 174 | else: 175 | self.messages.append(data) 176 | # print('---------------------------------'， 'Received: msg') 177 | # ------------------parsing info from unity--------------------- 178 | # self.send_back({'result': 1}) 179 | return 1 180 | 181 | def message_process(self): 182 | while True: 183 | if not self.state or self.stop_event.is_set(): 184 | self.state = 0 185 | print(self.state, 'Processing Completed') 186 | break 187 | if len(self.messages) > 0: 188 | for msg_i, msg in enumerate(self.messages): 189 | try: 190 | self.unpack(msg) 191 | except Exception as e: 192 | print('.........parsing error............', e, type(msg)) 193 | self.state = 0 194 | finally: 195 | del self.messages[msg_i] 196 | else: 197 | time.sleep(0.005) 198 | 199 | def receive_data(self): 200 | while True: 201 | # self.check_connection() 202 | for n_client in self.clients: 203 | try: 204 | # Processing received message 205 | res = self.handle_data(n_client[1]) 206 | except Exception as e: 207 | print(e, n_client[0], 'Connected closed') 208 | try: 209 | self.clients.remove(n_client) 210 | if len(self.clients) == 0 and self.state == 2: 211 | self.state = 0 212 | self.stop_event.set() 213 | except: 214 | pass 215 | time.sleep(0.005) 216 | if not self.state or self.stop_event.is_set(): 217 | print(self.state, 'Connection closed') 218 | self.sock.close() 219 | break 220 | 221 | def send_data(self, cmd=1, data={"requestIndex":10,"npcId":0,"actionId":0,"actionPara":""}, recv=0): 222 | send_finish = 0 223 | while not send_finish: 224 | if len(self.clients)==0: break 225 | for n_client in self.clients: 226 | self.check_connection() 227 | try: 228 | if cmd < 15: 229 | data['requestIndex'] = self.count 230 | self.count = self.count + 1 231 | elif cmd == 0: 232 | self.state = 0 233 | msg, msg_data = self.pack(cmd, data, recv) 234 | n_client[1].send(msg) 235 | send_finish = 1 236 | return data['requestIndex'] 237 | break 238 | except Exception as e: 239 | # print(e, n_client[0]) 240 | try: 241 | self.clients.remove(n_client) 242 | if len(self.clients) == 0: 243 | self.state = 0 244 | except: pass 245 | return False 246 | 247 | def send_back(self, response={'result': 0}): 248 | f = 0 249 | while not f: 250 | for n_client in self.clients: 251 | self.check_connection() 252 | try: 253 | info = json.dumps(response) 254 | n_client[1].send(info.encode("utf8")) 255 | print('Sent: ', info.encode("utf8")) 256 | f = 1 257 | return 1 258 | except Exception as e: 259 | print(e, n_client[0]) 260 | try: 261 | self.clients.remove(n_client) 262 | except: pass 263 | 264 | def pack(self, cmd, _body, _recv=0): 265 | body = json.dumps(_body) 266 | # Convert the message body to Json format and convert it to byte encoding 267 | header = [body.__len__(), cmd, _recv] 268 | # Form a list of message headers in order 269 | headPack= struct.pack("3I", *header) 270 | # Use struct to package message headers and obtain byte encoding 271 | sendData = headPack+body.encode("utf8") 272 | # Combine message header bytes and message body bytes together 273 | return sendData, body 274 | 275 | def handle_msg(self, headPack ,body): 276 | """Classify and process received message strings""" 277 | # data processing 278 | cmd= 'ad' 279 | try: 280 | cmd = MsgCmd(headPack[1]).name # Get the value of Code\ 281 | except Exception as e: 282 | print(headPack[1]) 283 | # print('python get================cmd is', cmd) 284 | is_recv = headPack[2] 285 | # print("Received 1 packet->bodySize:{}, cmd:{}, recv:{}".format(headPack[0], cmd, is_recv)) 286 | body = body.replace("false", "False") 287 | body = body.replace("true", "True") 288 | body = body.replace("none", "None") 289 | p = json.loads(body) # Decode and deserialize strings into JSON objects 290 | dict_data = ast.literal_eval(p) 291 | # self.information += str(cmd) + str(body) 292 | # Check the message type 293 | dict_d = copy.deepcopy(dict_data) 294 | del dict_d['requestIndex'] 295 | self.notes[dict_data['requestIndex']] = dict_d 296 | if cmd == "EXIT": 297 | self.state = 0 298 | print('0. Env is over, exit!') 299 | return 300 | elif cmd == "Result": pass 301 | # print('3、Execution results from Unity', dict_data) 302 | elif cmd == "State": pass 303 | # Storing parameter information 304 | # print('4、Detailed information obtained id: {}'.format(dict_data['requestIndex'])) 305 | elif cmd == "Control": pass 306 | # IK is here 307 | elif cmd == "Information": pass 308 | # print("6、This is robot information", dict_data['requestIndex'], ', length- ', len(dict_data),) 309 | else: pass 310 | # print("\nUnknown cmd: {0}".format(cmd)) 311 | # Continue receiving messages 312 | #self._recv_bytes() 313 | 314 | def unpack(self, data): 315 | headPack = struct.unpack('3I', bytearray(data[:self.headerSize])) 316 | bodySize = headPack[0] 317 | body = data[self.headerSize:self.headerSize + bodySize] 318 | try: 319 | self.handle_msg(headPack, body.decode("utf8")) 320 | except ValueError: 321 | if not self.header_length or len(self.byte_stream) == 0: 322 | self.header_length = headPack 323 | self.byte_stream += body 324 | else: 325 | self.byte_stream += data 326 | if len(self.byte_stream) >= self.header_length[0]: 327 | # data_byte = self.byte_stream.decode("utf8") 328 | self.handle_msg(self.header_length, self.byte_stream.decode()) 329 | self.byte_stream = bytes() 330 | self.header_length = 0 331 | return 1 332 | 333 | def unpack_pro(self, data, msgHandler): 334 | dataBuffer = bytes() 335 | if data: 336 | self.dataBuffer += data 337 | while True: 338 | # Jump out of the function to continue receiving data when there is insufficient data in the message header 339 | if len(self.dataBuffer) < self.headerSize: 340 | # print("Packet (% s Byte) is smaller than the length of the message header, causing a small loop to break out" % len(self.dataBuffer)) 341 | break 342 | # struct: represent Network order，3I represents 3个unsigned int 343 | # msg_length = struct.unpack("I", bytearray(msg[:4]))[0] 344 | # Obtain information length 345 | headPack = struct.unpack('3I', bytearray(self.dataBuffer[:self.headerSize])) 346 | # Decode the message header 347 | # Get message body length 348 | bodySize = headPack[0] 349 | # Handling subcontracting situations, jumping out of the function to continue receiving data 350 | if len(self.dataBuffer) < self.headerSize + bodySize: 351 | # print("Packet (% s Byte) incomplete (total of% s Bytes), skipping small loop“ % (len(self.dataBuffer), self.headerSize + bodySize)) 352 | break 353 | # Read the content of the message body 354 | body = self.dataBuffer[self.headerSize:self.headerSize + bodySize] 355 | self.handle_msg(headPack, body.decode("utf8")) 356 | # Handling of packet sticking and obtaining the next part of the data packet 357 | self.dataBuffer = self.dataBuffer[self.headerSize + bodySize:] 358 | if len(self.dataBuffer) != 0: 359 | return True # Continue receiving messages 360 | else: 361 | return False # No longer receiving messages 362 | else: 363 | return False # No longer receiving messages 364 | 365 | def wait_for_respond(self, id, times=60): 366 | info = None 367 | for ii in range(int(times)): 368 | time.sleep(0.1) 369 | try: 370 | info = self.notes[id] 371 | break 372 | except Exception as e: 373 | pass 374 | return info 375 | 376 | def object_query(self, obj_id=0): 377 | for i in range(5): 378 | instruction = {"requestIndex": 0, "targetType": 1, "targetId": obj_id} 379 | r_id = self.send_data(2, instruction, 1) 380 | object_info = self.wait_for_respond(r_id, 60) 381 | if object_info is not None: 382 | break 383 | time.sleep(0.1) 384 | if object_info: 385 | object_info = eval(object_info['statusDetail']) 386 | return object_info 387 | 388 | def object_nearby_detect(self, obj_id=0): 389 | instruction = {"requestIndex": 1, "targetType": 20, "targetId": obj_id} 390 | r_id = self.send_data(2, instruction, 1) 391 | object_info = self.wait_for_respond(r_id, 60) 392 | if object_info: 393 | object_info = eval(object_info['statusDetail']) 394 | return object_info['touchedIds'] 395 | if object_info: 396 | object_info = eval(object_info['statusDetail']) 397 | try: 398 | return object_info['touchedIds'] 399 | except: 400 | return None 401 | return None 402 | 403 | def object_transform(self, obj_type=0, target_id=4, pos=(0, 0, 0), rotation=0): 404 | # obj_type = 0: npc, obj_type = 1: items in the env 405 | try: 406 | position = {"x": pos[0], "y": pos[1], "z": pos[2]} 407 | except: 408 | position = {"x": pos['x'], "y": pos['y'], "z": pos['z']} 409 | instruction = {"requestIndex": 1, "objectTransformHandles": []} 410 | para = {"objectType": obj_type, "objectId": target_id, "objectPos": position, "objectDir": 411 | {"x": 0, "y": 90, "z": 0}} 412 | instruction['objectTransformHandles'].append(para) 413 | r_id = self.send_data(12, instruction, 1) 414 | object_info = self.wait_for_respond(r_id, 60) 415 | time.sleep(0.2) 416 | return object_info 417 | 418 | def env_finish(self, process, npcs): 419 | if process: 420 | process.terminate() 421 | # Waiting for the process to end (optional, but recommended) 422 | process.wait() 423 | self.send_data(0, {"requestIndex": 10, "actionId": 1}, 0) 424 | # movement demo 425 | self.state = 0 426 | for npc in npcs: 427 | npc.running = 0 428 | self.stop_event.set() 429 | self.sock.close() 430 | # print(self.state, type(self.state)) 431 | print(threading.active_count(), ' ------ env is ready to end') 432 | time.sleep(3) 433 | print(threading.active_count(), ' ------ thank you for using') 434 | 435 | 436 | class ObjectsData(object): 437 | def __init__(self): 438 | with open('unity/PRS_Data/StreamingAssets/itemInfo.json', 'r') as file: 439 | json_data = json.load(file) 440 | with open('env/data/map_room_data.json', 'r') as file: 441 | room_data = json.load(file) 442 | with open('env/data/map_receptacle_data.json', 'r') as file: 443 | receptacles = json.load(file) 444 | with open('env/data/npc_data.json', 'r') as file: 445 | json_npc = json.load(file) 446 | with open('env/data/room_mapping.json', 'r') as file: 447 | room_names = json.load(file) 448 | with open('env/data/room_sampling_points.json', 'r') as file: 449 | self.room_sampling_points = json.load(file) 450 | # decode JSON 451 | 452 | env_objects = [] 453 | for json_i in json_data['statusDetails']: 454 | data = json.loads(json_i) 455 | env_objects.append(data) 456 | env_rooms = [] 457 | for floor_i in list(room_data.keys()): 458 | for identifier, r_data in room_data[floor_i].items(): 459 | room_information = dict() 460 | room_name = room_names[floor_i][identifier] 461 | room_information['name'] = room_name 462 | room_information['floor'] = int(floor_i) 463 | x, y = (r_data['x'][1] + r_data['x'][0]) / 2, (r_data['y'][1] + r_data['y'][0])/2 464 | room_information['position'] = (int(floor_i), round(x), round(y)) 465 | room_information['x'], room_information['y'] = r_data['x'], r_data['y'] 466 | room_information['id'] = identifier 467 | room_information['semantic_name'] = floor_i + '_' + room_name 468 | try: 469 | room_information['receptacles'] = receptacles[floor_i][identifier]['receptacles'] 470 | room_information['receptacles_list'] = receptacles[floor_i][identifier]['receptacle_names'] 471 | except: 472 | room_information['receptacles'] = dict() 473 | room_information['receptacles_list'] = [] 474 | env_rooms.append(room_information) 475 | 476 | self.objects = env_objects 477 | self.room_area = env_rooms 478 | self.room_receptacles = receptacles 479 | self.characters = json_npc['npc'] 480 | self.room_receptacles = None 481 | map0 = np.loadtxt('env/data/semantic_map_0.txt', dtype=int, delimiter='\t') 482 | map1 = np.loadtxt('env/data/semantic_map_1.txt', dtype=int, delimiter='\t') 483 | map2 = np.loadtxt('env/data/semantic_map_2.txt', dtype=int, delimiter='\t') 484 | self.sematic_map = [map0, map1, map2] 485 | with open('env/data/semantic_map_tags.json', 'r') as file: 486 | self.semantic_tags = json.load(file) 487 | 488 | def point_determine(self, pos): 489 | # position not world coordinates, pos: [x, y, z], z is floor 490 | point_P = dict() 491 | try: 492 | point_P['x'], point_P['y'], point_P['z'] = pos['x'], pos['y'], pos['z'] 493 | except: 494 | point_P['x'], point_P['y'], point_P['z'] = pos[0], pos[1], pos[2] 495 | res = None 496 | for room_i in self.room_area: 497 | if round(point_P['z']) != round(room_i['floor']): 498 | continue 499 | if (room_i['x'][0] <= point_P['x'] <= room_i['x'][1]) and ( 500 | room_i['y'][0] <= point_P['y'] <= room_i['y'][1]): 501 | if abs(point_P['z']-room_i['floor']) < 1: 502 | res = room_i['semantic_name'] 503 | return res 504 | 505 | def segment(self): 506 | with open('unity/PRS_Data/StreamingAssets\segmentationTagColorInfo.json', 'r') as file: 507 | seg_tag_data = json.load(file) 508 | seg_data = [] 509 | rgb_id = dict() 510 | for index_tag, item_tag in enumerate(seg_tag_data['TagColors']): 511 | # data_i = json.loads(item_tag) 512 | seg_data.append(item_tag) 513 | r_n, g_n, b_n = float(item_tag['color']['r']), float(item_tag['color']['g']), float(item_tag['color']['b']) 514 | r_n, g_n, b_n = '{:.2f}'.format(r_n), '{:.2f}'.format(g_n), '{:.2f}'.format(b_n) 515 | rgb = (r_n, g_n, b_n) 516 | rgb_id[rgb] = index_tag 517 | if item_tag['tag'] == "UnTagged" or item_tag['tag'].lower() == "untagged": 518 | self.background = rgb 519 | self.segment_tag = seg_data 520 | self.rgb_to_id = rgb_id 521 | 522 | def object_parsing(self, ins, target=['Chair','Stool']): 523 | datas = eval(ins['statusDetail']) 524 | obj_closed = datas['closeRangeItemIds'] 525 | object = None 526 | for i, obj in enumerate(obj_closed): 527 | name = self.objects[obj]['itemName'] 528 | for ttt in target: 529 | if ttt.lower() in name.lower(): 530 | print("The target: ", name, obj, self.objects[obj]) 531 | return obj 532 | # print('There is no {}'.format(target)) 533 | return object 534 | # return None 535 | 536 | def object_query(self, target=['Chair', 'Stool']): 537 | tar = [] 538 | for i, obj in enumerate(self.objects): 539 | obj_i = obj['itemId'] 540 | obj_full_name = obj['itemName'] 541 | obj_now = ''.join([char for char in obj_full_name if not char.isdigit()]) 542 | for name in target: 543 | if name.lower() == obj_now.lower(): 544 | tar.append(obj_i) 545 | return tar 546 | 547 | def get_object_name(self, object_id=1): 548 | name = None 549 | for obj in self.objects: 550 | id = obj['itemId'] 551 | if id == object_id: 552 | name = obj['itemName'] 553 | return name 554 | 555 | def get_info_from_name(self, object_name): 556 | result = None 557 | for obj in self.objects: 558 | na = obj['itemName'] 559 | if na == object_name: 560 | result = obj 561 | return result 562 | 563 | def check_feedback(self, server, id): 564 | time.sleep(0.1) 565 | info = None 566 | for i in range(30): 567 | try: 568 | info = server.notes[id] 569 | break 570 | except Exception as e: 571 | print(len(server.notes)) 572 | time.sleep(0.1) 573 | return info 574 | 575 | 576 | def cleanup_function(stop_event): 577 | stop_event.set() 578 | # stop the loop 579 | 580 | 581 | class DevNull: 582 | def write(self, msg): 583 | pass 584 | 585 | 586 | class PrsEnv(object): 587 | def __init__(self, is_print=1, rendering=1, start_up_mode=0): 588 | # is_print: 0 without print, 1 print information to screen; 589 | # rendering=1 with unity render, 0 is headless mode; start_up_mode: 0 manual, 1 automatic 590 | print("PRS environment beta is starting without interaction") 591 | print('More PRS challenge task and benchmark come soon!') 592 | self.original_stdout = sys.stdout 593 | if not is_print: 594 | dev_null = DevNull() 595 | sys.stdout = dev_null 596 | self.stop_event = threading.Event() 597 | self.server = Server(self.stop_event) 598 | self.npc_running, self.time_running, self.agent_running = 0, 0, 0 599 | connection_thread = threading.Thread(target=self.server.wait_for_connection, args=()) 600 | receive_thread = threading.Thread(target=self.server.receive_data, args=()) 601 | parsing_thread = threading.Thread(target=self.server.message_process, args=()) 602 | connection_thread.start() 603 | receive_thread.start() 604 | parsing_thread.start() 605 | # ---------------server begin------------------- 606 | self.env_time = EnvTime() 607 | # ---------------time system ready------------------- 608 | self.process = 0 609 | # executable_path = 'start.sh' 610 | executable_path = './unity/PRS.x86_64' 611 | if rendering: 612 | command_args = [executable_path] 613 | else: 614 | command_args = [executable_path, '-batchmode'] 615 | try: 616 | if start_up_mode: 617 | # Start the Shell script using subprocess.Popen and capture stdout and stderr 618 | self.process = subprocess.Popen(command_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 619 | print("Starting Unity process...") 620 | # If needed, you can add more processing logic here, such as waiting for the process to finish, etc. 621 | else: 622 | print('Please open the Unity program (unity/start.sh)') 623 | except Exception as e: 624 | # Catch any exceptions that occur during startup and print the error message 625 | print(f"An error occurred during beginning: {e}") 626 | # --- unity exe start --- 627 | while True: 628 | time.sleep(0.3) 629 | state = self.server.state 630 | if state == 2 : break 631 | self.objs_data = ObjectsData() 632 | # --------------agent begin--------------- 633 | self.agent = Agent(self.server, self.env_time, self.objs_data) 634 | # agent_thread = threading.Thread(target=agent_plan, args=(self.server, self.agent)) 635 | self.agent.get_all_map() 636 | # agent_thread.start() 637 | self.objs_data.segment() 638 | # ----------------------- npc coming---------------------- 639 | npc_0 = Npc(0, self.server, self.env_time, self.objs_data) 640 | npc_1 = Npc(1, self.server, self.env_time, self.objs_data) 641 | npc_2 = Npc(2, self.server, self.env_time, self.objs_data) 642 | npc_3 = Npc(3, self.server, self.env_time, self.objs_data) 643 | npc_4 = Npc(4, self.server, self.env_time, self.objs_data) 644 | npc_5 = Npc(5, self.server, self.env_time, self.objs_data) 645 | npc_6 = Npc(6, self.server, self.env_time, self.objs_data) 646 | npc_7 = Npc(7, self.server, self.env_time, self.objs_data) 647 | npc_8 = Npc(8, self.server, self.env_time, self.objs_data) 648 | npc_9 = Npc(9, self.server, self.env_time, self.objs_data) 649 | 650 | print('start') 651 | self.task = {'type': None, 'npc': None, 'object': None, 'target': None, 'state': 0, 'result': None} 652 | self.npcs = [npc_0, npc_1, npc_2, npc_3, npc_4, npc_5, npc_6, npc_7, npc_8, npc_9] 653 | self.agent.npcs = self.npcs 654 | # self.receptacle_mark() 655 | with open('env/data/npc_data.json', 'r') as file: 656 | npc_data = json.load(file) 657 | self.npc_data = npc_data 658 | time.sleep(0.1) 659 | 660 | # # --------------------------robot ---------------------- 661 | 662 | def npc_start(self, number=1): 663 | if not self.time_running: 664 | time_thread = threading.Thread(target=self.env_time.time_simulation, args=(self.stop_event,)) 665 | time_thread.start() 666 | self.time_running = 1 667 | if not self.npc_running: 668 | for npc_i, npc in enumerate(self.npcs): 669 | if npc_i == number: 670 | break 671 | # running_thread = threading.Thread(target=npc.continuous_simulation, args=()) 672 | running_thread = threading.Thread(target=npc.random_walk, args=()) 673 | running_thread.start() 674 | time.sleep(2) 675 | self.npc_running = 1 676 | 677 | def finish_env(self): 678 | print('========== Env end ==========') 679 | self.stop_event.set() 680 | self.agent.running = 0 681 | self.server.env_finish(self.process, self.npcs) 682 | sys.stdout = self.original_stdout 683 | exit(0) 684 | 685 | def sim_speed(self, speed): 686 | instruction = {"requestIndex": 1, "timeScale": speed} 687 | action_id = self.server.send_data(12, instruction, 0) 688 | # print(self.env_time.time_multiplier, speed) 689 | res = self.server.wait_for_respond(action_id, 10) 690 | self.env_time.time_multiplier = speed 691 | return self.env_time.time_multiplier 692 | 693 | def object_query(self, obj_id=0): 694 | instruction = {"requestIndex": 0, "targetType": 1, "targetId": obj_id} 695 | r_id = self.server.send_data(2, instruction, 1) 696 | object_info = self.agent.wait_for_respond(r_id, 60) 697 | if object_info: 698 | object_info = eval(object_info['statusDetail']) 699 | return object_info 700 | 701 | def receptacle_mark(self): 702 | # maps_0 = copy.deepcopy(self.server.maps.maps_info[0]['grid']) 703 | # maps_1 = copy.deepcopy(self.server.maps.maps_info[1]['grid']) 704 | for floor_i in range(3): 705 | maps_2 = copy.deepcopy(self.server.maps.maps_info[floor_i]['grid']) 706 | record = dict() 707 | for rece in self.objs_data.receptacles: 708 | # {'name': name, 'id': id, 'x_max': x_max,'x_min': x_min, 'z_max': z_max, 'z_min': z_min} 709 | x_max, x_min, z_max, z_min, y = rece['x_max'], rece['x_min'], rece['z_max'], rece['z_min'], rece['y'] 710 | floor, map_i1, map_j1, iso = self.server.maps.get_point_info((x_max, y, z_max)) 711 | floor, map_i2, map_j2, iso = self.server.maps.get_point_info((x_min, y, z_min)) 712 | map_i_min, map_i_max = min(map_i1, map_i2), max(map_i1, map_i2) 713 | map_j_min, map_j_max = min(map_j1, map_j2), max(map_j1, map_j2) 714 | for ii in range(map_i_min, map_i_max + 1): 715 | for jj in range(map_j_min, map_j_max + 1): 716 | if maps_2[ii][jj] == 0: 717 | maps_2[ii][jj] = 2 718 | loc = self.objs_data.point_determine((x_min, floor, z_max)) 719 | rece['location'], rece['floor'] = loc, floor 720 | rece['map_i_min'], rece['map_i_max'] = map_i_min, map_i_max 721 | rece['map_j_min'], rece['map_j_max'] = map_j_min, map_j_max 722 | try: 723 | record[loc]['num'] += 1 724 | record[loc]['receptacles'].append(rece) 725 | except: 726 | record[loc] = {'num': 1} 727 | record[loc]['receptacles'] = [rece] 728 | self.objs_data.room_receptacles = record 729 | self.objs_data.sematic_map[floor_i] = maps_2 730 | 731 | # ---------------- delivery task - ------------------------ 732 | 733 | def delivery_task_import(self, task_setup): 734 | target_npc_id = task_setup['npc_id'] 735 | if self.time_running == 0: 736 | self.env_time.time_multiplier = 1 737 | time_thread = threading.Thread(target=self.env_time.time_simulation, args=(self.stop_event,)) 738 | time_thread.start() 739 | # self.sim_speed(1) 740 | self.time_running = 1 741 | if self.npc_running: 742 | return False 743 | else: 744 | for npc_i, npc in enumerate(self.npcs): 745 | if npc_i == target_npc_id: 746 | continue 747 | # npc.random_behavior(task_setup['npc_location'], 1) 748 | self.npcs[target_npc_id].directive_following(task_setup) 749 | basic_information, name_dict = self.npc_data["npc"][target_npc_id]["description"], { 750 | 'name': task_setup['npc_name']} 751 | npc_information = basic_information.format_map(name_dict) 752 | try: 753 | instruction = task_setup['directive'][4] 754 | except: 755 | instruction = task_setup['directive'][0] + task_setup['directive'][1] 756 | data = task_setup 757 | return instruction, npc_information, data 758 | 759 | def delivery_task_evaluate(self, task_data, score=0, save=0): 760 | now_time = self.env_time.current_date.isoformat() 761 | time.sleep(0.3) 762 | tar_npc_id, tar_obj = task_data['npc_id'], task_data['target_object_name'] 763 | tar_obj_inf = self.objs_data.get_info_from_name(tar_obj) 764 | tar_obj_id = tar_obj_inf['itemId'] 765 | npc_pos, npc_info = self.npcs[tar_npc_id].query_information() 766 | obj_info = self.server.object_query(tar_obj_id) 767 | try: 768 | obj_pos = obj_info['position'] 769 | except Exception as e: 770 | print(tar_obj_inf) 771 | print(e, tar_obj_id, obj_info) 772 | obj_pos = obj_info['position'] 773 | agent_pos, agent_info = self.agent.pos_query() 774 | agent_obj_id = self.agent.is_grasp 775 | if agent_obj_id is not None: 776 | agent_obj_info = self.server.object_query(agent_obj_id) 777 | agent_obj_pos = agent_obj_info['position'] 778 | agent_obj_name = agent_obj_info['itemName'] 779 | else: 780 | agent_obj_pos, agent_obj_name = None, None 781 | result = {'task_id': task_data['task_id'], 'tar_object_position': obj_pos, 'tar_object_name': tar_obj, 782 | 'target_object_type': task_data['target_object_type'], 783 | 'agent_object_name': agent_obj_name, 'agent_object_position': agent_obj_pos, 784 | 'npc_position': npc_pos, 'npc_id': tar_npc_id, 'agent_position': agent_pos, 785 | 'start_time': task_data['time'], 'end_time': now_time} 786 | if agent_obj_id is not None: 787 | self.agent.release_object() 788 | pos_original = self.objs_data.objects[agent_obj_id]['position'] 789 | self.server.object_transform(obj_type=1, target_id=agent_obj_id, pos=pos_original) 790 | if save: 791 | return result 792 | if score: 793 | grade = self.delivery_task_score(result) 794 | result['task_score'] = grade 795 | return result 796 | return result 797 | 798 | def delivery_task_score(self, result_data, task_id=0): 799 | task_res, res_grasp, res_find, human_find, res_deliver = 0, 0, 0, 0, 0 800 | start_t = datetime.datetime.fromisoformat(result_data['start_time']) 801 | end_t = datetime.datetime.fromisoformat(result_data['end_time']) 802 | time_cost = end_t - start_t 803 | seconds_cost = (end_t - start_t).total_seconds() 804 | minutes_cost = time_cost.total_seconds() / 60 805 | # 1. target find, 2. grasp target, 3. target object dis, 4. deliver 806 | if result_data['agent_object_name'] is not None: 807 | if result_data['tar_object_name'] == result_data['agent_object_name']: 808 | res_grasp, res_find = 1, 1 809 | else: 810 | if result_data['target_object_type'] in result_data['agent_object_name']: 811 | res_find = 0.5 812 | hr_dis = self.agent.env.calculate_distance(result_data['npc_position'], result_data['agent_position']) 813 | if hr_dis < 3: 814 | human_find = 1 815 | elif hr_dis < 5: 816 | human_find = 0.5 817 | res_deliver = human_find * res_grasp 818 | # Calculate distance, determine items in hand, and calculate time 819 | task_res = res_grasp + res_find + res_deliver + human_find 820 | task_result = {'sub_result': {'grasp': res_grasp, 'object_find': res_find, 821 | 'deliver': res_deliver, 'human_find': human_find}, 822 | 'result': task_res, 'time': minutes_cost} 823 | # task_result = {'sub_result': [res_grasp, res_find, res_deliver, human_find], 824 | # 'result': task_res, 'time': minutes_cost} 825 | return task_result 826 | 827 | 828 | if __name__ == '__main__': # pragma nocover 829 | server = Server() 830 | 831 | 832 | 833 | ''' 834 | -> Unity: {"requestIndex":10,"npcId":0,"actionId":0,"actionPara":""} 835 | -> Get : {"requestIndex":11, "result":1} 836 | 837 | 838 | -> Unity: {"requestIndex":10,"npcId":0,"actionId":0,"actionPara":""} 839 | -> Get : {"result":1} 840 | ''' 841 | --------------------------------------------------------------------------------