├── hacs.json ├── .github └── workflows │ ├── hassfest.yaml │ └── validate.yaml ├── examples ├── prompt │ ├── annoying │ │ └── README.md │ ├── default │ │ └── README.md │ ├── with_attributes │ │ └── README.md │ └── area │ │ └── README.md ├── function │ ├── notify │ │ └── README.md │ ├── say_tts │ │ └── README.md │ ├── attributes │ │ └── README.md │ ├── weather │ │ └── README.md │ ├── user_id_to_user │ │ └── README.md │ ├── netflix │ │ └── README.md │ ├── google_search │ │ └── README.md │ ├── fan │ │ └── README.md │ ├── kakao_bus │ │ └── README.md │ ├── history │ │ └── README.md │ ├── area │ │ └── README.md │ ├── energy │ │ └── README.md │ ├── automation │ │ ├── README.md │ │ └── README.ko.md │ ├── plex │ │ └── README.md │ ├── shopping_list │ │ └── README.md │ ├── calendar │ │ └── README.md │ ├── camera_image_query │ │ └── README.md │ ├── youtube │ │ └── README.md │ └── sqlite │ │ └── README.md └── component_function │ ├── o365 │ └── README.md │ ├── ytube_music_player │ └── README.md │ ├── 17track │ ├── README.md │ └── README.ko.md │ └── grocy │ └── README.md ├── custom_components └── extended_openai_conversation │ ├── manifest.json │ ├── services.yaml │ ├── strings.json │ ├── translations │ ├── en.json │ ├── ko.json │ ├── pl.json │ ├── hu.json │ ├── fr.json │ ├── nl.json │ ├── de.json │ ├── it.json │ ├── pt.json │ ├── el.json │ └── pt-BR.json │ ├── services.py │ ├── const.py │ ├── exceptions.py │ ├── config_flow.py │ ├── __init__.py │ └── helpers.py └── README.md /hacs.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "extended_openai_conversation", 3 | "render_readme": true, 4 | "homeassistant": "2025.12.0b0" 5 | } -------------------------------------------------------------------------------- /.github/workflows/hassfest.yaml: -------------------------------------------------------------------------------- 1 | name: Validate with hassfest 2 | 3 | on: 4 | push: 5 | pull_request: 6 | schedule: 7 | - cron: "0 0 * * *" 8 | 9 | jobs: 10 | validate: 11 | runs-on: "ubuntu-latest" 12 | steps: 13 | - uses: "actions/checkout@v4" 14 | - uses: "home-assistant/actions/hassfest@master" -------------------------------------------------------------------------------- /.github/workflows/validate.yaml: -------------------------------------------------------------------------------- 1 | name: Validate 2 | 3 | on: 4 | push: 5 | pull_request: 6 | schedule: 7 | - cron: "0 0 * * *" 8 | workflow_dispatch: 9 | 10 | jobs: 11 | validate-hacs: 12 | runs-on: "ubuntu-latest" 13 | steps: 14 | - uses: "actions/checkout@v3" 15 | - name: HACS validation 16 | uses: "hacs/action@main" 17 | with: 18 | category: "integration" -------------------------------------------------------------------------------- /examples/prompt/annoying/README.md: -------------------------------------------------------------------------------- 1 | ## Objective 2 | - Just for fun 3 | 4 | ## Prompt 5 | 6 | ````yaml 7 | You are the most annoying assistant of Home Assistant 8 | Always answer in a rude manner using a list of available devices. 9 | A list of available devices in this smart home: 10 | 11 | ```csv 12 | entity_id,name,state,aliases 13 | {% for entity in exposed_entities -%} 14 | {{ entity.entity_id }},{{ entity.name }},{{entity.state}},{{entity.aliases | join('/')}} 15 | {% endfor -%} 16 | ``` 17 | 18 | If user asks for devices that are not available, do not have to answer. 19 | ```` 20 | -------------------------------------------------------------------------------- /custom_components/extended_openai_conversation/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "domain": "extended_openai_conversation", 3 | "name": "Extended OpenAI Conversation", 4 | "codeowners": [ 5 | "@jekalmin" 6 | ], 7 | "config_flow": true, 8 | "dependencies": [ 9 | "conversation", 10 | "energy", 11 | "history", 12 | "recorder", 13 | "rest", 14 | "scrape" 15 | ], 16 | "documentation": "https://github.com/jekalmin/extended_openai_conversation", 17 | "integration_type": "service", 18 | "iot_class": "cloud_polling", 19 | "issue_tracker": "https://github.com/jekalmin/extended_openai_conversation/issues", 20 | "requirements": [ 21 | "openai~=2.8.0" 22 | ], 23 | "version": "1.2.1" 24 | } -------------------------------------------------------------------------------- /examples/function/notify/README.md: -------------------------------------------------------------------------------- 1 | ## Objective 2 | 3 | 4 | ## Function 5 | 6 | ### send_message_to_messenger 7 | ```yaml 8 | - spec: 9 | name: send_message_to_messenger 10 | description: Use this function to send message to messenger. 11 | parameters: 12 | type: object 13 | properties: 14 | message: 15 | type: string 16 | description: message you want to send 17 | required: 18 | - message 19 | function: 20 | type: script 21 | sequence: 22 | - service: notify.{YOUR_MESSENGER} 23 | data: 24 | message: "{{ message }}" 25 | ``` -------------------------------------------------------------------------------- /examples/function/say_tts/README.md: -------------------------------------------------------------------------------- 1 | ## Objective 2 | 3 | - say_tts will say a message on any text to speech device 4 | 5 | ## Function 6 | 7 | ### say_tts 8 | ```yaml 9 | - spec: 10 | name: say_tts 11 | description: Say message on a text to speech device 12 | parameters: 13 | type: object 14 | properties: 15 | message: 16 | type: string 17 | description: message you want to say 18 | device: 19 | type: string 20 | description: entity_id of media_player tts device 21 | required: 22 | - message 23 | - device 24 | function: 25 | type: script 26 | sequence: 27 | - service: tts.cloud_say 28 | data: 29 | entity_id: "{{device}}" 30 | message: "{{message}}" 31 | ``` 32 | -------------------------------------------------------------------------------- /examples/function/attributes/README.md: -------------------------------------------------------------------------------- 1 | ## Objective 2 | - Get attributes of entity 3 | 4 | 5 | 6 | 7 | ## Function 8 | 9 | ### get_attributes 10 | ```yaml 11 | - spec: 12 | name: get_attributes 13 | description: Get attributes of any home assistant entity 14 | parameters: 15 | type: object 16 | properties: 17 | entity_id: 18 | type: string 19 | description: entity_id 20 | required: 21 | - entity_id 22 | function: 23 | type: template 24 | value_template: "{{states[entity_id]}}" 25 | ``` -------------------------------------------------------------------------------- /examples/prompt/default/README.md: -------------------------------------------------------------------------------- 1 | ## Prompt 2 | 3 | ````yaml 4 | I want you to act as smart home manager of Home Assistant. 5 | I will provide information of smart home along with a question, you will truthfully make correction or answer using information provided in one sentence in everyday language. 6 | 7 | Current Time: {{now()}} 8 | 9 | Available Devices: 10 | ```csv 11 | entity_id,name,state,aliases 12 | {% for entity in exposed_entities -%} 13 | {{ entity.entity_id }},{{ entity.name }},{{ entity.state }},{{entity.aliases | join('/')}} 14 | {% endfor -%} 15 | ``` 16 | 17 | The current state of devices is provided in available devices. 18 | Use execute_services function only for requested action, not for current states. 19 | Do not execute service without user's confirmation. 20 | Do not restate or appreciate what user says, rather make a quick inquiry. 21 | ```` -------------------------------------------------------------------------------- /examples/function/weather/README.md: -------------------------------------------------------------------------------- 1 | ## Objective 2 | - Get current weather and forecasts 3 | 4 | 5 | 6 | 7 | ## Prerequisite 8 | Expose `weather.xxxxx` entity 9 | 10 | ## Function 11 | 12 | ### get_attributes 13 | ```yaml 14 | - spec: 15 | name: get_attributes 16 | description: Get attributes of any home assistant entity 17 | parameters: 18 | type: object 19 | properties: 20 | entity_id: 21 | type: string 22 | description: entity_id 23 | required: 24 | - entity_id 25 | function: 26 | type: template 27 | value_template: "{{states[entity_id]}}" 28 | ``` -------------------------------------------------------------------------------- /examples/function/user_id_to_user/README.md: -------------------------------------------------------------------------------- 1 | ## Objective 2 | - Map user_id to friendly user name string 3 | 4 | When the option to pass the current user to OpenAI via the user 5 | message context is enabled, we actually pass the user_id rather than a 6 | friendly user name as OpenAI has limitations on characters it accepts. 7 | This function can be used to resolve the user's name, without 8 | limitation on acceptable characters. 9 | 10 | ## Function 11 | 12 | ### get_user_from_user_id 13 | ```yaml 14 | - spec: 15 | name: get_user_from_user_id 16 | description: Retrieve users name from the supplied user id hash 17 | parameters: 18 | type: object 19 | properties: 20 | user_id: 21 | type: string 22 | description: user_id 23 | required: 24 | - user_id 25 | function: 26 | type: native 27 | name: get_user_from_user_id 28 | ``` -------------------------------------------------------------------------------- /custom_components/extended_openai_conversation/services.yaml: -------------------------------------------------------------------------------- 1 | query_image: 2 | fields: 3 | config_entry: 4 | required: true 5 | selector: 6 | config_entry: 7 | integration: extended_openai_conversation 8 | model: 9 | example: gpt-4-vision-preview 10 | selector: 11 | text: 12 | prompt: 13 | example: "What’s in this image?" 14 | required: true 15 | selector: 16 | text: 17 | multiline: true 18 | images: 19 | example: '{"url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"}' 20 | required: true 21 | default: [] 22 | selector: 23 | object: 24 | max_tokens: 25 | example: 300 26 | default: 300 27 | selector: 28 | number: 29 | min: 1 30 | mode: box 31 | -------------------------------------------------------------------------------- /examples/component_function/o365/README.md: -------------------------------------------------------------------------------- 1 | ## Requirement 2 | Assume using [o365](https://github.com/PTST/O365-HomeAssistant) 3 | 4 | ## Function 5 | ### get_email_inbox 6 | ```yaml 7 | - spec: 8 | name: get_email_inbox 9 | description: Use this function to retrieve the list of emails from the inbox. 10 | parameters: 11 | type: object 12 | properties: {} 13 | function: 14 | type: template 15 | value_template: >- 16 | {% set data = states['sensor.inbox'].attributes['data'] | list %} 17 | ```csv 18 | subject,received,to,sender,has_attachments,importance,is_read,body 19 | {% for email in data -%} 20 | "{{ email['subject'] }}","{{ email['received'] }}","{{ email['to'] | join(', ') }}","{{ email['sender'] }}",{{ email['has_attachments'] }},{{ email['importance'] }},{{ email['is_read'] }},"{{ email['body'] | replace('\n', ' ') | replace('"', '\\"') }}" 21 | {% endfor -%} 22 | ``` 23 | ``` -------------------------------------------------------------------------------- /examples/function/netflix/README.md: -------------------------------------------------------------------------------- 1 | ## Objective 2 | https://github.com/jekalmin/extended_openai_conversation/assets/2917984/64ba656e-3ae7-4003-9956-da71efaf06dc 3 | 4 | ## Prompt 5 | Add following text in your prompt 6 | ```` 7 | Netflix Video: 8 | ```csv 9 | video_id,title 10 | 81040344,Squid Game 11 | ``` 12 | ```` 13 | ## Function 14 | 15 | ### play_netflix 16 | #### webostv 17 | ```yaml 18 | - spec: 19 | name: play_netflix 20 | description: Use this function to play Netflix. 21 | parameters: 22 | type: object 23 | properties: 24 | video_id: 25 | type: string 26 | description: The video id. 27 | required: 28 | - video_id 29 | function: 30 | type: script 31 | sequence: 32 | - service: webostv.command 33 | data: 34 | entity_id: media_player.{YOUR_WEBOSTV} 35 | command: system.launcher/launch 36 | payload: 37 | id: netflix 38 | contentId: "m=https://www.netflix.com/watch/{{video_id}}" 39 | ``` -------------------------------------------------------------------------------- /examples/function/google_search/README.md: -------------------------------------------------------------------------------- 1 | ## Objective 2 | - Search from Google 3 | 4 | ## Prerequisite 5 | Needs Google API Key 6 | 7 | ## Function 8 | 9 | ### search_google 10 | 11 | ```yaml 12 | - spec: 13 | name: search_google 14 | description: Search Google using the Custom Search API. 15 | parameters: 16 | type: object 17 | properties: 18 | query: 19 | type: string 20 | description: The search query. 21 | required: 22 | - query 23 | function: 24 | type: rest 25 | resource_template: "https://www.googleapis.com/customsearch/v1?key=[GOOGLE_API_KEY]&cx=[GOOGLE_PROGRAMMING_SEARCH_ENGINE]:omuauf_lfve&q={{ query }}&num=3" 26 | value_template: >- 27 | {% if value_json.items %} 28 | ```csv 29 | title,link 30 | {% for item in value_json.items %} 31 | "{{ item.title | replace(',', ' ') }}","{{ item.link }}" 32 | {% endfor %} 33 | ``` 34 | {% else %} 35 | No results found, 36 | {% endif %} 37 | ``` -------------------------------------------------------------------------------- /examples/function/fan/README.md: -------------------------------------------------------------------------------- 1 | ## Objective 2 | This function is used to set a preset mode on a fan entity. Within this example the preset mode can be set to `off`, `low` or `high`. 3 | 4 | ## Function 5 | 6 | ### set_fan_preset_mode 7 | ```yaml 8 | - spec: 9 | name: set_fan_preset_mode 10 | description: Use this function to set the preset mode of a fan to "off", "low" or "high". 11 | parameters: 12 | type: object 13 | properties: 14 | entity_id: 15 | type: string 16 | description: entity_id of the fan 17 | preset_mode: 18 | type: string 19 | description: preset mode you want to set 20 | enum: 21 | - off 22 | - low 23 | - high 24 | required: 25 | - entity_id 26 | - preset_mode 27 | function: 28 | type: script 29 | sequence: 30 | - service: fan.set_preset_mode 31 | target: 32 | entity_id: "{{ entity_id }}" 33 | data: 34 | preset_mode: "{{ preset_mode }}" 35 | ``` -------------------------------------------------------------------------------- /examples/component_function/ytube_music_player/README.md: -------------------------------------------------------------------------------- 1 | ## Objective 2 | 3 | ## Requirement 4 | Assume using [ytube_music_player](https://github.com/KoljaWindeler/ytube_music_player) 5 | 6 | ## Function 7 | 8 | ### search_music 9 | ```yaml 10 | - spec: 11 | name: search_music 12 | description: Use this function to search music 13 | parameters: 14 | type: object 15 | properties: 16 | query: 17 | type: string 18 | description: The query 19 | required: 20 | - query 21 | function: 22 | type: composite 23 | sequence: 24 | - type: script 25 | sequence: 26 | - service: ytube_music_player.search 27 | data: 28 | entity_id: media_player.ytube_music_player 29 | query: "{{ query }}" 30 | - type: template 31 | value_template: >- 32 | media_content_type,media_content_id,title 33 | {% for media in state_attr('sensor.ytube_music_player_extra', 'search') -%} 34 | {{media.type}},{{media.id}},{{media.title}} 35 | {% endfor%} 36 | ``` -------------------------------------------------------------------------------- /examples/function/kakao_bus/README.md: -------------------------------------------------------------------------------- 1 | ## Function 2 | 3 | ### get_bus_info (scrape ver.) 4 | ```yaml 5 | - spec: 6 | name: get_bus_info 7 | description: Use this function to get bus information 8 | parameters: 9 | type: object 10 | properties: 11 | dummy: 12 | type: string 13 | description: Nothing 14 | function: 15 | type: scrape 16 | resource: https://m.map.kakao.com/actions/busStationInfo?busStopId=BS219668 17 | value_template: "remain time: {{[next | trim, next_of_next | trim]}}" 18 | sensor: 19 | - name: next 20 | select: "li[data-id='1100061486'] span.txt_situation" 21 | index: 0 22 | - name: next_of_next 23 | select: "li[data-id='1100061486'] span.txt_situation" 24 | index: 1 25 | ``` 26 | 27 | ### get_bus_info (rest ver.) 28 | ```yaml 29 | - spec: 30 | name: get_bus_info 31 | description: Use this function to get bus information 32 | parameters: 33 | type: object 34 | properties: 35 | dummy: 36 | type: string 37 | description: Nothing. 38 | function: 39 | type: rest 40 | resource: https://m.map.kakao.com/actions/busesInBusStopJson?busStopId=BS219668 41 | value_template: '{{value_json["busesList"] | selectattr("id", "==", "1100061486") | map(attribute="vehicleStateMessage") | list }}' 42 | ``` -------------------------------------------------------------------------------- /examples/function/history/README.md: -------------------------------------------------------------------------------- 1 | ## Objective 2 | 3 | 4 | 5 | 6 | ## Function 7 | 8 | ### get_history 9 | ```yaml 10 | - spec: 11 | name: get_history 12 | description: Retrieve historical data of specified entities. 13 | parameters: 14 | type: object 15 | properties: 16 | entity_ids: 17 | type: array 18 | items: 19 | type: string 20 | description: The entity id to filter. 21 | start_time: 22 | type: string 23 | description: Start of the history period in "%Y-%m-%dT%H:%M:%S%z". 24 | end_time: 25 | type: string 26 | description: End of the history period in "%Y-%m-%dT%H:%M:%S%z". 27 | required: 28 | - entity_ids 29 | function: 30 | type: composite 31 | sequence: 32 | - type: native 33 | name: get_history 34 | response_variable: history_result 35 | - type: template 36 | value_template: >- 37 | {% set ns = namespace(result = [], list = []) %} 38 | {% for item_list in history_result %} 39 | {% set ns.list = [] %} 40 | {% for item in item_list %} 41 | {% set last_changed = item.last_changed | as_timestamp | timestamp_local if item.last_changed else None %} 42 | {% set new_item = dict(item, last_changed=last_changed) %} 43 | {% set ns.list = ns.list + [new_item] %} 44 | {% endfor %} 45 | {% set ns.result = ns.result + [ns.list] %} 46 | {% endfor %} 47 | {{ ns.result }} 48 | ``` 49 | -------------------------------------------------------------------------------- /examples/function/area/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Objective 4 | - Call service via area_id 5 | 6 | 7 | 8 | 9 | ## Function 10 | 11 | ### execute_services 12 | ```yaml 13 | - spec: 14 | name: execute_services 15 | description: Execute service of devices in Home Assistant. 16 | parameters: 17 | type: object 18 | properties: 19 | list: 20 | type: array 21 | items: 22 | type: object 23 | properties: 24 | domain: 25 | type: string 26 | description: The domain of the service. 27 | service: 28 | type: string 29 | description: The service to be called 30 | service_data: 31 | type: object 32 | description: The service data object to indicate what to control. 33 | properties: 34 | entity_id: 35 | type: array 36 | items: 37 | type: string 38 | description: The entity_id retrieved from available devices. It must start with domain, followed by dot character. 39 | area_id: 40 | type: array 41 | items: 42 | type: string 43 | description: The id retrieved from areas. You can specify only area_id without entity_id to act on all entities in that area 44 | required: 45 | - domain 46 | - service 47 | - service_data 48 | function: 49 | type: native 50 | name: execute_service 51 | ``` -------------------------------------------------------------------------------- /examples/function/energy/README.md: -------------------------------------------------------------------------------- 1 | ## Objective 2 | - Get energy statistics 3 | 4 | 5 | 6 | 7 | ## Function 8 | 9 | ### get_energy_statistic_ids 10 | ```yaml 11 | - spec: 12 | name: get_energy_statistic_ids 13 | description: Get statistics 14 | parameters: 15 | type: object 16 | properties: 17 | dummy: 18 | type: string 19 | description: Nothing 20 | function: 21 | type: composite 22 | sequence: 23 | - type: native 24 | name: get_energy 25 | response_variable: result 26 | - type: template 27 | value_template: "{{result.device_consumption | map(attribute='stat_consumption') | list}}" 28 | ``` 29 | ### get_statistics 30 | ```yaml 31 | - spec: 32 | name: get_statistics 33 | description: Get statistics 34 | parameters: 35 | type: object 36 | properties: 37 | start_time: 38 | type: string 39 | description: The start datetime 40 | end_time: 41 | type: string 42 | description: The end datetime 43 | statistic_ids: 44 | type: array 45 | items: 46 | type: string 47 | description: The statistic ids 48 | period: 49 | type: string 50 | description: The period 51 | enum: 52 | - day 53 | - week 54 | - month 55 | required: 56 | - start_time 57 | - end_time 58 | - statistic_ids 59 | - period 60 | function: 61 | type: native 62 | name: get_statistics 63 | ``` -------------------------------------------------------------------------------- /examples/component_function/17track/README.md: -------------------------------------------------------------------------------- 1 | ## Requirement 2 | Assume using [17track](https://www.home-assistant.io/integrations/seventeentrack) 3 | 4 | ## Function 5 | 6 | ### get_incoming_packages 7 | 8 | ```yaml 9 | - spec: 10 | name: get_incoming_packages 11 | description: Use this function to retrieve information about incoming packages. 12 | parameters: 13 | type: object 14 | properties: {} 15 | function: 16 | type: template 17 | value_template: >- 18 | {% set ns = namespace(current_status=None) %} 19 | {% set statuses = { 20 | 'expired': states('sensor.17track_packages_expired')|int, 21 | 'undelivered': states('sensor.17track_packages_undelivered')|int, 22 | 'delivered': states('sensor.17track_packages_delivered')|int, 23 | 'ready_to_be_picked_up': states('sensor.17track_packages_ready_to_be_picked_up')|int, 24 | 'returned': states('sensor.17track_packages_returned')|int, 25 | 'in_transit': states('sensor.17track_packages_in_transit')|int, 26 | 'not_found': states('sensor.17track_packages_not_found')|int 27 | } %} 28 | {% set priority_order = ['expired', 'undelivered', 'delivered', 'ready_to_be_picked_up', 'returned', 'in_transit', 'not_found'] %} 29 | ```csv 30 | package status,package name,details 31 | {%- for status in priority_order %} 32 | {%- set current_status = status %} 33 | {%- set current_package_count = statuses[current_status] %} 34 | {%- if current_package_count > 0 %} 35 | {%- set package_details = state_attr('sensor.17track_packages_' + current_status, 'packages') %} 36 | {%- for package in package_details %} 37 | {{ current_status }},{{ package.friendly_name }},{{ package.info_text | replace(",", ";") }} 38 | {%- endfor %} 39 | {%- endif %} 40 | {%- endfor -%} 41 | ``` 42 | ``` -------------------------------------------------------------------------------- /examples/function/automation/README.md: -------------------------------------------------------------------------------- 1 | ## Objective 2 | 3 | 4 | 5 | ## Notice 6 | 7 | Before adding automation, I highly recommend set notification on `automation_registered_via_extended_openai_conversation` event and create separate "Extended OpenAI Assistant" and "Assistant" 8 | 9 | (Automation can be added even if conversation fails because of failure to get response message, not automation) 10 | 11 | | Create Assistant | Notify on created | 12 | |----------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| 13 | | 1 | 스크린샷 2023-10-13 오후 6 01 40 | 14 | 15 | 16 | Copy and paste below configuration into "Functions" 17 | 18 | ## Function 19 | ### add_automation 20 | ```yaml 21 | - spec: 22 | name: add_automation 23 | description: Use this function to add an automation in Home Assistant. 24 | parameters: 25 | type: object 26 | properties: 27 | automation_config: 28 | type: string 29 | description: A configuration for automation in a valid yaml format. Next line character should be \n. Use devices from the list. 30 | required: 31 | - automation_config 32 | function: 33 | type: native 34 | name: add_automation 35 | ``` -------------------------------------------------------------------------------- /examples/function/automation/README.ko.md: -------------------------------------------------------------------------------- 1 | ## Objective 2 | 3 | 4 | 5 | ## Notice 6 | 7 | Before adding automation, I highly recommend set notification on `automation_registered_via_extended_openai_conversation` event and create separate "Extended OpenAI Assistant" and "Assistant" 8 | 9 | (Automation can be added even if conversation fails because of failure to get response message, not automation) 10 | 11 | | Create Assistant | Notify on created | 12 | |----------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| 13 | | 1 | 스크린샷 2023-10-13 오후 6 01 40 | 14 | 15 | 16 | Copy and paste below configuration into "Functions" 17 | 18 | ## Function 19 | ### add_automation 20 | ```yaml 21 | - spec: 22 | name: add_automation 23 | description: Use this function to add an automation in Home Assistant. 24 | parameters: 25 | type: object 26 | properties: 27 | automation_config: 28 | type: string 29 | description: A configuration for automation in a valid yaml format. Next line character should be \\n, not \n. Use devices from the list. 30 | required: 31 | - automation_config 32 | function: 33 | type: native 34 | name: add_automation 35 | ``` 36 | 37 | -------------------------------------------------------------------------------- /examples/component_function/17track/README.ko.md: -------------------------------------------------------------------------------- 1 | ## Requirement 2 | Assume using [17track](https://www.home-assistant.io/integrations/seventeentrack) 3 | 4 | ## Function 5 | 6 | ### get_incoming_packages 7 | 8 | ```yaml 9 | - spec: 10 | name: get_incoming_packages 11 | description: Use this function to retrieve information about incoming packages. 12 | parameters: 13 | type: object 14 | properties: {} 15 | function: 16 | type: template 17 | value_template: >- 18 | {% set ns = namespace(current_status=None) %} 19 | {% set statuses = { 20 | 'expired': states('sensor.17track_packages_expired')|int, 21 | 'undelivered': states('sensor.17track_packages_undelivered')|int, 22 | 'delivered': states('sensor.17track_packages_delivered')|int, 23 | 'ready_to_be_picked_up': states('sensor.17track_packages_ready_to_be_picked_up')|int, 24 | 'returned': states('sensor.17track_packages_returned')|int, 25 | 'in_transit': states('sensor.17track_packages_in_transit')|int, 26 | 'not_found': states('sensor.17track_packages_not_found')|int 27 | } %} 28 | {% set priority_order = ['expired', 'undelivered', 'delivered', 'ready_to_be_picked_up', 'returned', 'in_transit', 'not_found'] %} 29 | {% set friendly_status = { 30 | 'expired': '만료', 31 | 'undelivered': '미배송', 32 | 'delivered': '배송 완료', 33 | 'ready_to_be_picked_up': '배송 출발', 34 | 'returned': '반송', 35 | 'in_transit': '배송 중', 36 | 'not_found': '찾을 수 없음' 37 | } %} 38 | ```csv 39 | package status,package name,details 40 | {%- for status in priority_order %} 41 | {%- set current_status = status %} 42 | {%- set current_package_count = statuses[current_status] %} 43 | {%- if current_package_count > 0 %} 44 | {%- set package_details = state_attr('sensor.17track_packages_' + current_status, 'packages') %} 45 | {%- for package in package_details %} 46 | {{ friendly_status[current_status] }},{{ package.friendly_name }},{{ package.info_text | replace(",", ";") }} 47 | {%- endfor %} 48 | {%- endif %} 49 | {%- endfor -%} 50 | ``` 51 | ``` -------------------------------------------------------------------------------- /examples/function/plex/README.md: -------------------------------------------------------------------------------- 1 | ## Function 2 | 3 | ### search_plex 4 | ```yaml 5 | - spec: 6 | name: search_plex 7 | description: Use this function to search for media in Plex. 8 | parameters: 9 | type: object 10 | properties: 11 | query: 12 | type: string 13 | description: The search query to look up media on Plex. 14 | required: 15 | - query 16 | - token 17 | function: 18 | type: rest 19 | resource_template: "https://YOUR.PLEX.SERVER.TLD/search?query={{query}}&X-Plex-Token=YOURPLEXTOKEN" 20 | value_template: >- 21 | ```csv 22 | title,year,director,type,key 23 | {% for metadata in value_json["MediaContainer"]["Metadata"] %} 24 | {{ metadata["title"]|replace(",", " ") }}, 25 | {{ metadata["year"] }}, 26 | {{ metadata["Director"][0]["tag"] if metadata["Director"] else "N/A" }}, 27 | {{ metadata["type"] }}, 28 | {{ metadata["key"] }} 29 | {% endfor -%} 30 | ``` 31 | ``` 32 | 33 | ### play_plex_media_in_apple_tv 34 | 35 | ```yaml 36 | - spec: 37 | name: play_plex_media_in_apple_tv 38 | description: Use this function to play Plex media on an Apple TV. 39 | parameters: 40 | type: object 41 | properties: 42 | key: 43 | type: string 44 | description: The key of the media in Plex. 45 | entity_id: 46 | type: string 47 | description: The entity ID of the Apple TV in Home Assistant. 48 | type: 49 | type: string 50 | enum: 51 | - movie 52 | - show 53 | - episode 54 | required: 55 | - key 56 | - entity_id 57 | - type 58 | function: 59 | type: script 60 | sequence: 61 | - service: script.play_plex_media_on_apple_tv 62 | data: 63 | kind: "{{ kind }}" 64 | content_id: "{{ content_id }}" 65 | player: "{{ player }}" 66 | ``` 67 | 68 | ```yaml 69 | script: 70 | play_plex_media_on_apple_tv: 71 | alias: "Play Plex Media on Apple TV" 72 | sequence: 73 | - service: media_player.play_media 74 | data_template: 75 | media_content_type: "{{ type }}" 76 | media_content_id: "plex://MYSERVERID/{{ key }}" 77 | target: 78 | entity_id: "{{ entity_id }}" 79 | ``` -------------------------------------------------------------------------------- /examples/function/shopping_list/README.md: -------------------------------------------------------------------------------- 1 | ## Objective 2 | 3 | 4 | ## Function 5 | 6 | ### add_item_to_list 7 | ```yaml 8 | - spec: 9 | name: add_item_to_list 10 | description: Add item to a list 11 | parameters: 12 | type: object 13 | properties: 14 | item: 15 | type: string 16 | description: The item to be added to the list 17 | list: 18 | type: string 19 | description: the entity id of the list to update 20 | enum: 21 | - todo.shopping_list 22 | - todo.to_do 23 | required: 24 | - item 25 | - list 26 | function: 27 | type: script 28 | sequence: 29 | - service: todo.add_item 30 | data: 31 | item: '{{item}}' 32 | target: 33 | entity_id: '{{list}}' 34 | ``` 35 | 36 | ### remove_item_from_list 37 | ```yaml 38 | - spec: 39 | name: remove_item_from_list 40 | description: Check an item off a list 41 | parameters: 42 | type: object 43 | properties: 44 | item: 45 | type: string 46 | description: The item to be removed from the list 47 | list: 48 | type: string 49 | description: the entity id of the list to update 50 | enum: 51 | - todo.shopping_list 52 | - todo.to_do 53 | required: 54 | - item 55 | - list 56 | function: 57 | type: script 58 | sequence: 59 | - service: todo.update_item 60 | data: 61 | item: '{{item}}' 62 | status: 'completed' 63 | target: 64 | entity_id: '{{list}}' 65 | ``` 66 | 67 | ### get_items_from_list 68 | ```yaml 69 | - spec: 70 | name: get_items_from_list 71 | description: Read back items from a list 72 | parameters: 73 | type: object 74 | properties: 75 | list: 76 | type: string 77 | description: the entity id of the list to update 78 | enum: 79 | - todo.shopping_list 80 | - todo.to_do 81 | required: 82 | - list 83 | function: 84 | type: script 85 | sequence: 86 | - service: todo.get_items 87 | data: 88 | status: 'needs_action' 89 | target: 90 | entity_id: '{{list}}' 91 | response_variable: _function_result 92 | ``` 93 | -------------------------------------------------------------------------------- /examples/function/calendar/README.md: -------------------------------------------------------------------------------- 1 | ## Objective 2 | 스크린샷 2023-10-31 오후 9 04 56 3 | 4 | ## Function 5 | ### 1. get_events 6 | ```yaml 7 | - spec: 8 | name: get_events 9 | description: Use this function to get list of calendar events. 10 | parameters: 11 | type: object 12 | properties: 13 | start_date_time: 14 | type: string 15 | description: The start date time in '%Y-%m-%dT%H:%M:%S%z' format 16 | end_date_time: 17 | type: string 18 | description: The end date time in '%Y-%m-%dT%H:%M:%S%z' format 19 | required: 20 | - start_date_time 21 | - end_date_time 22 | function: 23 | type: script 24 | sequence: 25 | - service: calendar.get_events 26 | data: 27 | start_date_time: "{{start_date_time}}" 28 | end_date_time: "{{end_date_time}}" 29 | target: 30 | entity_id: 31 | - calendar.[YourCalendarHere] 32 | - calendar.[MoreCalendarsArePossible] 33 | response_variable: _function_result 34 | ``` 35 | 36 | ### 2. create_event 37 | ```yaml 38 | - spec: 39 | name: create_event 40 | description: Adds a new calendar event. 41 | parameters: 42 | type: object 43 | properties: 44 | summary: 45 | type: string 46 | description: Defines the short summary or subject for the event. 47 | description: 48 | type: string 49 | description: A more complete description of the event than the one provided by the summary. 50 | start_date_time: 51 | type: string 52 | description: The date and time the event should start. 53 | end_date_time: 54 | type: string 55 | description: The date and time the event should end. 56 | location: 57 | type: string 58 | description: The location 59 | required: 60 | - summary 61 | function: 62 | type: script 63 | sequence: 64 | - service: calendar.create_event 65 | data: 66 | summary: "{{summary}}" 67 | description: "{{description}}" 68 | start_date_time: "{{start_date_time}}" 69 | end_date_time: "{{end_date_time}}" 70 | location: "{{location}}" 71 | target: 72 | entity_id: calendar.[YourCalendarHere] 73 | ``` -------------------------------------------------------------------------------- /examples/prompt/with_attributes/README.md: -------------------------------------------------------------------------------- 1 | ## Objective 2 | Add attributes of entities that are configured in `customize_glob_exposed_attributes`. 3 | It is similar to [customize_glob](https://www.home-assistant.io/docs/configuration/customizing-devices/) of Home Assistant. 4 | It uses regular expression as a pattern. 5 | 6 | If value is true, attribute is included. If false, attribute is excluded.
7 | If value is not boolean, the value is included, not value of attribute. 8 | 9 | 10 | ## Prompt 11 | 12 | ````yaml 13 | {%- set customize_glob_exposed_attributes = { 14 | ".*": { 15 | "friendly_name": true, 16 | }, 17 | "timer\..*": { 18 | "duration": true, 19 | }, 20 | "sun.sun": { 21 | "next_dawn": true, 22 | "next_midnight": true, 23 | }, 24 | "media_player.YOUR_WEBOS_TV": { 25 | "source_list": ["Netflix","YouTube","wavve"], 26 | "source": true, 27 | }, 28 | } %} 29 | 30 | {%- macro get_exposed_attributes(entity_id) -%} 31 | {%- set ns = namespace(exposed_attributes = {}, result = {}) %} 32 | {%- for pattern, attributes in customize_glob_exposed_attributes.items() -%} 33 | {%- if entity_id | regex_match(pattern) -%} 34 | {%- set ns.exposed_attributes = dict(ns.exposed_attributes, **attributes) -%} 35 | {%- endif -%} 36 | {%- endfor -%} 37 | {%- for attribute_key, should_include in ns.exposed_attributes.items() -%} 38 | {%- if should_include and state_attr(entity_id, attribute_key) != None -%} 39 | {%- set temp = {attribute_key: state_attr(entity_id, attribute_key)} if should_include is boolean else {attribute_key: should_include} -%} 40 | {%- set ns.result = dict(ns.result, **temp) -%} 41 | {%- endif -%} 42 | {%- endfor -%} 43 | {%- set result = ns.result | to_json if ns.result!={} else None -%} 44 | {{"'" + result + "'" if result != None else ''}} 45 | {%- endmacro -%} 46 | 47 | I want you to act as smart home manager of Home Assistant. 48 | I will provide information of smart home along with a question, you will truthfully make correction or answer using information provided in one sentence in everyday language. 49 | 50 | Current Time: {{now()}} 51 | 52 | Available Devices: 53 | ```csv 54 | entity_id,name,state,aliases,attributes 55 | {% for entity in exposed_entities -%} 56 | {{ entity.entity_id }},{{ entity.name }},{{ entity.state }},{{entity.aliases | join('/')}},{{get_exposed_attributes(entity.entity_id)}} 57 | {% endfor -%} 58 | ``` 59 | 60 | The current state of devices is provided in available devices. 61 | Use execute_services function only for requested action, not for current states. 62 | Do not execute service without user's confirmation. 63 | Do not restate or appreciate what user says, rather make a quick inquiry. 64 | ```` -------------------------------------------------------------------------------- /custom_components/extended_openai_conversation/strings.json: -------------------------------------------------------------------------------- 1 | { 2 | "config": { 3 | "step": { 4 | "user": { 5 | "data": { 6 | "name": "[%key:common::config_flow::data::name%]", 7 | "api_key": "[%key:common::config_flow::data::api_key%]", 8 | "base_url": "[%key:common::config_flow::data::base_url%]", 9 | "api_version": "[%key:common::config_flow::data::api_version%]", 10 | "organization": "[%key:common::config_flow::data::organization%]", 11 | "skip_authentication": "[%key:common::config_flow::data::skip_authentication%]" 12 | } 13 | } 14 | }, 15 | "error": { 16 | "cannot_connect": "[%key:common::config_flow::error::cannot_connect%]", 17 | "invalid_auth": "[%key:common::config_flow::error::invalid_auth%]", 18 | "unknown": "[%key:common::config_flow::error::unknown%]" 19 | } 20 | }, 21 | "options": { 22 | "step": { 23 | "init": { 24 | "data": { 25 | "prompt": "Prompt Template", 26 | "model": "Completion Model", 27 | "max_tokens": "Maximum tokens to return in response", 28 | "temperature": "Temperature", 29 | "top_p": "Top P", 30 | "max_function_calls_per_conversation": "Maximum function calls per conversation", 31 | "functions": "Functions", 32 | "attach_username": "Attach Username to Message", 33 | "use_tools": "Use Tools", 34 | "context_threshold": "Context Threshold", 35 | "context_truncate_strategy": "Context truncation strategy when exceeded threshold" 36 | } 37 | } 38 | } 39 | }, 40 | "services": { 41 | "query_image": { 42 | "name": "Query image", 43 | "description": "Take in images and answer questions about them", 44 | "fields": { 45 | "config_entry": { 46 | "name": "Config Entry", 47 | "description": "The config entry to use for this service" 48 | }, 49 | "model": { 50 | "name": "Model", 51 | "description": "The model", 52 | "example": "gpt-4-vision-preview" 53 | }, 54 | "prompt": { 55 | "name": "Prompt", 56 | "description": "The text to ask about image", 57 | "example": "What’s in this image?" 58 | }, 59 | "images": { 60 | "name": "Images", 61 | "description": "A list of images that would be asked", 62 | "example": "{\"url\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg\"}" 63 | }, 64 | "max_tokens": { 65 | "name": "Max Tokens", 66 | "description": "The maximum tokens", 67 | "example": "300" 68 | } 69 | } 70 | } 71 | } 72 | } -------------------------------------------------------------------------------- /custom_components/extended_openai_conversation/translations/en.json: -------------------------------------------------------------------------------- 1 | { 2 | "config": { 3 | "error": { 4 | "cannot_connect": "Failed to connect", 5 | "invalid_auth": "Invalid authentication", 6 | "unknown": "Unexpected error" 7 | }, 8 | "step": { 9 | "user": { 10 | "data": { 11 | "name": "Name", 12 | "api_key": "API Key", 13 | "base_url": "Base Url", 14 | "api_version": "Api Version", 15 | "organization": "Organization", 16 | "skip_authentication": "Skip Authentication" 17 | } 18 | } 19 | } 20 | }, 21 | "options": { 22 | "step": { 23 | "init": { 24 | "data": { 25 | "max_tokens": "Maximum tokens to return in response", 26 | "model": "Completion Model", 27 | "prompt": "Prompt Template", 28 | "temperature": "Temperature", 29 | "top_p": "Top P", 30 | "max_function_calls_per_conversation": "Maximum function calls per conversation", 31 | "functions": "Functions", 32 | "attach_username": "Attach Username to Message", 33 | "use_tools": "Use Tools", 34 | "context_threshold": "Context Threshold", 35 | "context_truncate_strategy": "Context truncation strategy when exceeded threshold" 36 | } 37 | } 38 | } 39 | }, 40 | "services": { 41 | "query_image": { 42 | "name": "Query image", 43 | "description": "Take in images and answer questions about them", 44 | "fields": { 45 | "config_entry": { 46 | "name": "Config Entry", 47 | "description": "The config entry to use for this service" 48 | }, 49 | "model": { 50 | "name": "Model", 51 | "description": "The model", 52 | "example": "gpt-4-vision-preview" 53 | }, 54 | "prompt": { 55 | "name": "Prompt", 56 | "description": "The text to ask about image", 57 | "example": "What’s in this image?" 58 | }, 59 | "images": { 60 | "name": "Images", 61 | "description": "A list of images that would be asked", 62 | "example": "{\"url\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg\"}" 63 | }, 64 | "max_tokens": { 65 | "name": "Max Tokens", 66 | "description": "The maximum tokens", 67 | "example": "300" 68 | } 69 | } 70 | } 71 | } 72 | } -------------------------------------------------------------------------------- /custom_components/extended_openai_conversation/translations/ko.json: -------------------------------------------------------------------------------- 1 | { 2 | "config": { 3 | "error": { 4 | "cannot_connect": "Failed to connect", 5 | "invalid_auth": "Invalid authentication", 6 | "unknown": "Unexpected error" 7 | }, 8 | "step": { 9 | "user": { 10 | "data": { 11 | "name": "Name", 12 | "api_key": "API Key", 13 | "base_url": "Base Url", 14 | "api_version": "Api Version", 15 | "organization": "Organization", 16 | "skip_authentication": "Skip Authentication" 17 | } 18 | } 19 | } 20 | }, 21 | "options": { 22 | "step": { 23 | "init": { 24 | "data": { 25 | "max_tokens": "Maximum tokens to return in response", 26 | "model": "Completion Model", 27 | "prompt": "Prompt Template", 28 | "temperature": "Temperature", 29 | "top_p": "Top P", 30 | "max_function_calls_per_conversation": "Maximum function calls per conversation", 31 | "functions": "Functions", 32 | "attach_username": "Attach Username to Message", 33 | "use_tools": "Use Tools", 34 | "context_threshold": "Context Threshold", 35 | "context_truncate_strategy": "Context truncation strategy when exceeded threshold" 36 | } 37 | } 38 | } 39 | }, 40 | "services": { 41 | "query_image": { 42 | "name": "Query image", 43 | "description": "Take in images and answer questions about them", 44 | "fields": { 45 | "config_entry": { 46 | "name": "Config Entry", 47 | "description": "The config entry to use for this service" 48 | }, 49 | "model": { 50 | "name": "Model", 51 | "description": "The model", 52 | "example": "gpt-4-vision-preview" 53 | }, 54 | "prompt": { 55 | "name": "Prompt", 56 | "description": "The text to ask about image", 57 | "example": "What’s in this image?" 58 | }, 59 | "images": { 60 | "name": "Images", 61 | "description": "A list of images that would be asked", 62 | "example": "{\"url\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg\"}" 63 | }, 64 | "max_tokens": { 65 | "name": "Max Tokens", 66 | "description": "The maximum tokens", 67 | "example": "300" 68 | } 69 | } 70 | } 71 | } 72 | } -------------------------------------------------------------------------------- /custom_components/extended_openai_conversation/translations/pl.json: -------------------------------------------------------------------------------- 1 | { 2 | "config": { 3 | "error": { 4 | "cannot_connect": "Nie udało się połączyć", 5 | "invalid_auth": "Błąd authentykacji", 6 | "unknown": "Nieznany błąd" 7 | }, 8 | "step": { 9 | "user": { 10 | "data": { 11 | "name": "Imię", 12 | "api_key": "Klucz API", 13 | "base_url": "Bazowy URL", 14 | "api_version": "Wersja API", 15 | "organization": "Organizacja", 16 | "skip_authentication": "Pomiń authentykację" 17 | } 18 | } 19 | } 20 | }, 21 | "options": { 22 | "step": { 23 | "init": { 24 | "data": { 25 | "max_tokens": "Maksymalna liczba tokenów w odpowiedzi", 26 | "model": "Model", 27 | "prompt": "Prompt", 28 | "temperature": "Temperatura", 29 | "top_p": "Top P", 30 | "max_function_calls_per_conversation": "Maksymalna liczba wywołań funkcji na rozmowę", 31 | "functions": "Funkcje", 32 | "attach_username": "Dodaj nazwę użytkownika do wiadomości", 33 | "use_tools": "Używaj narzędzi", 34 | "context_threshold": "Limit kontekstu", 35 | "context_truncate_strategy": "Co zrobić z kontekstem, gdy przekroczony limit" 36 | } 37 | } 38 | } 39 | }, 40 | "services": { 41 | "query_image": { 42 | "name": "Query image", 43 | "description": "Take in images and answer questions about them", 44 | "fields": { 45 | "config_entry": { 46 | "name": "Config Entry", 47 | "description": "The config entry to use for this service" 48 | }, 49 | "model": { 50 | "name": "Model", 51 | "description": "The model", 52 | "example": "gpt-4-vision-preview" 53 | }, 54 | "prompt": { 55 | "name": "Prompt", 56 | "description": "The text to ask about image", 57 | "example": "What’s in this image?" 58 | }, 59 | "images": { 60 | "name": "Images", 61 | "description": "A list of images that would be asked", 62 | "example": "{\"url\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg\"}" 63 | }, 64 | "max_tokens": { 65 | "name": "Max Tokens", 66 | "description": "The maximum tokens", 67 | "example": "300" 68 | } 69 | } 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /custom_components/extended_openai_conversation/translations/hu.json: -------------------------------------------------------------------------------- 1 | { 2 | "config": { 3 | "error": { 4 | "cannot_connect": "Nem sikerült csatlakozni", 5 | "invalid_auth": "Azonosítás sikertelen", 6 | "unknown": "Váratlan hiba" 7 | }, 8 | "step": { 9 | "user": { 10 | "data": { 11 | "name": "Név", 12 | "api_key": "API Kulcs", 13 | "base_url": "Base Url", 14 | "api_version": "API Verzió", 15 | "organization": "Organization", 16 | "skip_authentication": "Azonosítás átugrása" 17 | } 18 | } 19 | } 20 | }, 21 | "options": { 22 | "step": { 23 | "init": { 24 | "data": { 25 | "max_tokens": "A válaszként viszaküldhető maximum tokenek száma", 26 | "model": "Model", 27 | "prompt": "Kiindulási szöveg sablon", 28 | "temperature": "Hőmérséklet", 29 | "top_p": "Top P", 30 | "max_function_calls_per_conversation": "Beszélgetésenkénti maximum funkcióhívások száma", 31 | "functions": "Funkciók", 32 | "attach_username": "Felhasználónév hozzácsatolása az üzenethez", 33 | "use_tools": "Use Tools", 34 | "context_threshold": "Context Threshold", 35 | "context_truncate_strategy": "Context truncation strategy when exceeded threshold" 36 | } 37 | } 38 | } 39 | }, 40 | "services": { 41 | "query_image": { 42 | "name": "Query image", 43 | "description": "Take in images and answer questions about them", 44 | "fields": { 45 | "config_entry": { 46 | "name": "Config Entry", 47 | "description": "The config entry to use for this service" 48 | }, 49 | "model": { 50 | "name": "Model", 51 | "description": "The model", 52 | "example": "gpt-4-vision-preview" 53 | }, 54 | "prompt": { 55 | "name": "Prompt", 56 | "description": "The text to ask about image", 57 | "example": "What’s in this image?" 58 | }, 59 | "images": { 60 | "name": "Images", 61 | "description": "A list of images that would be asked", 62 | "example": "{\"url\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg\"}" 63 | }, 64 | "max_tokens": { 65 | "name": "Max Tokens", 66 | "description": "The maximum tokens", 67 | "example": "300" 68 | } 69 | } 70 | } 71 | } 72 | } -------------------------------------------------------------------------------- /custom_components/extended_openai_conversation/translations/fr.json: -------------------------------------------------------------------------------- 1 | { 2 | "config": { 3 | "error": { 4 | "cannot_connect": "Échec de la connexion", 5 | "invalid_auth": "Authentification invalide", 6 | "unknown": "Erreur inconnue" 7 | }, 8 | "step": { 9 | "user": { 10 | "data": { 11 | "name": "Nom", 12 | "api_key": "Clé d'API", 13 | "base_url": "Base de l'URL", 14 | "api_version": "Version de l'API", 15 | "organization": "Organization", 16 | "skip_authentication": "Ignorer l'authentification" 17 | } 18 | } 19 | } 20 | }, 21 | "options": { 22 | "step": { 23 | "init": { 24 | "data": { 25 | "max_tokens": "Nombre maximum de jetons facturé pour la réponse", 26 | "model": "Modèle d'IA", 27 | "prompt": "Prompt", 28 | "temperature": "Température", 29 | "top_p": "Top P", 30 | "max_function_calls_per_conversation": "Nombre maximal d'appels de fonction par conversation", 31 | "functions": "Fonctions", 32 | "attach_username": "Joindre le nom d'utilisateur au message", 33 | "use_tools": "Use Tools", 34 | "context_threshold": "Context Threshold", 35 | "context_truncate_strategy": "Context truncation strategy when exceeded threshold" 36 | } 37 | } 38 | } 39 | }, 40 | "services": { 41 | "query_image": { 42 | "name": "Query image", 43 | "description": "Take in images and answer questions about them", 44 | "fields": { 45 | "config_entry": { 46 | "name": "Config Entry", 47 | "description": "The config entry to use for this service" 48 | }, 49 | "model": { 50 | "name": "Model", 51 | "description": "The model", 52 | "example": "gpt-4-vision-preview" 53 | }, 54 | "prompt": { 55 | "name": "Prompt", 56 | "description": "The text to ask about image", 57 | "example": "What’s in this image?" 58 | }, 59 | "images": { 60 | "name": "Images", 61 | "description": "A list of images that would be asked", 62 | "example": "{\"url\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg\"}" 63 | }, 64 | "max_tokens": { 65 | "name": "Max Tokens", 66 | "description": "The maximum tokens", 67 | "example": "300" 68 | } 69 | } 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /custom_components/extended_openai_conversation/translations/nl.json: -------------------------------------------------------------------------------- 1 | { 2 | "config": { 3 | "error": { 4 | "cannot_connect": "Verbinden mislukt", 5 | "invalid_auth": "Ongeldige authenticatie", 6 | "unknown": "Onverwachte fout" 7 | }, 8 | "step": { 9 | "user": { 10 | "data": { 11 | "name": "Naam", 12 | "api_key": "API Sleutel", 13 | "base_url": "Basis URL", 14 | "api_version": "API Version", 15 | "organization": "Organization", 16 | "skip_authentication": "Authenticatie overslaan" 17 | } 18 | } 19 | } 20 | }, 21 | "options": { 22 | "step": { 23 | "init": { 24 | "data": { 25 | "max_tokens": "Maximale aantal tokens dat mag worden gegenereerd", 26 | "model": "Completion Model", 27 | "prompt": "Prompt Sjabloon", 28 | "temperature": "Temperatuur", 29 | "top_p": "Top P", 30 | "max_function_calls_per_conversation": "Maximale keren functies mogen worden aangeroepen per conversatie", 31 | "functions": "Functies", 32 | "attach_username": "Gebruikersnaam aan bericht toevoegen", 33 | "use_tools": "Use Tools", 34 | "context_threshold": "Context Threshold", 35 | "context_truncate_strategy": "Context truncation strategy when exceeded threshold" 36 | } 37 | } 38 | } 39 | }, 40 | "services": { 41 | "query_image": { 42 | "name": "Query image", 43 | "description": "Take in images and answer questions about them", 44 | "fields": { 45 | "config_entry": { 46 | "name": "Config Entry", 47 | "description": "The config entry to use for this service" 48 | }, 49 | "model": { 50 | "name": "Model", 51 | "description": "The model", 52 | "example": "gpt-4-vision-preview" 53 | }, 54 | "prompt": { 55 | "name": "Prompt", 56 | "description": "The text to ask about image", 57 | "example": "What’s in this image?" 58 | }, 59 | "images": { 60 | "name": "Images", 61 | "description": "A list of images that would be asked", 62 | "example": "{\"url\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg\"}" 63 | }, 64 | "max_tokens": { 65 | "name": "Max Tokens", 66 | "description": "The maximum tokens", 67 | "example": "300" 68 | } 69 | } 70 | } 71 | } 72 | } -------------------------------------------------------------------------------- /custom_components/extended_openai_conversation/translations/de.json: -------------------------------------------------------------------------------- 1 | { 2 | "config": { 3 | "error": { 4 | "cannot_connect": "Fehler beim Verbindungsaufbau", 5 | "invalid_auth": "Authentifizierung fehlgeschlagen", 6 | "unknown": "Unbekannter Fehler" 7 | }, 8 | "step": { 9 | "user": { 10 | "data": { 11 | "name": "Name", 12 | "api_key": "API Key", 13 | "base_url": "Base Url", 14 | "api_version": "Api Version", 15 | "organization": "Organization", 16 | "skip_authentication": "Authentifizierung überspringen" 17 | } 18 | } 19 | } 20 | }, 21 | "options": { 22 | "step": { 23 | "init": { 24 | "data": { 25 | "max_tokens": "Maximale Anzahl an Tokens, die in einer Antwort zurückgegeben werden", 26 | "model": "Completion Model", 27 | "prompt": "Prompt Vorlage", 28 | "temperature": "Temperatur", 29 | "top_p": "Top P", 30 | "max_function_calls_per_conversation": "Maximale Anzahl an Funktionsaufrufen pro Konversation", 31 | "functions": "Funktionen", 32 | "attach_username": "Benutzernamen mitgeben", 33 | "use_tools": "Use Tools", 34 | "context_threshold": "Context Threshold", 35 | "context_truncate_strategy": "Context truncation strategy when exceeded threshold" 36 | } 37 | } 38 | } 39 | }, 40 | "services": { 41 | "query_image": { 42 | "name": "Query image", 43 | "description": "Take in images and answer questions about them", 44 | "fields": { 45 | "config_entry": { 46 | "name": "Config Entry", 47 | "description": "The config entry to use for this service" 48 | }, 49 | "model": { 50 | "name": "Model", 51 | "description": "The model", 52 | "example": "gpt-4-vision-preview" 53 | }, 54 | "prompt": { 55 | "name": "Prompt", 56 | "description": "The text to ask about image", 57 | "example": "What’s in this image?" 58 | }, 59 | "images": { 60 | "name": "Images", 61 | "description": "A list of images that would be asked", 62 | "example": "{\"url\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg\"}" 63 | }, 64 | "max_tokens": { 65 | "name": "Max Tokens", 66 | "description": "The maximum tokens", 67 | "example": "300" 68 | } 69 | } 70 | } 71 | } 72 | } -------------------------------------------------------------------------------- /custom_components/extended_openai_conversation/translations/it.json: -------------------------------------------------------------------------------- 1 | { 2 | "config": { 3 | "error": { 4 | "cannot_connect": "Connessione non riuscita", 5 | "invalid_auth": "Autenticazione non valida", 6 | "unknown": "Errore imprevisto" 7 | }, 8 | "step": { 9 | "user": { 10 | "data": { 11 | "name": "Nome", 12 | "api_key": "Chiave API", 13 | "base_url": "URL di base", 14 | "api_version": "Versione API", 15 | "skip_authentication": "Ignora autenticazione" 16 | } 17 | } 18 | } 19 | }, 20 | "options": { 21 | "step": { 22 | "init": { 23 | "data": { 24 | "max_tokens": "Numero massimo token da restituire nella risposta", 25 | "model": "Modello di completamento", 26 | "prompt": "Modello di prompt", 27 | "temperature": "Temperatura", 28 | "top_p": "Top P", 29 | "max_function_calls_per_conversation": "Numero massimo di chiamate di funzioni per conversazione", 30 | "functions": "Funzioni", 31 | "attach_username": "Allega nome utente al messaggio", 32 | "use_tools": "Utilizza strumenti", 33 | "context_threshold": "Soglia di contesto", 34 | "context_truncate_strategy": "Strategia di troncamento del contesto quando superata la soglia" 35 | } 36 | } 37 | } 38 | }, 39 | "services": { 40 | "query_image": { 41 | "name": "Interrogazione immagine", 42 | "description": "Prendi le immagini e rispondi alle domande su di esse", 43 | "fields": { 44 | "config_entry": { 45 | "name": "Voce di configurazione", 46 | "description": "La voce di configurazione da utilizzare per questo servizio" 47 | }, 48 | "model": { 49 | "name": "Modello", 50 | "description": "Il modello", 51 | "example": "gpt-4-vision-preview" 52 | }, 53 | "prompt": { 54 | "name": "Prompt", 55 | "description": "Il testo da chiedere riguardo all'immagine", 56 | "example": "Cosa c'è in questa immagine?" 57 | }, 58 | "images": { 59 | "name": "Immagini", 60 | "description": "Un elenco delle immagini richieste", 61 | "example": "{\"url\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg\"}" 62 | }, 63 | "max_tokens": { 64 | "name": "Max Token", 65 | "description": "I token massimi", 66 | "example": "300" 67 | } 68 | } 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /custom_components/extended_openai_conversation/translations/pt.json: -------------------------------------------------------------------------------- 1 | { 2 | "config": { 3 | "error": { 4 | "cannot_connect": "Não é possível ligar", 5 | "invalid_auth": "Autenticação inválida", 6 | "unknown": "Erro desconhecido" 7 | }, 8 | "step": { 9 | "user": { 10 | "data": { 11 | "name": "Nome", 12 | "api_key": "Chave API", 13 | "base_url": "Base Url", 14 | "api_version": "Versão da API", 15 | "organization": "Organização", 16 | "skip_authentication": "Saltar autenticação" 17 | } 18 | } 19 | } 20 | }, 21 | "options": { 22 | "step": { 23 | "init": { 24 | "data": { 25 | "max_tokens": "Número máximo de tokens da resposta", 26 | "model": "Modelo da Conclusão", 27 | "prompt": "Template do Prompt", 28 | "temperature": "Temperatura", 29 | "top_p": "Top P", 30 | "max_function_calls_per_conversation": "Quantidade máxima de chamadas por conversação", 31 | "functions": "Funções", 32 | "attach_username": "Anexar nome do usuário na mensagem", 33 | "use_tools": "Use ferramentas", 34 | "context_threshold": "Limite do contexto", 35 | "context_truncate_strategy": "Estratégia de limite de contexto quando o limite é excedido" 36 | } 37 | } 38 | } 39 | }, 40 | "services": { 41 | "query_image": { 42 | "name": "Consultar imagem", 43 | "description": "Receber imagens e responda perguntas sobre elas", 44 | "fields": { 45 | "config_entry": { 46 | "name": "Registro de configuração", 47 | "description": "O registo de configuração para utilizar neste serviço" 48 | }, 49 | "model": { 50 | "name": "Modelo", 51 | "description": "Especificar modelo", 52 | "example": "gpt-4-vision-preview" 53 | }, 54 | "prompt": { 55 | "name": "Prompt", 56 | "description": "O texto para fazer a pergunta sobre a imagem", 57 | "example": "O que tem nesta imagem?" 58 | }, 59 | "images": { 60 | "name": "Imagens", 61 | "description": "Uma lista de imagens que serão analisadas", 62 | "example": "{\"url\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg\"}" 63 | }, 64 | "max_tokens": { 65 | "name": "Max Tokens", 66 | "description": "Quantidade máxima de tokens", 67 | "example": "300" 68 | } 69 | } 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /custom_components/extended_openai_conversation/translations/el.json: -------------------------------------------------------------------------------- 1 | { 2 | "config": { 3 | "error": { 4 | "cannot_connect": "Αποτυχία σύνδεσης", 5 | "invalid_auth": "Μη έγκυρος έλεγχος ταυτότητας", 6 | "unknown": "Απροσδόκητο σφάλμα" 7 | }, 8 | "step": { 9 | "user": { 10 | "data": { 11 | "name": "Όνομα", 12 | "api_key": "Κλειδί API", 13 | "base_url": "Βασική Διεύθυνση", 14 | "api_version": "Έκδοση API", 15 | "organization": "Οργανισμός", 16 | "skip_authentication": "Παράλειψη Ελέγχου Ταυτότητας" 17 | } 18 | } 19 | } 20 | }, 21 | "options": { 22 | "step": { 23 | "init": { 24 | "data": { 25 | "max_tokens": "Μέγιστα tokens προς επιστροφή στην απάντηση", 26 | "model": "Μοντέλο Ολοκλήρωσης", 27 | "prompt": "Πρότυπο Προτροπής", 28 | "temperature": "Θερμοκρασία", 29 | "top_p": "Top P", 30 | "max_function_calls_per_conversation": "Μέγιστες κλήσεις συναρτήσεων ανά συνομιλία", 31 | "functions": "Συναρτήσεις", 32 | "attach_username": "Επισύναψη Ονόματος Χρήστη στο Μήνυμα", 33 | "use_tools": "Χρήση Εργαλείων", 34 | "context_threshold": "Όριο Περιεχομένου", 35 | "context_truncate_strategy": "Στρατηγική περικοπής περιεχομένου όταν ξεπεραστεί το όριο" 36 | } 37 | } 38 | } 39 | }, 40 | "services": { 41 | "query_image": { 42 | "name": "Ερώτημα εικόνας", 43 | "description": "Λάβετε εικόνες και απαντήστε ερωτήσεις για αυτές", 44 | "fields": { 45 | "config_entry": { 46 | "name": "Καταχώρηση Διαμόρφωσης", 47 | "description": "Η καταχώρηση διαμόρφωσης που θα χρησιμοποιηθεί για αυτή την υπηρεσία" 48 | }, 49 | "model": { 50 | "name": "Μοντέλο", 51 | "description": "Το μοντέλο", 52 | "example": "gpt-4-vision-preview" 53 | }, 54 | "prompt": { 55 | "name": "Προτροπή", 56 | "description": "Το κείμενο για να ρωτήσετε σχετικά με την εικόνα", 57 | "example": "Τι υπάρχει σε αυτή την εικόνα;" 58 | }, 59 | "images": { 60 | "name": "Εικόνες", 61 | "description": "Μια λίστα εικόνων που θα ερωτηθούν", 62 | "example": "{\"url\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg\"}" 63 | }, 64 | "max_tokens": { 65 | "name": "Μέγιστα Tokens", 66 | "description": "Τα μέγιστα tokens", 67 | "example": "300" 68 | } 69 | } 70 | } 71 | } 72 | } -------------------------------------------------------------------------------- /custom_components/extended_openai_conversation/translations/pt-BR.json: -------------------------------------------------------------------------------- 1 | { 2 | "config": { 3 | "error": { 4 | "cannot_connect": "Não é possível conectar", 5 | "invalid_auth": "Autenticação inválida", 6 | "unknown": "Erro desconhecido" 7 | }, 8 | "step": { 9 | "user": { 10 | "data": { 11 | "name": "Nome", 12 | "api_key": "Chave API", 13 | "base_url": "Base Url", 14 | "api_version": "Versão da API", 15 | "organization": "Organização", 16 | "skip_authentication": "Pular autenticação" 17 | } 18 | } 19 | } 20 | }, 21 | "options": { 22 | "step": { 23 | "init": { 24 | "data": { 25 | "max_tokens": "Número máximo de tokens da resposta", 26 | "model": "Modelo da Conclusão", 27 | "prompt": "Template do Prompt", 28 | "temperature": "Temperatura", 29 | "top_p": "Top P", 30 | "max_function_calls_per_conversation": "Quantidade máxima de chamadas por conversação", 31 | "functions": "Funções", 32 | "attach_username": "Anexar nome do usuário na mensagem", 33 | "use_tools": "Use ferramentas", 34 | "context_threshold": "Limite do contexto", 35 | "context_truncate_strategy": "Estratégia de truncamento de contexto quando o limite é excedido" 36 | } 37 | } 38 | } 39 | }, 40 | "services": { 41 | "query_image": { 42 | "name": "Consultar imagem", 43 | "description": "Receba imagens e responda perguntas sobre elas", 44 | "fields": { 45 | "config_entry": { 46 | "name": "Registro de configuração", 47 | "description": "O registro de configuração para utilizar neste serviço" 48 | }, 49 | "model": { 50 | "name": "Modelo", 51 | "description": "Especificar modelo", 52 | "example": "gpt-4-vision-preview" 53 | }, 54 | "prompt": { 55 | "name": "Prompt", 56 | "description": "O texto para fazer a pergunta sobre a imagem", 57 | "example": "O que tem nesta imagem?" 58 | }, 59 | "images": { 60 | "name": "Imagens", 61 | "description": "Uma lista de imagens que serão analisadas", 62 | "example": "{\"url\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg\"}" 63 | }, 64 | "max_tokens": { 65 | "name": "Max Tokens", 66 | "description": "Quantidade máxima de tokens", 67 | "example": "300" 68 | } 69 | } 70 | } 71 | } 72 | } -------------------------------------------------------------------------------- /examples/component_function/grocy/README.md: -------------------------------------------------------------------------------- 1 | ## Requirement 2 | Assume using [grocy](https://github.com/custom-components/grocy) 3 | 4 | ## Function 5 | ### get_today_chore 6 | ```yaml 7 | - spec: 8 | name: get_today_chore 9 | description: Use this function to retrieve a list of chores due for today or before. 10 | parameters: 11 | type: object 12 | properties: {} 13 | function: 14 | type: template 15 | value_template: >- 16 | {% set now_date = now().date() %} 17 | {% set chores_data = state_attr('sensor.grocy_chores', 'chores') %} 18 | 19 | {% set overdue_chores = chores_data | selectattr('next_estimated_execution_time', 'string') | map(attribute='next_estimated_execution_time') | map('regex_replace', 'T\\d{2}:\\d{2}:\\d{2}', '') | select('lt', now_date | string) | list %} 20 | {% set chores_due_today = chores_data | selectattr('next_estimated_execution_time', 'string') | map(attribute='next_estimated_execution_time') | map('regex_replace', 'T\\d{2}:\\d{2}:\\d{2}', '') | select('equalto', now_date | string) | list %} 21 | {% set combined_chores = overdue_chores + chores_due_today %} 22 | 23 | ```csv 24 | name,last_tracked_time,next_estimated_execution_time 25 | {% for chore in combined_chores %} 26 | {{ chore['name'] | replace(",", " ") }},{{ chore['last_tracked_time'] }},{{ chore['next_estimated_execution_time'] }} 27 | {% endfor -%} 28 | ``` 29 | ``` 30 | 31 | ### execute_chore 32 | 33 | ```yaml 34 | - spec: 35 | name: execute_chore 36 | description: Use this function to execute a chore in Home Assistant. 37 | parameters: 38 | type: object 39 | properties: 40 | chore_id: 41 | type: string 42 | description: The ID of the chore to be executed. 43 | required: 44 | - chore_id 45 | function: 46 | type: script 47 | sequence: 48 | - service: script.execute_chore 49 | data: 50 | chore_id: "{{ chore_id }}" 51 | ``` 52 | 53 | ```yaml 54 | script: 55 | execute_chore: 56 | alias: "Execute Chore" 57 | sequence: 58 | - service: grocy.execute_chore 59 | data: 60 | chore_id: "{{ chore_id }}" 61 | ``` 62 | 63 | ### get_inventory_stock 64 | ```yaml 65 | - spec: 66 | name: get_inventory_stock 67 | description: Use this function to retrieve the inventory entries data. 68 | parameters: 69 | type: object 70 | properties: {} 71 | function: 72 | type: template 73 | value_template: >- 74 | {% set data = states['sensor.grocy_stock'].attributes['products'] | list %} 75 | ```csv 76 | name,id,product_group_id,available_amount,amount_aggregated,amount_opened,amount_opened_aggregated,is_aggregated_amount,best_before_date 77 | {% for product in data -%} 78 | {{ product['name'] }},{{ product['id'] }},{{ product['product_group_id'] }}, 79 | {%- if product['available_amount'] == 0 -%} 80 | {{ product['amount_aggregated'] }}, 81 | {%- else -%} 82 | {{ product['available_amount'] }}, 83 | {%- endif -%} 84 | {{ product['amount_aggregated'] }},{{ product['amount_opened'] }},{{ product['amount_opened_aggregated'] }},{{ product['is_aggregated_amount'] }},{{ product['best_before_date'] }} 85 | {% endfor -%} 86 | ``` 87 | ``` -------------------------------------------------------------------------------- /examples/prompt/area/README.md: -------------------------------------------------------------------------------- 1 | ## Objective 2 | - Let gpt know about area information, so [execute_services](https://github.com/jekalmin/extended_openai_conversation/tree/v1.0.2/examples/function/area#execute_services) can be called using `area_id` 3 | - Use area awareness feature like [Year of Voice Chapter 5](https://www.home-assistant.io/blog/2023/12/13/year-of-the-voice-chapter-5/#area-awareness) 4 | 5 | ## How to use area awareness? 6 | 1. Assign area to your ESP-S3-BOX or Atom echo. 7 | 2. Copy and paste prompt below. 8 | 3. Ask "turn on light", "turn off light" 9 | 10 | 11 | ## Prompt 12 | 13 | ### 1. List areas and entities separately 14 | ````yaml 15 | I want you to act as smart home manager of Home Assistant. 16 | I will provide information of smart home along with a question, you will truthfully make correction or answer using information provided in one sentence in everyday language. 17 | 18 | Current Time: {{now()}} 19 | Current Area: {{area_id(current_device_id)}} 20 | 21 | Available Devices: 22 | ```csv 23 | entity_id,name,state,area_id,aliases 24 | {% for entity in exposed_entities -%} 25 | {{ entity.entity_id }},{{ entity.name }},{{ entity.state }},{{area_id(entity.entity_id)}},{{entity.aliases | join('/')}} 26 | {% endfor -%} 27 | ``` 28 | 29 | Areas: 30 | ```csv 31 | area_id,name 32 | {% for area_id in areas() -%} 33 | {{area_id}},{{area_name(area_id)}} 34 | {% endfor -%} 35 | ``` 36 | 37 | 38 | The current state of devices is provided in available devices. 39 | Use execute_services function only for requested action, not for current states. 40 | Do not execute service without user's confirmation. 41 | Do not restate or appreciate what user says, rather make a quick inquiry. 42 | Make decisions based on current area first. 43 | ```` 44 | 45 | ### 2. Categorize entities by areas 46 | ````yaml 47 | I want you to act as smart home manager of Home Assistant. 48 | I will provide information of smart home along with a question, you will truthfully make correction or answer using information provided in one sentence in everyday language. 49 | 50 | Current Time: {{now()}} 51 | Current Area: {{area_name(current_device_id)}} 52 | 53 | An overview of the areas and the available devices: 54 | {%- set area_entities = namespace(mapping={}) %} 55 | {%- for entity in exposed_entities %} 56 | {%- set current_area_id = area_id(entity.entity_id) or "etc" %} 57 | {%- set entities = (area_entities.mapping.get(current_area_id) or []) + [entity] %} 58 | {%- set area_entities.mapping = dict(area_entities.mapping, **{current_area_id: entities}) -%} 59 | {%- endfor %} 60 | 61 | {%- for current_area_id, entities in area_entities.mapping.items() %} 62 | 63 | {%- if current_area_id == "etc" %} 64 | Etc: 65 | {%- else %} 66 | {{area_name(current_area_id)}}({{current_area_id}}): 67 | {%- endif %} 68 | ```csv 69 | entity_id,name,state,aliases 70 | {%- for entity in entities %} 71 | {{ entity.entity_id }},{{ entity.name }},{{ entity.state }},{{entity.aliases | join('/')}} 72 | {%- endfor %} 73 | ``` 74 | {%- endfor %} 75 | 76 | The current state of devices is provided in available devices. 77 | Use execute_services function only for requested action, not for current states. 78 | Do not execute service without user's confirmation. 79 | Do not restate or appreciate what user says, rather make a quick inquiry. 80 | Make decisions based on current area first. 81 | ```` -------------------------------------------------------------------------------- /examples/function/camera_image_query/README.md: -------------------------------------------------------------------------------- 1 | ## Objective 2 | 3 | This example provides three functions: 4 | 5 | - query_image: generic query image 6 | - camera_snapshot: generic take camera snapshot and store in /media 7 | - camera_query: a combined take camera snapshot and query image 8 | 9 | Use the first two functions instead of the combined camera_query 10 | function for more flexiblity as they could be used independantly. You 11 | may need to grant file system access via home assistant configuration 12 | of allowlist_external_dirs to /media or your choosen directory. 13 | 14 | ## Function 15 | 16 | ### query_image 17 | ```yaml 18 | - spec: 19 | name: query_image 20 | description: Get description of items or scene from an image 21 | parameters: 22 | type: object 23 | properties: 24 | url: 25 | type: string 26 | description: path or url for image 27 | required: 28 | - url 29 | function: 30 | type: composite 31 | sequence: 32 | - type: script 33 | sequence: 34 | - service: extended_openai_conversation.query_image 35 | data: 36 | model: gpt-4-vision-preview 37 | prompt: What's in this image? 38 | images: 39 | - url: "{{url}}" 40 | max_tokens: 500 41 | config_entry: YOUR_CONFIG_ENTRY 42 | response_variable: _function_result 43 | response_variable: image_result 44 | - type: template 45 | value_template: "{{image_result.choices[0].message.content}}" 46 | ``` 47 | 48 | ### camera_snapshot 49 | ```yaml 50 | - spec: 51 | name: camera_snapshot 52 | description: Generate an image from a camera 53 | parameters: 54 | type: object 55 | properties: 56 | entity_id: 57 | type: string 58 | description: Camera entity 59 | filename: 60 | type: string 61 | description: full path and name of file to generate. Please name it as /media/camera_entity_latest.jpg 62 | required: 63 | - item 64 | function: 65 | type: script 66 | sequence: 67 | - service: camera.snapshot 68 | target: 69 | entity_id: "{{entity_id}}" 70 | data: 71 | filename: '{{filename}}' 72 | ``` 73 | 74 | ### camera_query 75 | ```yaml 76 | - spec: 77 | name: camera_query 78 | description: Get a description of items or scene from a camera 79 | parameters: 80 | type: object 81 | properties: 82 | entity_id: 83 | type: string 84 | description: Camera entity 85 | filename: 86 | type: string 87 | description: full path and name of file to generate. Please name it as /media/camera_entity_latest.jpg 88 | required: 89 | - item 90 | function: 91 | type: composite 92 | sequence: 93 | - type: script 94 | sequence: 95 | - service: camera.snapshot 96 | target: 97 | entity_id: "{{entity_id}}" 98 | data: 99 | filename: '{{filename}}' 100 | - service: extended_openai_conversation.query_image 101 | data: 102 | model: gpt-4-vision-preview 103 | prompt: What's in this image? 104 | images: 105 | - url: "{{filename}}" 106 | max_tokens: 500 107 | config_entry: YOUR_CONFIG_ENTRY 108 | response_variable: _function_result 109 | response_variable: image_result 110 | - type: template 111 | value_template: "{{image_result.choices[0].message.content}}" 112 | ``` 113 | -------------------------------------------------------------------------------- /custom_components/extended_openai_conversation/services.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import logging 3 | import mimetypes 4 | from pathlib import Path 5 | from urllib.parse import urlparse 6 | 7 | from openai import AsyncOpenAI 8 | from openai._exceptions import OpenAIError 9 | from openai.types.chat.chat_completion_content_part_image_param import ( 10 | ChatCompletionContentPartImageParam, 11 | ) 12 | import voluptuous as vol 13 | 14 | from homeassistant.core import ( 15 | HomeAssistant, 16 | ServiceCall, 17 | ServiceResponse, 18 | SupportsResponse, 19 | ) 20 | from homeassistant.exceptions import HomeAssistantError 21 | from homeassistant.helpers import config_validation as cv, selector 22 | from homeassistant.helpers.typing import ConfigType 23 | 24 | from .const import DOMAIN, SERVICE_QUERY_IMAGE 25 | 26 | QUERY_IMAGE_SCHEMA = vol.Schema( 27 | { 28 | vol.Required("config_entry"): selector.ConfigEntrySelector( 29 | { 30 | "integration": DOMAIN, 31 | } 32 | ), 33 | vol.Required("model", default="gpt-4-vision-preview"): cv.string, 34 | vol.Required("prompt"): cv.string, 35 | vol.Required("images"): vol.All(cv.ensure_list, [{"url": cv.string}]), 36 | vol.Optional("max_tokens", default=300): cv.positive_int, 37 | } 38 | ) 39 | 40 | _LOGGER = logging.getLogger(__package__) 41 | 42 | 43 | async def async_setup_services(hass: HomeAssistant, config: ConfigType) -> None: 44 | """Set up services for the extended openai conversation component.""" 45 | 46 | async def query_image(call: ServiceCall) -> ServiceResponse: 47 | """Query an image.""" 48 | try: 49 | model = call.data["model"] 50 | images = [ 51 | {"type": "image_url", "image_url": to_image_param(hass, image)} 52 | for image in call.data["images"] 53 | ] 54 | 55 | messages = [ 56 | { 57 | "role": "user", 58 | "content": [{"type": "text", "text": call.data["prompt"]}] + images, 59 | } 60 | ] 61 | _LOGGER.info("Prompt for %s: %s", model, messages) 62 | 63 | response = await AsyncOpenAI( 64 | api_key=hass.data[DOMAIN][call.data["config_entry"]]["api_key"] 65 | ).chat.completions.create( 66 | model=model, 67 | messages=messages, 68 | max_tokens=call.data["max_tokens"], 69 | ) 70 | response_dict = response.model_dump() 71 | _LOGGER.info("Response %s", response_dict) 72 | except OpenAIError as err: 73 | raise HomeAssistantError(f"Error generating image: {err}") from err 74 | 75 | return response_dict 76 | 77 | hass.services.async_register( 78 | DOMAIN, 79 | SERVICE_QUERY_IMAGE, 80 | query_image, 81 | schema=QUERY_IMAGE_SCHEMA, 82 | supports_response=SupportsResponse.ONLY, 83 | ) 84 | 85 | 86 | def to_image_param(hass: HomeAssistant, image) -> ChatCompletionContentPartImageParam: 87 | """Convert url to base64 encoded image if local.""" 88 | url = image["url"] 89 | 90 | if urlparse(url).scheme in cv.EXTERNAL_URL_PROTOCOL_SCHEMA_LIST: 91 | return image 92 | 93 | if not hass.config.is_allowed_path(url): 94 | raise HomeAssistantError( 95 | f"Cannot read `{url}`, no access to path; " 96 | "`allowlist_external_dirs` may need to be adjusted in " 97 | "`configuration.yaml`" 98 | ) 99 | if not Path(url).exists(): 100 | raise HomeAssistantError(f"`{url}` does not exist") 101 | mime_type, _ = mimetypes.guess_type(url) 102 | if mime_type is None or not mime_type.startswith("image"): 103 | raise HomeAssistantError(f"`{url}` is not an image") 104 | 105 | image["url"] = f"data:{mime_type};base64,{encode_image(url)}" 106 | return image 107 | 108 | 109 | def encode_image(image_path): 110 | """Convert to base64 encoded image.""" 111 | with open(image_path, "rb") as image_file: 112 | return base64.b64encode(image_file.read()).decode("utf-8") 113 | -------------------------------------------------------------------------------- /custom_components/extended_openai_conversation/const.py: -------------------------------------------------------------------------------- 1 | """Constants for the Extended OpenAI Conversation integration.""" 2 | 3 | DOMAIN = "extended_openai_conversation" 4 | DEFAULT_NAME = "Extended OpenAI Conversation" 5 | CONF_ORGANIZATION = "organization" 6 | CONF_BASE_URL = "base_url" 7 | DEFAULT_CONF_BASE_URL = "https://api.openai.com/v1" 8 | CONF_API_VERSION = "api_version" 9 | CONF_SKIP_AUTHENTICATION = "skip_authentication" 10 | DEFAULT_SKIP_AUTHENTICATION = False 11 | 12 | EVENT_AUTOMATION_REGISTERED = "automation_registered_via_extended_openai_conversation" 13 | EVENT_CONVERSATION_FINISHED = "extended_openai_conversation.conversation.finished" 14 | 15 | CONF_PROMPT = "prompt" 16 | DEFAULT_PROMPT = """I want you to act as smart home manager of Home Assistant. 17 | I will provide information of smart home along with a question, you will truthfully make correction or answer using information provided in one sentence in everyday language. 18 | 19 | Current Time: {{now()}} 20 | 21 | Available Devices: 22 | ```csv 23 | entity_id,name,state,aliases 24 | {% for entity in exposed_entities -%} 25 | {{ entity.entity_id }},{{ entity.name }},{{ entity.state }},{{entity.aliases | join('/')}} 26 | {% endfor -%} 27 | ``` 28 | 29 | The current state of devices is provided in available devices. 30 | Use execute_services function only for requested action, not for current states. 31 | Do not execute service without user's confirmation. 32 | Do not restate or appreciate what user says, rather make a quick inquiry. 33 | """ 34 | CONF_CHAT_MODEL = "chat_model" 35 | DEFAULT_CHAT_MODEL = "gpt-4o-mini" 36 | CONF_MAX_TOKENS = "max_tokens" 37 | DEFAULT_MAX_TOKENS = 150 38 | CONF_TOP_P = "top_p" 39 | DEFAULT_TOP_P = 1 40 | CONF_TEMPERATURE = "temperature" 41 | DEFAULT_TEMPERATURE = 0.5 42 | CONF_MAX_FUNCTION_CALLS_PER_CONVERSATION = "max_function_calls_per_conversation" 43 | DEFAULT_MAX_FUNCTION_CALLS_PER_CONVERSATION = 1 44 | CONF_FUNCTIONS = "functions" 45 | DEFAULT_CONF_FUNCTIONS = [ 46 | { 47 | "spec": { 48 | "name": "execute_services", 49 | "description": "Use this function to execute service of devices in Home Assistant.", 50 | "parameters": { 51 | "type": "object", 52 | "properties": { 53 | "list": { 54 | "type": "array", 55 | "items": { 56 | "type": "object", 57 | "properties": { 58 | "domain": { 59 | "type": "string", 60 | "description": "The domain of the service", 61 | }, 62 | "service": { 63 | "type": "string", 64 | "description": "The service to be called", 65 | }, 66 | "service_data": { 67 | "type": "object", 68 | "description": "The service data object to indicate what to control.", 69 | "properties": { 70 | "entity_id": { 71 | "type": "string", 72 | "description": "The entity_id retrieved from available devices. It must start with domain, followed by dot character.", 73 | } 74 | }, 75 | "required": ["entity_id"], 76 | }, 77 | }, 78 | "required": ["domain", "service", "service_data"], 79 | }, 80 | } 81 | }, 82 | }, 83 | }, 84 | "function": {"type": "native", "name": "execute_service"}, 85 | } 86 | ] 87 | CONF_ATTACH_USERNAME = "attach_username" 88 | DEFAULT_ATTACH_USERNAME = False 89 | CONF_USE_TOOLS = "use_tools" 90 | DEFAULT_USE_TOOLS = False 91 | CONF_CONTEXT_THRESHOLD = "context_threshold" 92 | DEFAULT_CONTEXT_THRESHOLD = 13000 93 | CONTEXT_TRUNCATE_STRATEGIES = [{"key": "clear", "label": "Clear All Messages"}] 94 | CONF_CONTEXT_TRUNCATE_STRATEGY = "context_truncate_strategy" 95 | DEFAULT_CONTEXT_TRUNCATE_STRATEGY = CONTEXT_TRUNCATE_STRATEGIES[0]["key"] 96 | 97 | SERVICE_QUERY_IMAGE = "query_image" 98 | 99 | CONF_PAYLOAD_TEMPLATE = "payload_template" 100 | -------------------------------------------------------------------------------- /custom_components/extended_openai_conversation/exceptions.py: -------------------------------------------------------------------------------- 1 | """The exceptions used by Extended OpenAI Conversation.""" 2 | from homeassistant.exceptions import HomeAssistantError 3 | 4 | 5 | class EntityNotFound(HomeAssistantError): 6 | """When referenced entity not found.""" 7 | 8 | def __init__(self, entity_id: str) -> None: 9 | """Initialize error.""" 10 | super().__init__(self, f"entity {entity_id} not found") 11 | self.entity_id = entity_id 12 | 13 | def __str__(self) -> str: 14 | """Return string representation.""" 15 | return f"Unable to find entity {self.entity_id}" 16 | 17 | 18 | class EntityNotExposed(HomeAssistantError): 19 | """When referenced entity not exposed.""" 20 | 21 | def __init__(self, entity_id: str) -> None: 22 | """Initialize error.""" 23 | super().__init__(self, f"entity {entity_id} not exposed") 24 | self.entity_id = entity_id 25 | 26 | def __str__(self) -> str: 27 | """Return string representation.""" 28 | return f"entity {self.entity_id} is not exposed" 29 | 30 | 31 | class CallServiceError(HomeAssistantError): 32 | """Error during service calling""" 33 | 34 | def __init__(self, domain: str, service: str, data: object) -> None: 35 | """Initialize error.""" 36 | super().__init__( 37 | self, 38 | f"unable to call service {domain}.{service} with data {data}. One of 'entity_id', 'area_id', or 'device_id' is required", 39 | ) 40 | self.domain = domain 41 | self.service = service 42 | self.data = data 43 | 44 | def __str__(self) -> str: 45 | """Return string representation.""" 46 | return f"unable to call service {self.domain}.{self.service} with data {self.data}. One of 'entity_id', 'area_id', or 'device_id' is required" 47 | 48 | 49 | class FunctionNotFound(HomeAssistantError): 50 | """When referenced function not found.""" 51 | 52 | def __init__(self, function: str) -> None: 53 | """Initialize error.""" 54 | super().__init__(self, f"function '{function}' does not exist") 55 | self.function = function 56 | 57 | def __str__(self) -> str: 58 | """Return string representation.""" 59 | return f"function '{self.function}' does not exist" 60 | 61 | 62 | class NativeNotFound(HomeAssistantError): 63 | """When native function not found.""" 64 | 65 | def __init__(self, name: str) -> None: 66 | """Initialize error.""" 67 | super().__init__(self, f"native function '{name}' does not exist") 68 | self.name = name 69 | 70 | def __str__(self) -> str: 71 | """Return string representation.""" 72 | return f"native function '{self.name}' does not exist" 73 | 74 | 75 | class FunctionLoadFailed(HomeAssistantError): 76 | """When function load failed.""" 77 | 78 | def __init__(self) -> None: 79 | """Initialize error.""" 80 | super().__init__( 81 | self, 82 | "failed to load functions. Verify functions are valid in a yaml format", 83 | ) 84 | 85 | def __str__(self) -> str: 86 | """Return string representation.""" 87 | return "failed to load functions. Verify functions are valid in a yaml format" 88 | 89 | 90 | class ParseArgumentsFailed(HomeAssistantError): 91 | """When parse arguments failed.""" 92 | 93 | def __init__(self, arguments: str) -> None: 94 | """Initialize error.""" 95 | super().__init__( 96 | self, 97 | f"failed to parse arguments `{arguments}`. Increase maximum token to avoid the issue.", 98 | ) 99 | self.arguments = arguments 100 | 101 | def __str__(self) -> str: 102 | """Return string representation.""" 103 | return f"failed to parse arguments `{self.arguments}`. Increase maximum token to avoid the issue." 104 | 105 | 106 | class TokenLengthExceededError(HomeAssistantError): 107 | """When openai return 'length' as 'finish_reason'.""" 108 | 109 | def __init__(self, token: int) -> None: 110 | """Initialize error.""" 111 | super().__init__( 112 | self, 113 | f"token length(`{token}`) exceeded. Increase maximum token to avoid the issue.", 114 | ) 115 | self.token = token 116 | 117 | def __str__(self) -> str: 118 | """Return string representation.""" 119 | return f"token length(`{self.token}`) exceeded. Increase maximum token to avoid the issue." 120 | 121 | 122 | class InvalidFunction(HomeAssistantError): 123 | """When function validation failed.""" 124 | 125 | def __init__(self, function_name: str) -> None: 126 | """Initialize error.""" 127 | super().__init__( 128 | self, 129 | f"failed to validate function `{function_name}`", 130 | ) 131 | self.function_name = function_name 132 | 133 | def __str__(self) -> str: 134 | """Return string representation.""" 135 | return f"failed to validate function `{self.function_name}` ({self.__cause__})" 136 | -------------------------------------------------------------------------------- /examples/function/youtube/README.md: -------------------------------------------------------------------------------- 1 | ## Objective 2 | 3 | 4 | ## Prompt 5 | Add following text in your prompt 6 | ```` 7 | Youtube Channels: 8 | ```csv 9 | channel_id,channel_name 10 | UCLkAepWjdylmXSltofFvsYQ,BANGTANTV 11 | ``` 12 | ```` 13 | 14 | ## Function 15 | ### play_youtube 16 | #### webostv 17 | ```yaml 18 | - spec: 19 | name: play_youtube 20 | description: Use this function to play Youtube. 21 | parameters: 22 | type: object 23 | properties: 24 | video_id: 25 | type: string 26 | description: The video id. 27 | required: 28 | - video_id 29 | function: 30 | type: script 31 | sequence: 32 | - service: webostv.command 33 | data: 34 | entity_id: media_player.{YOUR_WEBOSTV} 35 | command: system.launcher/launch 36 | payload: 37 | id: youtube.leanback.v4 38 | contentId: "{{video_id}}" 39 | - delay: 40 | hours: 0 41 | minutes: 0 42 | seconds: 10 43 | milliseconds: 0 44 | - service: webostv.button 45 | data: 46 | entity_id: media_player.{YOUR_WEBOSTV} 47 | button: ENTER 48 | ``` 49 | #### Apple TV 50 | ```yaml 51 | - spec: 52 | name: play_youtube_on_apple_tv 53 | description: Use this function to play YouTube content on a specified Apple TV. 54 | parameters: 55 | type: object 56 | properties: 57 | kind: 58 | type: string 59 | enum: 60 | - video 61 | - channel 62 | - playlist 63 | description: The type of YouTube content. 64 | content_id: 65 | type: string 66 | description: ID of the YouTube content (can be videoId, channelId, or playlistId). 67 | entity_id: 68 | type: string 69 | description: entity_id of Apple TV. 70 | required: 71 | - kind 72 | - content_id 73 | - entity_id 74 | function: 75 | type: script 76 | sequence: 77 | - service: script.play_youtube_on_apple_tv 78 | data: 79 | kind: "{{ kind }}" 80 | content_id: "{{ content_id }}" 81 | player: "{{ player }}" 82 | ``` 83 | 84 | ```yaml 85 | script: 86 | play_youtube_on_apple_tv: 87 | alias: "Play YouTube on Apple TV" 88 | sequence: 89 | - service: media_player.play_media 90 | data_template: 91 | media_content_type: url 92 | media_content_id: >- 93 | {% if kind == 'video' %} 94 | youtube://www.youtube.com/watch?v={{content_id}} 95 | {% elif kind == 'channel' %} 96 | youtube://www.youtube.com/channel/{{content_id}} 97 | {% else %} 98 | youtube://www.youtube.com/playlist?list={{content_id}} 99 | {% endif %} 100 | target: 101 | entity_id: "{{ entity_id }}" 102 | ``` 103 | 104 | #### Android TV 105 | ```yaml 106 | - spec: 107 | name: play_youtube_on_android_tv 108 | description: Use this function to play YouTube content on a specified Android TV. 109 | parameters: 110 | type: object 111 | properties: 112 | kind: 113 | type: string 114 | enum: 115 | - video 116 | - channel 117 | - playlist 118 | description: The type of YouTube content. 119 | content_id: 120 | type: string 121 | description: ID of the YouTube content (can be videoId, channelId, or playlistId). 122 | player: 123 | type: string 124 | description: media_player entity. 125 | required: 126 | - kind 127 | - content_id 128 | - player 129 | function: 130 | type: script 131 | sequence: 132 | - service: script.play_youtube_on_android_tv 133 | data: 134 | kind: "{{ kind }}" 135 | content_id: "{{ content_id }}" 136 | player: "{{ player }}" 137 | ``` 138 | 139 | ```yaml 140 | script: 141 | play_youtube_on_android_tv: 142 | alias: "Play YouTube on Android TV" 143 | sequence: 144 | - service: remote.turn_on 145 | data: 146 | activity: >- 147 | {% if kind == 'video' %} 148 | https://www.youtube.com/watch?v={{content_id}} 149 | {% elif kind == 'channel' %} 150 | https://www.youtube.com/channel/{{content_id}} 151 | {% else %} {# playlist kind #} 152 | https://www.youtube.com/playlist?list={{content_id}} 153 | {% endif %} 154 | target: 155 | entity_id: "{{ player }}" 156 | ``` 157 | 158 | ### get_recent_youtube 159 | ```yaml 160 | - spec: 161 | name: get_recent_youtube_videos 162 | description: Use this function to get recent videos of youtube. 163 | parameters: 164 | type: object 165 | properties: 166 | channel_id: 167 | type: string 168 | description: The channel id of Youtube 169 | required: 170 | - channel_id 171 | function: 172 | type: rest 173 | resource_template: "https://www.youtube.com/feeds/videos.xml?channel_id={{channel_id}}" 174 | value_template: >- 175 | ```csv 176 | video_id,title 177 | {% for item in value_json["feed"]["entry"] %} 178 | {{item["yt:videoId"]}},{{item["title"][0:10]}} 179 | {% endfor -%} 180 | ``` 181 | ``` 182 | 183 | ### search_youtube 184 | - Replace "YOUROWNSUPERSECRETYOUTUBEAPIV3KEY" with your API Key 185 | 186 | ```yaml 187 | - spec: 188 | name: search_youtube 189 | description: Use this function to search for YouTube videos, channels, or playlists based on a query. 190 | parameters: 191 | type: object 192 | properties: 193 | query: 194 | type: string 195 | description: The search query to look up on YouTube. 196 | type: 197 | type: string 198 | enum: 199 | - video 200 | - channel 201 | - playlist 202 | default: video 203 | description: The type of content to search for on YouTube. 204 | required: 205 | - query 206 | - type 207 | function: 208 | type: rest 209 | resource_template: "https://www.googleapis.com/youtube/v3/search?part=snippet&q={{query}}&type={{type}}&key={YOUROWNSUPERSECRETYOUTUBEAPIV3KEY}" 210 | value_template: >- 211 | ```csv 212 | kind,id,title 213 | {% for item in value_json["items"] %} 214 | {{item["id"]["kind"]|replace("youtube#", "")}},{{item["id"][type + "Id"]}},{{item["snippet"]["title"]|replace(",", " ")|truncate(50, True, "...")}} 215 | {% endfor -%} 216 | ``` 217 | ``` 218 | -------------------------------------------------------------------------------- /custom_components/extended_openai_conversation/config_flow.py: -------------------------------------------------------------------------------- 1 | """Config flow for OpenAI Conversation integration.""" 2 | 3 | from __future__ import annotations 4 | 5 | import logging 6 | import types 7 | from types import MappingProxyType 8 | from typing import Any 9 | 10 | from openai._exceptions import APIConnectionError, AuthenticationError 11 | import voluptuous as vol 12 | import yaml 13 | 14 | from homeassistant.config_entries import ( 15 | ConfigEntry, 16 | ConfigFlow, 17 | ConfigFlowResult, 18 | OptionsFlow, 19 | ) 20 | from homeassistant.const import CONF_API_KEY, CONF_NAME 21 | from homeassistant.core import HomeAssistant 22 | from homeassistant.helpers.selector import ( 23 | BooleanSelector, 24 | NumberSelector, 25 | NumberSelectorConfig, 26 | SelectOptionDict, 27 | SelectSelector, 28 | SelectSelectorConfig, 29 | SelectSelectorMode, 30 | TemplateSelector, 31 | ) 32 | 33 | from .const import ( 34 | CONF_API_VERSION, 35 | CONF_ATTACH_USERNAME, 36 | CONF_BASE_URL, 37 | CONF_CHAT_MODEL, 38 | CONF_CONTEXT_THRESHOLD, 39 | CONF_CONTEXT_TRUNCATE_STRATEGY, 40 | CONF_FUNCTIONS, 41 | CONF_MAX_FUNCTION_CALLS_PER_CONVERSATION, 42 | CONF_MAX_TOKENS, 43 | CONF_ORGANIZATION, 44 | CONF_PROMPT, 45 | CONF_SKIP_AUTHENTICATION, 46 | CONF_TEMPERATURE, 47 | CONF_TOP_P, 48 | CONF_USE_TOOLS, 49 | CONTEXT_TRUNCATE_STRATEGIES, 50 | DEFAULT_ATTACH_USERNAME, 51 | DEFAULT_CHAT_MODEL, 52 | DEFAULT_CONF_BASE_URL, 53 | DEFAULT_CONF_FUNCTIONS, 54 | DEFAULT_CONTEXT_THRESHOLD, 55 | DEFAULT_CONTEXT_TRUNCATE_STRATEGY, 56 | DEFAULT_MAX_FUNCTION_CALLS_PER_CONVERSATION, 57 | DEFAULT_MAX_TOKENS, 58 | DEFAULT_NAME, 59 | DEFAULT_PROMPT, 60 | DEFAULT_SKIP_AUTHENTICATION, 61 | DEFAULT_TEMPERATURE, 62 | DEFAULT_TOP_P, 63 | DEFAULT_USE_TOOLS, 64 | DOMAIN, 65 | ) 66 | from .helpers import validate_authentication 67 | 68 | _LOGGER = logging.getLogger(__name__) 69 | 70 | STEP_USER_DATA_SCHEMA = vol.Schema( 71 | { 72 | vol.Optional(CONF_NAME): str, 73 | vol.Required(CONF_API_KEY): str, 74 | vol.Optional(CONF_BASE_URL, default=DEFAULT_CONF_BASE_URL): str, 75 | vol.Optional(CONF_API_VERSION): str, 76 | vol.Optional(CONF_ORGANIZATION): str, 77 | vol.Optional( 78 | CONF_SKIP_AUTHENTICATION, default=DEFAULT_SKIP_AUTHENTICATION 79 | ): bool, 80 | } 81 | ) 82 | 83 | DEFAULT_CONF_FUNCTIONS_STR = yaml.dump(DEFAULT_CONF_FUNCTIONS, sort_keys=False) 84 | 85 | DEFAULT_OPTIONS = types.MappingProxyType( 86 | { 87 | CONF_PROMPT: DEFAULT_PROMPT, 88 | CONF_CHAT_MODEL: DEFAULT_CHAT_MODEL, 89 | CONF_MAX_TOKENS: DEFAULT_MAX_TOKENS, 90 | CONF_MAX_FUNCTION_CALLS_PER_CONVERSATION: DEFAULT_MAX_FUNCTION_CALLS_PER_CONVERSATION, 91 | CONF_TOP_P: DEFAULT_TOP_P, 92 | CONF_TEMPERATURE: DEFAULT_TEMPERATURE, 93 | CONF_FUNCTIONS: DEFAULT_CONF_FUNCTIONS_STR, 94 | CONF_ATTACH_USERNAME: DEFAULT_ATTACH_USERNAME, 95 | CONF_USE_TOOLS: DEFAULT_USE_TOOLS, 96 | CONF_CONTEXT_THRESHOLD: DEFAULT_CONTEXT_THRESHOLD, 97 | CONF_CONTEXT_TRUNCATE_STRATEGY: DEFAULT_CONTEXT_TRUNCATE_STRATEGY, 98 | } 99 | ) 100 | 101 | 102 | async def validate_input(hass: HomeAssistant, data: dict[str, Any]) -> None: 103 | """Validate the user input allows us to connect. 104 | 105 | Data has the keys from STEP_USER_DATA_SCHEMA with values provided by the user. 106 | """ 107 | api_key = data[CONF_API_KEY] 108 | base_url = data.get(CONF_BASE_URL) 109 | api_version = data.get(CONF_API_VERSION) 110 | organization = data.get(CONF_ORGANIZATION) 111 | skip_authentication = data.get(CONF_SKIP_AUTHENTICATION) 112 | 113 | if base_url == DEFAULT_CONF_BASE_URL: 114 | # Do not set base_url if using OpenAI for case of OpenAI's base_url change 115 | base_url = None 116 | data.pop(CONF_BASE_URL) 117 | 118 | await validate_authentication( 119 | hass=hass, 120 | api_key=api_key, 121 | base_url=base_url, 122 | api_version=api_version, 123 | organization=organization, 124 | skip_authentication=skip_authentication, 125 | ) 126 | 127 | 128 | class ExtendedOpenAIConversationConfigFlow(ConfigFlow, domain=DOMAIN): 129 | """Handle a config flow for OpenAI Conversation.""" 130 | 131 | VERSION = 1 132 | 133 | async def async_step_user( 134 | self, user_input: dict[str, Any] | None = None 135 | ) -> ConfigFlowResult: 136 | """Handle the initial step.""" 137 | if user_input is None: 138 | return self.async_show_form( 139 | step_id="user", data_schema=STEP_USER_DATA_SCHEMA 140 | ) 141 | 142 | errors = {} 143 | 144 | try: 145 | await validate_input(self.hass, user_input) 146 | except APIConnectionError: 147 | errors["base"] = "cannot_connect" 148 | except AuthenticationError: 149 | errors["base"] = "invalid_auth" 150 | except Exception: # pylint: disable=broad-except 151 | _LOGGER.exception("Unexpected exception") 152 | errors["base"] = "unknown" 153 | else: 154 | return self.async_create_entry( 155 | title=user_input.get(CONF_NAME, DEFAULT_NAME), data=user_input 156 | ) 157 | 158 | return self.async_show_form( 159 | step_id="user", data_schema=STEP_USER_DATA_SCHEMA, errors=errors 160 | ) 161 | 162 | @staticmethod 163 | def async_get_options_flow( 164 | config_entry: ConfigEntry, 165 | ) -> OptionsFlow: 166 | """Create the options flow.""" 167 | return ExtendedOpenAIConversationOptionsFlow() 168 | 169 | 170 | class ExtendedOpenAIConversationOptionsFlow(OptionsFlow): 171 | """OpenAI config flow options handler.""" 172 | 173 | async def async_step_init( 174 | self, user_input: dict[str, Any] | None = None 175 | ) -> ConfigFlowResult: 176 | """Manage the options.""" 177 | if user_input is not None: 178 | return self.async_create_entry( 179 | title=user_input.get(CONF_NAME, DEFAULT_NAME), data=user_input 180 | ) 181 | schema = self.openai_config_option_schema(self.config_entry.options) 182 | return self.async_show_form( 183 | step_id="init", 184 | data_schema=vol.Schema(schema), 185 | ) 186 | 187 | def openai_config_option_schema(self, options: MappingProxyType[str, Any]) -> dict: 188 | """Return a schema for OpenAI completion options.""" 189 | if not options: 190 | options = DEFAULT_OPTIONS 191 | 192 | return { 193 | vol.Optional( 194 | CONF_PROMPT, 195 | description={"suggested_value": options[CONF_PROMPT]}, 196 | default=DEFAULT_PROMPT, 197 | ): TemplateSelector(), 198 | vol.Optional( 199 | CONF_CHAT_MODEL, 200 | description={ 201 | # New key in HA 2023.4 202 | "suggested_value": options.get(CONF_CHAT_MODEL, DEFAULT_CHAT_MODEL) 203 | }, 204 | default=DEFAULT_CHAT_MODEL, 205 | ): str, 206 | vol.Optional( 207 | CONF_MAX_TOKENS, 208 | description={"suggested_value": options[CONF_MAX_TOKENS]}, 209 | default=DEFAULT_MAX_TOKENS, 210 | ): int, 211 | vol.Optional( 212 | CONF_TOP_P, 213 | description={"suggested_value": options[CONF_TOP_P]}, 214 | default=DEFAULT_TOP_P, 215 | ): NumberSelector(NumberSelectorConfig(min=0, max=1, step=0.05)), 216 | vol.Optional( 217 | CONF_TEMPERATURE, 218 | description={"suggested_value": options[CONF_TEMPERATURE]}, 219 | default=DEFAULT_TEMPERATURE, 220 | ): NumberSelector(NumberSelectorConfig(min=0, max=1, step=0.05)), 221 | vol.Optional( 222 | CONF_MAX_FUNCTION_CALLS_PER_CONVERSATION, 223 | description={ 224 | "suggested_value": options[CONF_MAX_FUNCTION_CALLS_PER_CONVERSATION] 225 | }, 226 | default=DEFAULT_MAX_FUNCTION_CALLS_PER_CONVERSATION, 227 | ): int, 228 | vol.Optional( 229 | CONF_FUNCTIONS, 230 | description={"suggested_value": options.get(CONF_FUNCTIONS)}, 231 | default=DEFAULT_CONF_FUNCTIONS_STR, 232 | ): TemplateSelector(), 233 | vol.Optional( 234 | CONF_ATTACH_USERNAME, 235 | description={"suggested_value": options.get(CONF_ATTACH_USERNAME)}, 236 | default=DEFAULT_ATTACH_USERNAME, 237 | ): BooleanSelector(), 238 | vol.Optional( 239 | CONF_USE_TOOLS, 240 | description={"suggested_value": options.get(CONF_USE_TOOLS)}, 241 | default=DEFAULT_USE_TOOLS, 242 | ): BooleanSelector(), 243 | vol.Optional( 244 | CONF_CONTEXT_THRESHOLD, 245 | description={"suggested_value": options.get(CONF_CONTEXT_THRESHOLD)}, 246 | default=DEFAULT_CONTEXT_THRESHOLD, 247 | ): int, 248 | vol.Optional( 249 | CONF_CONTEXT_TRUNCATE_STRATEGY, 250 | description={ 251 | "suggested_value": options.get(CONF_CONTEXT_TRUNCATE_STRATEGY) 252 | }, 253 | default=DEFAULT_CONTEXT_TRUNCATE_STRATEGY, 254 | ): SelectSelector( 255 | SelectSelectorConfig( 256 | options=[ 257 | SelectOptionDict(value=strategy["key"], label=strategy["label"]) 258 | for strategy in CONTEXT_TRUNCATE_STRATEGIES 259 | ], 260 | mode=SelectSelectorMode.DROPDOWN, 261 | ) 262 | ), 263 | } 264 | -------------------------------------------------------------------------------- /examples/function/sqlite/README.md: -------------------------------------------------------------------------------- 1 | ## Objective 2 | - Query anything from database. 3 | 4 | ## Function 5 | 6 | ### 1. SQL generated by LLM 7 | - Try with few examples and find one that best fits for your case. 8 | - Tweak name and description to find better result of query 9 | 10 | #### simple (with no validation) 11 | ```yaml 12 | - spec: 13 | name: query_histories_from_db 14 | description: >- 15 | Use this function to query histories from Home Assistant SQLite database. 16 | Example: 17 | Question: How long was livingroom light on in Nov 15? 18 | Answer: SELECT datetime(s.last_updated_ts, 'unixepoch', 'localtime') last_updated, s.state, old.state as prev_state FROM states s INNER JOIN states_meta sm ON s.metadata_id = sm.metadata_id INNER JOIN states old ON s.old_state_id = old.state_id WHERE sm.entity_id = 'switch.livingroom' AND s.state != old.state AND datetime(s.last_updated_ts, 'unixepoch', 'localtime') BETWEEN '2023-11-15 00:00:00' AND '2023-11-15 23:59:59' 19 | parameters: 20 | type: object 21 | properties: 22 | query: 23 | type: string 24 | description: A fully formed SQL query. 25 | function: 26 | type: sqlite 27 | ``` 28 | 29 | ```yaml 30 | - spec: 31 | name: query_histories_from_db 32 | description: >- 33 | Use this function to query histories from Home Assistant SQLite database. 34 | Example: 35 | Question: When did bedroom light turn on? 36 | Answer: SELECT datetime(s.last_updated_ts, 'unixepoch', 'localtime') last_updated_ts FROM states s INNER JOIN states_meta sm ON s.metadata_id = sm.metadata_id INNER JOIN states old ON s.old_state_id = old.state_id WHERE sm.entity_id = 'light.bedroom' AND s.state = 'on' AND s.state != old.state ORDER BY s.last_updated_ts DESC LIMIT 1 37 | Question: Was livingroom light on at 9 am? 38 | Answer: SELECT datetime(s.last_updated_ts, 'unixepoch', 'localtime') last_updated, s.state FROM states s INNER JOIN states_meta sm ON s.metadata_id = sm.metadata_id INNER JOIN states old ON s.old_state_id = old.state_id WHERE sm.entity_id = 'switch.livingroom' AND s.state != old.state AND datetime(s.last_updated_ts, 'unixepoch', 'localtime') < '2023-11-17 08:00:00' ORDER BY s.last_updated_ts DESC LIMIT 1 39 | parameters: 40 | type: object 41 | properties: 42 | query: 43 | type: string 44 | description: A fully formed SQL query. 45 | function: 46 | type: sqlite 47 | ``` 48 | 49 | #### with minimum validation (still not enough) 50 | ```yaml 51 | - spec: 52 | name: query_histories_from_db 53 | description: >- 54 | Use this function to query histories from Home Assistant SQLite database. 55 | Example: 56 | Question: How long was livingroom light on in Nov 15? 57 | Answer: SELECT datetime(s.last_updated_ts, 'unixepoch', 'localtime') last_updated, s.state, old.state as prev_state FROM states s INNER JOIN states_meta sm ON s.metadata_id = sm.metadata_id INNER JOIN states old ON s.old_state_id = old.state_id WHERE sm.entity_id = 'switch.livingroom' AND s.state != old.state AND datetime(s.last_updated_ts, 'unixepoch', 'localtime') BETWEEN '2023-11-15 00:00:00' AND '2023-11-15 23:59:59' 58 | parameters: 59 | type: object 60 | properties: 61 | query: 62 | type: string 63 | description: A fully formed SQL query. 64 | function: 65 | type: sqlite 66 | query: >- 67 | {%- if is_exposed_entity_in_query(query) -%} 68 | {{ query }} 69 | {%- else -%} 70 | {{ raise("entity_id should be exposed.") }} 71 | {%- endif -%} 72 | ``` 73 | 74 | ```yaml 75 | - spec: 76 | name: query_histories_from_db 77 | description: >- 78 | Use this function to query histories from Home Assistant SQLite database. 79 | Example: 80 | Question: When did bedroom light turn on? 81 | Answer: SELECT datetime(s.last_updated_ts, 'unixepoch', 'localtime') last_updated_ts FROM states s INNER JOIN states_meta sm ON s.metadata_id = sm.metadata_id INNER JOIN states old ON s.old_state_id = old.state_id WHERE sm.entity_id = 'light.bedroom' AND s.state = 'on' AND s.state != old.state ORDER BY s.last_updated_ts DESC LIMIT 1 82 | Question: Was livingroom light on at 9 am? 83 | Answer: SELECT datetime(s.last_updated_ts, 'unixepoch', 'localtime') last_updated, s.state FROM states s INNER JOIN states_meta sm ON s.metadata_id = sm.metadata_id INNER JOIN states old ON s.old_state_id = old.state_id WHERE sm.entity_id = 'switch.livingroom' AND s.state != old.state AND datetime(s.last_updated_ts, 'unixepoch', 'localtime') < '2023-11-17 08:00:00' ORDER BY s.last_updated_ts DESC LIMIT 1 84 | parameters: 85 | type: object 86 | properties: 87 | query: 88 | type: string 89 | description: A fully formed SQL query. 90 | function: 91 | type: sqlite 92 | query: >- 93 | {%- if is_exposed_entity_in_query(query) -%} 94 | {{ query }} 95 | {%- else -%} 96 | {{ raise("entity_id should be exposed.") }} 97 | {%- endif -%} 98 | ``` 99 | 100 | 101 | ### 2. Defined SQL manually 102 | 103 | #### 2-1. get_state_at_time 104 | 105 | 106 | 107 | ```yaml 108 | - spec: 109 | name: get_state_at_time 110 | description: > 111 | Use this function to get state at time 112 | parameters: 113 | type: object 114 | properties: 115 | entity_id: 116 | type: string 117 | description: The target entity 118 | datetime: 119 | type: string 120 | description: The datetime in '%Y-%m-%d %H:%M:%S' format 121 | required: 122 | - entity_id 123 | - datetime 124 | - limit 125 | function: 126 | type: sqlite 127 | query: >- 128 | {%- if is_exposed(entity_id) -%} 129 | SELECT datetime(s.last_updated_ts, 'unixepoch', 'localtime') as state_updated_at, s.state 130 | FROM states s 131 | INNER JOIN states_meta sm ON s.metadata_id = sm.metadata_id 132 | INNER JOIN states old ON s.old_state_id = old.state_id 133 | WHERE sm.entity_id = '{{entity_id}}' 134 | AND s.state != old.state 135 | AND datetime(s.last_updated_ts, 'unixepoch', 'localtime') < '{{datetime}}' 136 | ORDER BY s.last_updated_ts DESC 137 | LIMIT 1 138 | {%- else -%} 139 | {{ raise("entity_id should be exposed.") }} 140 | {%- endif -%} 141 | ``` 142 | 143 | 144 | #### 2-2. get_states_between 145 | 146 | 147 | ```yaml 148 | - spec: 149 | name: get_states_between 150 | description: > 151 | Use this function to get non-numeric states between two dates. 152 | parameters: 153 | type: object 154 | properties: 155 | entity_id: 156 | type: string 157 | description: The target entity 158 | state: 159 | type: string 160 | description: The state 161 | state_operator: 162 | type: string 163 | description: The state operator 164 | enum: 165 | - ">" 166 | - "<" 167 | - "=" 168 | - ">=" 169 | - "<=" 170 | start_datetime: 171 | type: string 172 | description: The start datetime in '%Y-%m-%d %H:%M:%S' format 173 | end_datetime: 174 | type: string 175 | description: The end datetime in '%Y-%m-%d %H:%M:%S' format 176 | order: 177 | type: string 178 | description: The order of datetime, defaults to desc 179 | enum: 180 | - asc 181 | - desc 182 | page: 183 | type: integer 184 | description: The page number 185 | limit: 186 | type: integer 187 | description: The page size defaults to 10 188 | required: 189 | - entity_id 190 | - start_datetime 191 | - end_datetime 192 | - order 193 | - page 194 | - limit 195 | function: 196 | type: composite 197 | sequence: 198 | - type: sqlite 199 | query: >- 200 | {%- if is_exposed(entity_id) -%} 201 | SELECT datetime(s.last_updated_ts, 'unixepoch', 'localtime') as updated_at, s.state 202 | FROM states s 203 | INNER JOIN states_meta sm ON s.metadata_id = sm.metadata_id 204 | INNER JOIN states old ON s.old_state_id = old.state_id 205 | WHERE sm.entity_id = '{{entity_id}}' 206 | AND s.state != old.state 207 | AND (('{{state | default('')}}' = '') OR (s.state {{state_operator | default('=')}} '{{state | default('')}}')) 208 | AND datetime(s.last_updated_ts, 'unixepoch', 'localtime') >= '{{start_datetime}}' 209 | AND datetime(s.last_updated_ts, 'unixepoch', 'localtime') < '{{end_datetime}}' 210 | ORDER BY s.last_updated_ts {{order}} 211 | LIMIT {{(page-1) * limit}}, {{limit}} 212 | {%- else -%} 213 | {{ raise("entity_id should be exposed.") }} 214 | {%- endif -%} 215 | response_variable: data 216 | - type: sqlite 217 | single: true 218 | query: >- 219 | SELECT count(*) as count 220 | FROM states s 221 | INNER JOIN states_meta sm ON s.metadata_id = sm.metadata_id 222 | INNER JOIN states old ON s.old_state_id = old.state_id 223 | WHERE sm.entity_id = '{{entity_id}}' 224 | AND s.state != old.state 225 | AND (('{{state | default('')}}' = '') OR (s.state {{state_operator | default('=')}} '{{state | default('')}}')) 226 | AND datetime(s.last_updated_ts, 'unixepoch', 'localtime') >= '{{start_datetime}}' 227 | AND datetime(s.last_updated_ts, 'unixepoch', 'localtime') < '{{end_datetime}}' 228 | response_variable: total 229 | - type: template 230 | value_template: '{"data": {{data}}, "total": {{total.count}}}' 231 | ``` 232 | 233 | #### 2-3. get_total_time_of_entity_state 234 | 235 | 236 | 237 | ```yaml 238 | - spec: 239 | name: get_total_time_of_entity_state 240 | description: > 241 | Use this function to get total time of state of entity between two dates 242 | parameters: 243 | type: object 244 | properties: 245 | entity_id: 246 | type: string 247 | description: The target entity 248 | state: 249 | type: string 250 | description: The non-numeric target state 251 | start_datetime: 252 | type: string 253 | description: The start datetime in '%Y-%m-%d %H:%M:%S' format 254 | end_datetime: 255 | type: string 256 | description: The end datetime in '%Y-%m-%d %H:%M:%S' format 257 | required: 258 | - entity_id 259 | - state 260 | - start_datetime 261 | - end_datetime 262 | function: 263 | type: composite 264 | sequence: 265 | - type: sqlite 266 | query: >- 267 | {%- if is_exposed(entity_id) -%} 268 | WITH stat_data AS ( 269 | WITH lead_data AS ( 270 | SELECT datetime(old.last_updated_ts, 'unixepoch', 'localtime') AS prev_last_updated, 271 | old.state AS prev_state, 272 | datetime(s.last_updated_ts, 'unixepoch', 'localtime') AS last_updated, 273 | s.state, 274 | COALESCE(LEAD(datetime(s.last_updated_ts, 'unixepoch', 'localtime')) OVER (ORDER BY s.last_updated), '{{end_datetime}}') AS lead_last_updated, 275 | LEAD(s.state) OVER (ORDER BY s.last_updated) AS lead_state 276 | FROM states s 277 | INNER JOIN states_meta sm ON s.metadata_id = sm.metadata_id 278 | INNER JOIN states old ON s.old_state_id = old.state_id 279 | WHERE sm.entity_id = '{{entity_id}}' 280 | AND s.state != old.state 281 | AND datetime(s.last_updated_ts, 'unixepoch', 'localtime') BETWEEN '{{start_datetime}}' AND '{{end_datetime}}' 282 | ) 283 | SELECT max(prev_last_updated, '{{start_datetime}}') AS prev_last_updated, 284 | prev_state, 285 | last_updated AS last_updated, 286 | state 287 | FROM lead_data 288 | WHERE last_updated = (SELECT MIN(last_updated) FROM lead_data) 289 | 290 | UNION ALL 291 | 292 | SELECT last_updated AS prev_last_updated, state AS prev_state, min(lead_last_updated, strftime('%Y-%m-%d %H:%M:%S', 'now', 'localtime')) AS last_updated, lead_state AS state 293 | FROM lead_data 294 | ) 295 | SELECT SUM(CASE WHEN prev_state = '{{state}}' THEN cast(strftime('%s', last_updated, 'utc') as real) - cast(strftime('%s', prev_last_updated, 'utc') as real) ELSE 0 END) AS total_time_in_sec FROM stat_data 296 | {%- else -%} 297 | {{ raise("entity_id should be exposed.") }} 298 | {%- endif -%} 299 | response_variable: result 300 | - type: template 301 | value_template: >- 302 | {%- if result and result[0] and result[0].total_time_in_sec -%} 303 | {%- set duration = result[0].total_time_in_sec | int -%} 304 | 305 | {%- set days = (duration // 86400) | int -%} 306 | {%- set hours = ((duration % 86400) // 3600) | int -%} 307 | {%- set minutes = ((duration % 3600) // 60) | int -%} 308 | {%- set remaining_seconds = (duration % 60) | int -%} 309 | 310 | {{ "{0}d ".format(days) if days > 0 else "" }}{{ "{0}h ".format(hours) if hours > 0 else "" }}{{ "{0}m ".format(minutes) if minutes > 0 else "" }}{{ "{0}s".format(remaining_seconds) if remaining_seconds > 0 else "" }} 311 | {%- else -%} 312 | unkown 313 | {%- endif -%} 314 | ``` -------------------------------------------------------------------------------- /custom_components/extended_openai_conversation/__init__.py: -------------------------------------------------------------------------------- 1 | """The OpenAI Conversation integration.""" 2 | 3 | from __future__ import annotations 4 | 5 | import json 6 | import logging 7 | from typing import Literal 8 | 9 | from openai import AsyncAzureOpenAI, AsyncOpenAI 10 | from openai._exceptions import AuthenticationError, OpenAIError 11 | from openai.types.chat.chat_completion import ( 12 | ChatCompletion, 13 | ChatCompletionMessage, 14 | Choice, 15 | ) 16 | import yaml 17 | 18 | from homeassistant.components import conversation 19 | from homeassistant.components.conversation import ( 20 | ChatLog, 21 | ConversationInput, 22 | ConversationResult, 23 | async_get_chat_log, 24 | ) 25 | from homeassistant.components.homeassistant.exposed_entities import async_should_expose 26 | from homeassistant.config_entries import ConfigEntry 27 | from homeassistant.const import ATTR_NAME, CONF_API_KEY, MATCH_ALL 28 | from homeassistant.core import HomeAssistant 29 | from homeassistant.exceptions import ( 30 | ConfigEntryNotReady, 31 | HomeAssistantError, 32 | TemplateError, 33 | ) 34 | from homeassistant.helpers import ( 35 | config_validation as cv, 36 | entity_registry as er, 37 | intent, 38 | template, 39 | ) 40 | from homeassistant.helpers.chat_session import async_get_chat_session 41 | from homeassistant.helpers.httpx_client import get_async_client 42 | from homeassistant.helpers.typing import ConfigType 43 | from homeassistant.util import ulid 44 | 45 | from .const import ( 46 | CONF_API_VERSION, 47 | CONF_ATTACH_USERNAME, 48 | CONF_BASE_URL, 49 | CONF_CHAT_MODEL, 50 | CONF_CONTEXT_THRESHOLD, 51 | CONF_CONTEXT_TRUNCATE_STRATEGY, 52 | CONF_FUNCTIONS, 53 | CONF_MAX_FUNCTION_CALLS_PER_CONVERSATION, 54 | CONF_MAX_TOKENS, 55 | CONF_ORGANIZATION, 56 | CONF_PROMPT, 57 | CONF_SKIP_AUTHENTICATION, 58 | CONF_TEMPERATURE, 59 | CONF_TOP_P, 60 | CONF_USE_TOOLS, 61 | DEFAULT_ATTACH_USERNAME, 62 | DEFAULT_CHAT_MODEL, 63 | DEFAULT_CONF_FUNCTIONS, 64 | DEFAULT_CONTEXT_THRESHOLD, 65 | DEFAULT_CONTEXT_TRUNCATE_STRATEGY, 66 | DEFAULT_MAX_FUNCTION_CALLS_PER_CONVERSATION, 67 | DEFAULT_MAX_TOKENS, 68 | DEFAULT_PROMPT, 69 | DEFAULT_SKIP_AUTHENTICATION, 70 | DEFAULT_TEMPERATURE, 71 | DEFAULT_TOP_P, 72 | DEFAULT_USE_TOOLS, 73 | DOMAIN, 74 | EVENT_CONVERSATION_FINISHED, 75 | ) 76 | from .exceptions import ( 77 | FunctionLoadFailed, 78 | FunctionNotFound, 79 | InvalidFunction, 80 | ParseArgumentsFailed, 81 | TokenLengthExceededError, 82 | ) 83 | from .helpers import get_function_executor, is_azure, validate_authentication 84 | from .services import async_setup_services 85 | 86 | _LOGGER = logging.getLogger(__name__) 87 | 88 | CONFIG_SCHEMA = cv.config_entry_only_config_schema(DOMAIN) 89 | 90 | 91 | # hass.data key for agent. 92 | DATA_AGENT = "agent" 93 | 94 | 95 | async def async_setup(hass: HomeAssistant, config: ConfigType) -> bool: 96 | """Set up OpenAI Conversation.""" 97 | await async_setup_services(hass, config) 98 | return True 99 | 100 | 101 | async def async_setup_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool: 102 | """Set up OpenAI Conversation from a config entry.""" 103 | 104 | try: 105 | await validate_authentication( 106 | hass=hass, 107 | api_key=entry.data[CONF_API_KEY], 108 | base_url=entry.data.get(CONF_BASE_URL), 109 | api_version=entry.data.get(CONF_API_VERSION), 110 | organization=entry.data.get(CONF_ORGANIZATION), 111 | skip_authentication=entry.data.get( 112 | CONF_SKIP_AUTHENTICATION, DEFAULT_SKIP_AUTHENTICATION 113 | ), 114 | ) 115 | except AuthenticationError as err: 116 | _LOGGER.error("Invalid API key: %s", err) 117 | return False 118 | except OpenAIError as err: 119 | raise ConfigEntryNotReady(err) from err 120 | 121 | agent = OpenAIAgent(hass, entry) 122 | 123 | data = hass.data.setdefault(DOMAIN, {}).setdefault(entry.entry_id, {}) 124 | data[CONF_API_KEY] = entry.data[CONF_API_KEY] 125 | data[DATA_AGENT] = agent 126 | 127 | conversation.async_set_agent(hass, entry, agent) 128 | return True 129 | 130 | 131 | async def async_unload_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool: 132 | """Unload OpenAI.""" 133 | hass.data[DOMAIN].pop(entry.entry_id) 134 | conversation.async_unset_agent(hass, entry) 135 | return True 136 | 137 | 138 | class OpenAIAgent(conversation.AbstractConversationAgent): 139 | """OpenAI conversation agent.""" 140 | 141 | def __init__(self, hass: HomeAssistant, entry: ConfigEntry) -> None: 142 | """Initialize the agent.""" 143 | self.hass = hass 144 | self.entry = entry 145 | self.history: dict[str, list[dict]] = {} 146 | base_url = entry.data.get(CONF_BASE_URL) 147 | if is_azure(base_url): 148 | self.client = AsyncAzureOpenAI( 149 | api_key=entry.data[CONF_API_KEY], 150 | azure_endpoint=base_url, 151 | api_version=entry.data.get(CONF_API_VERSION), 152 | organization=entry.data.get(CONF_ORGANIZATION), 153 | http_client=get_async_client(hass), 154 | ) 155 | else: 156 | self.client = AsyncOpenAI( 157 | api_key=entry.data[CONF_API_KEY], 158 | base_url=base_url, 159 | organization=entry.data.get(CONF_ORGANIZATION), 160 | http_client=get_async_client(hass), 161 | ) 162 | # Cache current platform data which gets added to each request (caching done by library) 163 | _ = hass.async_add_executor_job(self.client.platform_headers) 164 | 165 | @property 166 | def supported_languages(self) -> list[str] | Literal["*"]: 167 | """Return a list of supported languages.""" 168 | return MATCH_ALL 169 | 170 | async def async_process(self, user_input: ConversationInput) -> ConversationResult: 171 | """Process a sentence.""" 172 | with ( 173 | async_get_chat_session(self.hass, user_input.conversation_id) as session, 174 | async_get_chat_log(self.hass, session, user_input) as chat_log, 175 | ): 176 | return await self._async_handle_message(user_input, chat_log) 177 | 178 | async def _async_handle_message( 179 | self, 180 | user_input: ConversationInput, 181 | chat_log: ChatLog, 182 | ) -> ConversationResult: 183 | """Call the API.""" 184 | exposed_entities = self.get_exposed_entities() 185 | 186 | conversation_id = chat_log.conversation_id 187 | if conversation_id in self.history: 188 | messages = self.history[conversation_id] 189 | else: 190 | user_input.conversation_id = conversation_id 191 | try: 192 | system_message = self._generate_system_message( 193 | exposed_entities, user_input 194 | ) 195 | except TemplateError as err: 196 | _LOGGER.error("Error rendering prompt: %s", err) 197 | intent_response = intent.IntentResponse(language=user_input.language) 198 | intent_response.async_set_error( 199 | intent.IntentResponseErrorCode.UNKNOWN, 200 | f"Sorry, I had a problem with my template: {err}", 201 | ) 202 | return conversation.ConversationResult( 203 | response=intent_response, conversation_id=conversation_id 204 | ) 205 | messages = [system_message] 206 | user_message = {"role": "user", "content": user_input.text} 207 | if self.entry.options.get(CONF_ATTACH_USERNAME, DEFAULT_ATTACH_USERNAME): 208 | user = user_input.context.user_id 209 | if user is not None: 210 | user_message[ATTR_NAME] = user 211 | 212 | messages.append(user_message) 213 | 214 | try: 215 | query_response = await self.query(user_input, messages, exposed_entities, 0) 216 | except OpenAIError as err: 217 | _LOGGER.error(err) 218 | intent_response = intent.IntentResponse(language=user_input.language) 219 | intent_response.async_set_error( 220 | intent.IntentResponseErrorCode.UNKNOWN, 221 | f"Sorry, I had a problem talking to OpenAI: {err}", 222 | ) 223 | return conversation.ConversationResult( 224 | response=intent_response, conversation_id=conversation_id 225 | ) 226 | except HomeAssistantError as err: 227 | _LOGGER.error(err, exc_info=err) 228 | intent_response = intent.IntentResponse(language=user_input.language) 229 | intent_response.async_set_error( 230 | intent.IntentResponseErrorCode.UNKNOWN, 231 | f"Something went wrong: {err}", 232 | ) 233 | return conversation.ConversationResult( 234 | response=intent_response, conversation_id=conversation_id 235 | ) 236 | 237 | messages.append(query_response.message.model_dump(exclude_none=True)) 238 | self.history[conversation_id] = messages 239 | 240 | self.hass.bus.async_fire( 241 | EVENT_CONVERSATION_FINISHED, 242 | { 243 | "response": query_response.response.model_dump(), 244 | "user_input": user_input, 245 | "messages": messages, 246 | }, 247 | ) 248 | 249 | intent_response = intent.IntentResponse(language=user_input.language) 250 | intent_response.async_set_speech(query_response.message.content) 251 | 252 | # Detect if LLM is asking a follow-up question to enable continued conversation 253 | response_text = query_response.message.content or "" 254 | should_continue = ( 255 | response_text.rstrip().endswith("?") or 256 | any(phrase in response_text.lower() for phrase in [ 257 | "which one", "would you like", "do you want", "would you prefer", 258 | "which do you", "what would you", "shall i", "should i", 259 | "choose from", "select from", "pick from" 260 | ]) 261 | ) 262 | 263 | return conversation.ConversationResult( 264 | response=intent_response, 265 | conversation_id=conversation_id, 266 | continue_conversation=should_continue 267 | ) 268 | 269 | def _generate_system_message( 270 | self, exposed_entities, user_input: conversation.ConversationInput 271 | ): 272 | raw_prompt = self.entry.options.get(CONF_PROMPT, DEFAULT_PROMPT) 273 | prompt = self._async_generate_prompt(raw_prompt, exposed_entities, user_input) 274 | return {"role": "system", "content": prompt} 275 | 276 | def _async_generate_prompt( 277 | self, 278 | raw_prompt: str, 279 | exposed_entities, 280 | user_input: conversation.ConversationInput, 281 | ) -> str: 282 | """Generate a prompt for the user.""" 283 | return template.Template(raw_prompt, self.hass).async_render( 284 | { 285 | "ha_name": self.hass.config.location_name, 286 | "exposed_entities": exposed_entities, 287 | "current_device_id": user_input.device_id, 288 | }, 289 | parse_result=False, 290 | ) 291 | 292 | def get_exposed_entities(self): 293 | states = [ 294 | state 295 | for state in self.hass.states.async_all() 296 | if async_should_expose(self.hass, conversation.DOMAIN, state.entity_id) 297 | ] 298 | entity_registry = er.async_get(self.hass) 299 | exposed_entities = [] 300 | for state in states: 301 | entity_id = state.entity_id 302 | entity = entity_registry.async_get(entity_id) 303 | 304 | aliases = [] 305 | if entity and entity.aliases: 306 | aliases = entity.aliases 307 | 308 | exposed_entities.append( 309 | { 310 | "entity_id": entity_id, 311 | "name": state.name, 312 | "state": self.hass.states.get(entity_id).state, 313 | "aliases": aliases, 314 | } 315 | ) 316 | return exposed_entities 317 | 318 | def get_functions(self): 319 | try: 320 | function = self.entry.options.get(CONF_FUNCTIONS) 321 | result = yaml.safe_load(function) if function else DEFAULT_CONF_FUNCTIONS 322 | if result: 323 | for setting in result: 324 | function_executor = get_function_executor( 325 | setting["function"]["type"] 326 | ) 327 | setting["function"] = function_executor.to_arguments( 328 | setting["function"] 329 | ) 330 | return result 331 | except (InvalidFunction, FunctionNotFound) as e: 332 | raise e 333 | except: 334 | raise FunctionLoadFailed() 335 | 336 | async def truncate_message_history( 337 | self, messages, exposed_entities, user_input: conversation.ConversationInput 338 | ): 339 | """Truncate message history.""" 340 | strategy = self.entry.options.get( 341 | CONF_CONTEXT_TRUNCATE_STRATEGY, DEFAULT_CONTEXT_TRUNCATE_STRATEGY 342 | ) 343 | 344 | if strategy == "clear": 345 | last_user_message_index = None 346 | for i in reversed(range(len(messages))): 347 | if messages[i]["role"] == "user": 348 | last_user_message_index = i 349 | break 350 | 351 | if last_user_message_index is not None: 352 | del messages[1:last_user_message_index] 353 | # refresh system prompt when all messages are deleted 354 | messages[0] = self._generate_system_message( 355 | exposed_entities, user_input 356 | ) 357 | 358 | async def query( 359 | self, 360 | user_input: conversation.ConversationInput, 361 | messages, 362 | exposed_entities, 363 | n_requests, 364 | ) -> OpenAIQueryResponse: 365 | """Process a sentence.""" 366 | model = self.entry.options.get(CONF_CHAT_MODEL, DEFAULT_CHAT_MODEL) 367 | max_tokens = self.entry.options.get(CONF_MAX_TOKENS, DEFAULT_MAX_TOKENS) 368 | top_p = self.entry.options.get(CONF_TOP_P, DEFAULT_TOP_P) 369 | temperature = self.entry.options.get(CONF_TEMPERATURE, DEFAULT_TEMPERATURE) 370 | use_tools = self.entry.options.get(CONF_USE_TOOLS, DEFAULT_USE_TOOLS) 371 | context_threshold = self.entry.options.get( 372 | CONF_CONTEXT_THRESHOLD, DEFAULT_CONTEXT_THRESHOLD 373 | ) 374 | functions = list(map(lambda s: s["spec"], self.get_functions())) 375 | function_call = "auto" 376 | if n_requests == self.entry.options.get( 377 | CONF_MAX_FUNCTION_CALLS_PER_CONVERSATION, 378 | DEFAULT_MAX_FUNCTION_CALLS_PER_CONVERSATION, 379 | ): 380 | function_call = "none" 381 | 382 | tool_kwargs = {"functions": functions, "function_call": function_call} 383 | if use_tools: 384 | tool_kwargs = { 385 | "tools": [{"type": "function", "function": func} for func in functions], 386 | "tool_choice": function_call, 387 | } 388 | 389 | if len(functions) == 0: 390 | tool_kwargs = {} 391 | 392 | _LOGGER.info("Prompt for %s: %s", model, json.dumps(messages)) 393 | 394 | response: ChatCompletion = await self.client.chat.completions.create( 395 | model=model, 396 | messages=messages, 397 | max_tokens=max_tokens, 398 | top_p=top_p, 399 | temperature=temperature, 400 | user=user_input.conversation_id, 401 | **tool_kwargs, 402 | ) 403 | 404 | _LOGGER.info("Response %s", json.dumps(response.model_dump(exclude_none=True))) 405 | 406 | if response.usage and response.usage.total_tokens > context_threshold: 407 | await self.truncate_message_history(messages, exposed_entities, user_input) 408 | 409 | choice: Choice = response.choices[0] 410 | message = choice.message 411 | 412 | if choice.finish_reason == "function_call" or ( 413 | choice.finish_reason == "stop" and choice.message.function_call is not None 414 | ): 415 | return await self.execute_function_call( 416 | user_input, messages, message, exposed_entities, n_requests + 1 417 | ) 418 | if choice.finish_reason == "tool_calls" or ( 419 | choice.finish_reason == "stop" and choice.message.tool_calls is not None 420 | ): 421 | return await self.execute_tool_calls( 422 | user_input, messages, message, exposed_entities, n_requests + 1 423 | ) 424 | if choice.finish_reason == "length": 425 | raise TokenLengthExceededError(response.usage.completion_tokens) 426 | 427 | return OpenAIQueryResponse(response=response, message=message) 428 | 429 | async def execute_function_call( 430 | self, 431 | user_input: conversation.ConversationInput, 432 | messages, 433 | message: ChatCompletionMessage, 434 | exposed_entities, 435 | n_requests, 436 | ) -> OpenAIQueryResponse: 437 | function_name = message.function_call.name 438 | function = next( 439 | (s for s in self.get_functions() if s["spec"]["name"] == function_name), 440 | None, 441 | ) 442 | if function is not None: 443 | return await self.execute_function( 444 | user_input, 445 | messages, 446 | message, 447 | exposed_entities, 448 | n_requests, 449 | function, 450 | ) 451 | raise FunctionNotFound(function_name) 452 | 453 | async def execute_function( 454 | self, 455 | user_input: conversation.ConversationInput, 456 | messages, 457 | message: ChatCompletionMessage, 458 | exposed_entities, 459 | n_requests, 460 | function, 461 | ) -> OpenAIQueryResponse: 462 | function_executor = get_function_executor(function["function"]["type"]) 463 | 464 | try: 465 | arguments = json.loads(message.function_call.arguments) 466 | except json.decoder.JSONDecodeError as err: 467 | raise ParseArgumentsFailed(message.function_call.arguments) from err 468 | 469 | result = await function_executor.execute( 470 | self.hass, function["function"], arguments, user_input, exposed_entities 471 | ) 472 | 473 | messages.append( 474 | { 475 | "role": "function", 476 | "name": message.function_call.name, 477 | "content": str(result), 478 | } 479 | ) 480 | return await self.query(user_input, messages, exposed_entities, n_requests) 481 | 482 | async def execute_tool_calls( 483 | self, 484 | user_input: conversation.ConversationInput, 485 | messages, 486 | message: ChatCompletionMessage, 487 | exposed_entities, 488 | n_requests, 489 | ) -> OpenAIQueryResponse: 490 | messages.append(message.model_dump(exclude_none=True)) 491 | for tool in message.tool_calls: 492 | function_name = tool.function.name 493 | function = next( 494 | (s for s in self.get_functions() if s["spec"]["name"] == function_name), 495 | None, 496 | ) 497 | if function is not None: 498 | result = await self.execute_tool_function( 499 | user_input, 500 | tool, 501 | exposed_entities, 502 | function, 503 | ) 504 | 505 | messages.append( 506 | { 507 | "tool_call_id": tool.id, 508 | "role": "tool", 509 | "name": function_name, 510 | "content": str(result), 511 | } 512 | ) 513 | else: 514 | raise FunctionNotFound(function_name) 515 | return await self.query(user_input, messages, exposed_entities, n_requests) 516 | 517 | async def execute_tool_function( 518 | self, 519 | user_input: conversation.ConversationInput, 520 | tool, 521 | exposed_entities, 522 | function, 523 | ) -> OpenAIQueryResponse: 524 | function_executor = get_function_executor(function["function"]["type"]) 525 | 526 | try: 527 | arguments = json.loads(tool.function.arguments) 528 | except json.decoder.JSONDecodeError as err: 529 | raise ParseArgumentsFailed(tool.function.arguments) from err 530 | 531 | result = await function_executor.execute( 532 | self.hass, function["function"], arguments, user_input, exposed_entities 533 | ) 534 | return result 535 | 536 | 537 | class OpenAIQueryResponse: 538 | """OpenAI query response value object.""" 539 | 540 | def __init__( 541 | self, response: ChatCompletion, message: ChatCompletionMessage 542 | ) -> None: 543 | """Initialize OpenAI query response value object.""" 544 | self.response = response 545 | self.message = message 546 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Extended OpenAI Conversation 2 | This is custom component of Home Assistant. 3 | 4 | Derived from [OpenAI Conversation](https://www.home-assistant.io/integrations/openai_conversation/) with some new features such as call-service. 5 | 6 | ## Additional Features 7 | - Ability to call service of Home Assistant 8 | - Ability to create automation 9 | - Ability to get data from external API or web page 10 | - Ability to retrieve state history of entities 11 | - Option to pass the current user's name to OpenAI via the user message context 12 | 13 | ## How it works 14 | Extended OpenAI Conversation uses OpenAI API's feature of [function calling](https://platform.openai.com/docs/guides/function-calling) to call service of Home Assistant. 15 | 16 | Since OpenAI models already know how to call service of Home Assistant in general, you just have to let model know what devices you have by [exposing entities](https://github.com/jekalmin/extended_openai_conversation#preparation) 17 | 18 | ## Installation 19 | 1. Install via registering as a custom repository of HACS or by copying `extended_openai_conversation` folder into `/custom_components` 20 | 2. Restart Home Assistant 21 | 3. Go to Settings > Devices & Services. 22 | 4. In the bottom right corner, select the Add Integration button. 23 | 5. Follow the instructions on screen to complete the setup (API Key is required). 24 | - [Generating an API Key](https://www.home-assistant.io/integrations/openai_conversation/#generate-an-api-key) 25 | - Specify "Base Url" if using OpenAI compatible servers like Azure OpenAI (also with APIM), LocalAI, otherwise leave as it is. 26 | 6. Go to Settings > [Voice Assistants](https://my.home-assistant.io/redirect/voice_assistants/). 27 | 7. Click to edit Assistant (named "Home Assistant" by default). 28 | 8. Select "Extended OpenAI Conversation" from "Conversation agent" tab. 29 |
30 | 31 | guide image 32 | 스크린샷 2023-10-07 오후 6 15 29 33 | 34 |
35 | 36 | ## Preparation 37 | After installed, you need to expose entities from "http://{your-home-assistant}/config/voice-assistants/expose". 38 | 39 | ## Examples 40 | ### 1. Turn on single entity 41 | https://github.com/jekalmin/extended_openai_conversation/assets/2917984/938dee95-8907-44fd-9fb8-dc8cd559fea2 42 | 43 | ### 2. Turn on multiple entities 44 | https://github.com/jekalmin/extended_openai_conversation/assets/2917984/528f5965-94a7-4cbe-908a-e24f7bbb0a93 45 | 46 | ### 3. Hook with custom notify function 47 | https://github.com/jekalmin/extended_openai_conversation/assets/2917984/4a575ee7-0188-41eb-b2db-6eab61499a99 48 | 49 | ### 4. Add automation 50 | https://github.com/jekalmin/extended_openai_conversation/assets/2917984/04b93aa6-085e-450a-a554-34c1ed1fbb36 51 | 52 | ### 5. Play Netflix 53 | https://github.com/jekalmin/extended_openai_conversation/assets/2917984/64ba656e-3ae7-4003-9956-da71efaf06dc 54 | 55 | ## Configuration 56 | ### Options 57 | By clicking a button from Edit Assist, Options can be customized.
58 | Options include [OpenAI Conversation](https://www.home-assistant.io/integrations/openai_conversation/) options and two new options. 59 | 60 | - `Attach Username`: Pass the active user's name (if applicable) to OpenAI via the message payload. Currently, this only applies to conversations through the UI or REST API. 61 | 62 | - `Maximum Function Calls Per Conversation`: limit the number of function calls in a single conversation. 63 | (Sometimes function is called over and over again, possibly running into infinite loop) 64 | - `Functions`: A list of mappings of function spec to function. 65 | - `spec`: Function which would be passed to [functions](https://platform.openai.com/docs/api-reference/chat/create#chat-create-functions) of [chat API](https://platform.openai.com/docs/api-reference/chat/create). 66 | - `function`: function that will be called. 67 | 68 | 69 | | Edit Assist | Options | 70 | |----------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| 71 | | 1 | 스크린샷 2023-10-10 오후 10 53 57 | 72 | 73 | 74 | ### Functions 75 | 76 | #### Supported function types 77 | - `native`: built-in function provided by "extended_openai_conversation". 78 | - Currently supported native functions and parameters are: 79 | - `execute_service` 80 | - `domain`(string): domain to be passed to `hass.services.async_call` 81 | - `service`(string): service to be passed to `hass.services.async_call` 82 | - `service_data`(object): service_data to be passed to `hass.services.async_call`. 83 | - `entity_id`(string): target entity 84 | - `device_id`(string): target device 85 | - `area_id`(string): target area 86 | - `add_automation` 87 | - `automation_config`(string): An automation configuration in a yaml format 88 | - `get_history` 89 | - `entity_ids`(list): a list of entity ids to filter 90 | - `start_time`(string): defaults to 1 day before the time of the request. It determines the beginning of the period 91 | - `end_time`(string): the end of the period in URL encoded format (defaults to 1 day) 92 | - `minimal_response`(boolean): only return last_changed and state for states other than the first and last state (defaults to true) 93 | - `no_attributes`(boolean): skip returning attributes from the database (defaults to true) 94 | - `significant_changes_only`(boolean): only return significant state changes (defaults to true) 95 | - `script`: A list of services that will be called 96 | - `template`: The value to be returned from function. 97 | - `rest`: Getting data from REST API endpoint. 98 | - `scrape`: Scraping information from website 99 | - `composite`: A sequence of functions to execute. 100 | 101 | Below is a default configuration of functions. 102 | 103 | ```yaml 104 | - spec: 105 | name: execute_services 106 | description: Use this function to execute service of devices in Home Assistant. 107 | parameters: 108 | type: object 109 | properties: 110 | list: 111 | type: array 112 | items: 113 | type: object 114 | properties: 115 | domain: 116 | type: string 117 | description: The domain of the service 118 | service: 119 | type: string 120 | description: The service to be called 121 | service_data: 122 | type: object 123 | description: The service data object to indicate what to control. 124 | properties: 125 | entity_id: 126 | type: string 127 | description: The entity_id retrieved from available devices. It must start with domain, followed by dot character. 128 | required: 129 | - entity_id 130 | required: 131 | - domain 132 | - service 133 | - service_data 134 | function: 135 | type: native 136 | name: execute_service 137 | ``` 138 | 139 | ## Function Usage 140 | This is an example of configuration of functions. 141 | 142 | Copy and paste below yaml configuration into "Functions".
143 | Then you will be able to let OpenAI call your function. 144 | 145 | ### 1. template 146 | #### 1-1. Get current weather 147 | 148 | For real world example, see [weather](https://github.com/jekalmin/extended_openai_conversation/tree/main/examples/function/weather).
149 | This is just an example from [OpenAI documentation](https://platform.openai.com/docs/guides/function-calling/common-use-cases) 150 | 151 | ```yaml 152 | - spec: 153 | name: get_current_weather 154 | description: Get the current weather in a given location 155 | parameters: 156 | type: object 157 | properties: 158 | location: 159 | type: string 160 | description: The city and state, e.g. San Francisco, CA 161 | unit: 162 | type: string 163 | enum: 164 | - celcius 165 | - farenheit 166 | required: 167 | - location 168 | function: 169 | type: template 170 | value_template: The temperature in {{ location }} is 25 {{unit}} 171 | ``` 172 | 173 | 스크린샷 2023-10-07 오후 7 56 27 174 | 175 | ### 2. script 176 | #### 2-1. Add item to shopping cart 177 | ```yaml 178 | - spec: 179 | name: add_item_to_shopping_cart 180 | description: Add item to shopping cart 181 | parameters: 182 | type: object 183 | properties: 184 | item: 185 | type: string 186 | description: The item to be added to cart 187 | required: 188 | - item 189 | function: 190 | type: script 191 | sequence: 192 | - service: shopping_list.add_item 193 | data: 194 | name: '{{item}}' 195 | ``` 196 | 197 | 스크린샷 2023-10-07 오후 7 54 56 198 | 199 | #### 2-2. Send messages to another messenger 200 | 201 | In order to accomplish "send it to Line" like [example3](https://github.com/jekalmin/extended_openai_conversation#3-hook-with-custom-notify-function), register a notify function like below. 202 | 203 | ```yaml 204 | - spec: 205 | name: send_message_to_line 206 | description: Use this function to send message to Line. 207 | parameters: 208 | type: object 209 | properties: 210 | message: 211 | type: string 212 | description: message you want to send 213 | required: 214 | - message 215 | function: 216 | type: script 217 | sequence: 218 | - service: script.notify_all 219 | data: 220 | message: "{{ message }}" 221 | ``` 222 | 223 | 224 | 225 | #### 2-3. Get events from calendar 226 | 227 | In order to pass result of calling service to OpenAI, set response variable to `_function_result`. 228 | 229 | ```yaml 230 | - spec: 231 | name: get_events 232 | description: Use this function to get list of calendar events. 233 | parameters: 234 | type: object 235 | properties: 236 | start_date_time: 237 | type: string 238 | description: The start date time in '%Y-%m-%dT%H:%M:%S%z' format 239 | end_date_time: 240 | type: string 241 | description: The end date time in '%Y-%m-%dT%H:%M:%S%z' format 242 | required: 243 | - start_date_time 244 | - end_date_time 245 | function: 246 | type: script 247 | sequence: 248 | - service: calendar.get_events 249 | data: 250 | start_date_time: "{{start_date_time}}" 251 | end_date_time: "{{end_date_time}}" 252 | target: 253 | entity_id: 254 | - calendar.[YourCalendarHere] 255 | - calendar.[MoreCalendarsArePossible] 256 | response_variable: _function_result 257 | ``` 258 | 259 | 스크린샷 2023-10-31 오후 9 04 56 260 | 261 | #### 2-4. Play Youtube on TV 262 | 263 | ```yaml 264 | - spec: 265 | name: play_youtube 266 | description: Use this function to play Youtube. 267 | parameters: 268 | type: object 269 | properties: 270 | video_id: 271 | type: string 272 | description: The video id. 273 | required: 274 | - video_id 275 | function: 276 | type: script 277 | sequence: 278 | - service: webostv.command 279 | data: 280 | entity_id: media_player.{YOUR_WEBOSTV} 281 | command: system.launcher/launch 282 | payload: 283 | id: youtube.leanback.v4 284 | contentId: "{{video_id}}" 285 | - delay: 286 | hours: 0 287 | minutes: 0 288 | seconds: 10 289 | milliseconds: 0 290 | - service: webostv.button 291 | data: 292 | entity_id: media_player.{YOUR_WEBOSTV} 293 | button: ENTER 294 | ``` 295 | 296 | 297 | 298 | #### 2-5. Play Netflix on TV 299 | 300 | ```yaml 301 | - spec: 302 | name: play_netflix 303 | description: Use this function to play Netflix. 304 | parameters: 305 | type: object 306 | properties: 307 | video_id: 308 | type: string 309 | description: The video id. 310 | required: 311 | - video_id 312 | function: 313 | type: script 314 | sequence: 315 | - service: webostv.command 316 | data: 317 | entity_id: media_player.{YOUR_WEBOSTV} 318 | command: system.launcher/launch 319 | payload: 320 | id: netflix 321 | contentId: "m=https://www.netflix.com/watch/{{video_id}}" 322 | ``` 323 | 324 | 325 | 326 | ### 3. native 327 | 328 | #### 3-1. Add automation 329 | 330 | Before adding automation, I highly recommend set notification on `automation_registered_via_extended_openai_conversation` event and create separate "Extended OpenAI Assistant" and "Assistant" 331 | 332 | (Automation can be added even if conversation fails because of failure to get response message, not automation) 333 | 334 | | Create Assistant | Notify on created | 335 | |----------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| 336 | | 1 | 스크린샷 2023-10-13 오후 6 01 40 | 337 | 338 | 339 | Copy and paste below configuration into "Functions" 340 | 341 | **For English** 342 | ```yaml 343 | - spec: 344 | name: add_automation 345 | description: Use this function to add an automation in Home Assistant. 346 | parameters: 347 | type: object 348 | properties: 349 | automation_config: 350 | type: string 351 | description: A configuration for automation in a valid yaml format. Next line character should be \n. Use devices from the list. 352 | required: 353 | - automation_config 354 | function: 355 | type: native 356 | name: add_automation 357 | ``` 358 | 359 | **For Korean** 360 | ```yaml 361 | - spec: 362 | name: add_automation 363 | description: Use this function to add an automation in Home Assistant. 364 | parameters: 365 | type: object 366 | properties: 367 | automation_config: 368 | type: string 369 | description: A configuration for automation in a valid yaml format. Next line character should be \\n, not \n. Use devices from the list. 370 | required: 371 | - automation_config 372 | function: 373 | type: native 374 | name: add_automation 375 | ``` 376 | 377 | 스크린샷 2023-10-31 오후 9 32 27 378 | 379 | #### 3-2. Get History 380 | Get state history of entities 381 | 382 | ```yaml 383 | - spec: 384 | name: get_history 385 | description: Retrieve historical data of specified entities. 386 | parameters: 387 | type: object 388 | properties: 389 | entity_ids: 390 | type: array 391 | items: 392 | type: string 393 | description: The entity id to filter. 394 | start_time: 395 | type: string 396 | description: Start of the history period in "%Y-%m-%dT%H:%M:%S%z". 397 | end_time: 398 | type: string 399 | description: End of the history period in "%Y-%m-%dT%H:%M:%S%z". 400 | required: 401 | - entity_ids 402 | function: 403 | type: composite 404 | sequence: 405 | - type: native 406 | name: get_history 407 | response_variable: history_result 408 | - type: template 409 | value_template: >- 410 | {% set ns = namespace(result = [], list = []) %} 411 | {% for item_list in history_result %} 412 | {% set ns.list = [] %} 413 | {% for item in item_list %} 414 | {% set last_changed = item.last_changed | as_timestamp | timestamp_local if item.last_changed else None %} 415 | {% set new_item = dict(item, last_changed=last_changed) %} 416 | {% set ns.list = ns.list + [new_item] %} 417 | {% endfor %} 418 | {% set ns.result = ns.result + [ns.list] %} 419 | {% endfor %} 420 | {{ ns.result }} 421 | ``` 422 | 423 | 424 | 425 | ### 4. scrape 426 | #### 4-1. Get current HA version 427 | Scrape version from webpage, "https://www.home-assistant.io" 428 | 429 | Unlike [scrape](https://www.home-assistant.io/integrations/scrape/), "value_template" is added at root level in which scraped data from sensors are passed. 430 | 431 | ```yaml 432 | - spec: 433 | name: get_ha_version 434 | description: Use this function to get Home Assistant version 435 | parameters: 436 | type: object 437 | properties: 438 | dummy: 439 | type: string 440 | description: Nothing 441 | function: 442 | type: scrape 443 | resource: https://www.home-assistant.io 444 | value_template: "version: {{version}}, release_date: {{release_date}}" 445 | sensor: 446 | - name: version 447 | select: ".current-version h1" 448 | value_template: '{{ value.split(":")[1] }}' 449 | - name: release_date 450 | select: ".release-date" 451 | value_template: '{{ value.lower() }}' 452 | ``` 453 | 454 | 스크린샷 2023-10-31 오후 9 46 07 455 | 456 | ### 5. rest 457 | #### 5-1. Get friend names 458 | - Sample URL: https://jsonplaceholder.typicode.com/users 459 | ```yaml 460 | - spec: 461 | name: get_friend_names 462 | description: Use this function to get friend_names 463 | parameters: 464 | type: object 465 | properties: 466 | dummy: 467 | type: string 468 | description: Nothing. 469 | function: 470 | type: rest 471 | resource: https://jsonplaceholder.typicode.com/users 472 | value_template: '{{value_json | map(attribute="name") | list }}' 473 | ``` 474 | 475 | 스크린샷 2023-10-31 오후 9 48 36 476 | 477 | 478 | ### 6. composite 479 | #### 6-1. Search Youtube Music 480 | When using [ytube_music_player](https://github.com/KoljaWindeler/ytube_music_player), after `ytube_music_player.search` service is called, result is stored in attribute of `sensor.ytube_music_player_extra` entity.
481 | 482 | 483 | ```yaml 484 | - spec: 485 | name: search_music 486 | description: Use this function to search music 487 | parameters: 488 | type: object 489 | properties: 490 | query: 491 | type: string 492 | description: The query 493 | required: 494 | - query 495 | function: 496 | type: composite 497 | sequence: 498 | - type: script 499 | sequence: 500 | - service: ytube_music_player.search 501 | data: 502 | entity_id: media_player.ytube_music_player 503 | query: "{{ query }}" 504 | - type: template 505 | value_template: >- 506 | media_content_type,media_content_id,title 507 | {% for media in state_attr('sensor.ytube_music_player_extra', 'search') -%} 508 | {{media.type}},{{media.id}},{{media.title}} 509 | {% endfor%} 510 | ``` 511 | 512 | 스크린샷 2023-11-02 오후 8 40 36 513 | 514 | ### 7. sqlite 515 | #### 7-1. Let model generate a query 516 | - Without examples, a query tries to fetch data only from "states" table like below 517 | > Question: When did bedroom light turn on?
518 | Query(generated by gpt): SELECT * FROM states WHERE entity_id = 'input_boolean.livingroom_light_2' AND state = 'on' ORDER BY last_changed DESC LIMIT 1 519 | - Since "entity_id" is stored in "states_meta" table, we need to give examples of question and query. 520 | - Not secured, but flexible way 521 | 522 | ```yaml 523 | - spec: 524 | name: query_histories_from_db 525 | description: >- 526 | Use this function to query histories from Home Assistant SQLite database. 527 | Example: 528 | Question: When did bedroom light turn on? 529 | Answer: SELECT datetime(s.last_updated_ts, 'unixepoch', 'localtime') last_updated_ts FROM states s INNER JOIN states_meta sm ON s.metadata_id = sm.metadata_id INNER JOIN states old ON s.old_state_id = old.state_id WHERE sm.entity_id = 'light.bedroom' AND s.state = 'on' AND s.state != old.state ORDER BY s.last_updated_ts DESC LIMIT 1 530 | Question: Was livingroom light on at 9 am? 531 | Answer: SELECT datetime(s.last_updated_ts, 'unixepoch', 'localtime') last_updated, s.state FROM states s INNER JOIN states_meta sm ON s.metadata_id = sm.metadata_id INNER JOIN states old ON s.old_state_id = old.state_id WHERE sm.entity_id = 'switch.livingroom' AND s.state != old.state AND datetime(s.last_updated_ts, 'unixepoch', 'localtime') < '2023-11-17 08:00:00' ORDER BY s.last_updated_ts DESC LIMIT 1 532 | parameters: 533 | type: object 534 | properties: 535 | query: 536 | type: string 537 | description: A fully formed SQL query. 538 | function: 539 | type: sqlite 540 | ``` 541 | 542 | Get last changed date time of state | Get state at specific time 543 | --|-- 544 | 스크린샷 2023-11-19 오후 5 32 56 |스크린샷 2023-11-19 오후 5 32 30 545 | 546 | 547 | **FAQ** 548 | 1. Can gpt modify or delete data? 549 | > No, since connection is created in a read only mode, data are only used for fetching. 550 | 2. Can gpt query data that are not exposed in database? 551 | > Yes, it is hard to validate whether a query is only using exposed entities. 552 | 3. Query uses UTC time. Is there any way to adjust timezone? 553 | > Yes. Set "TZ" environment variable to your [region](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) (eg. `Asia/Seoul`).
554 | Or use plus/minus hours to adjust instead of 'localtime' (eg. `datetime(s.last_updated_ts, 'unixepoch', '+9 hours')`). 555 | 556 | 557 | #### 7-2. Let model generate a query (with minimum validation) 558 | - If need to check at least "entity_id" of exposed entities is present in a query, use "is_exposed_entity_in_query" in combination with "raise". 559 | - Not secured enough, but flexible way 560 | ```yaml 561 | - spec: 562 | name: query_histories_from_db 563 | description: >- 564 | Use this function to query histories from Home Assistant SQLite database. 565 | Example: 566 | Question: When did bedroom light turn on? 567 | Answer: SELECT datetime(s.last_updated_ts, 'unixepoch', 'localtime') last_updated_ts FROM states s INNER JOIN states_meta sm ON s.metadata_id = sm.metadata_id INNER JOIN states old ON s.old_state_id = old.state_id WHERE sm.entity_id = 'light.bedroom' AND s.state = 'on' AND s.state != old.state ORDER BY s.last_updated_ts DESC LIMIT 1 568 | Question: Was livingroom light on at 9 am? 569 | Answer: SELECT datetime(s.last_updated_ts, 'unixepoch', 'localtime') last_updated, s.state FROM states s INNER JOIN states_meta sm ON s.metadata_id = sm.metadata_id INNER JOIN states old ON s.old_state_id = old.state_id WHERE sm.entity_id = 'switch.livingroom' AND s.state != old.state AND datetime(s.last_updated_ts, 'unixepoch', 'localtime') < '2023-11-17 08:00:00' ORDER BY s.last_updated_ts DESC LIMIT 1 570 | parameters: 571 | type: object 572 | properties: 573 | query: 574 | type: string 575 | description: A fully formed SQL query. 576 | function: 577 | type: sqlite 578 | query: >- 579 | {%- if is_exposed_entity_in_query(query) -%} 580 | {{ query }} 581 | {%- else -%} 582 | {{ raise("entity_id should be exposed.") }} 583 | {%- endif -%} 584 | ``` 585 | 586 | #### 7-3. Defined SQL manually 587 | - Use a user defined query, which is verified. And model passes a requested entity to get data from database. 588 | - Secured, but less flexible way 589 | ```yaml 590 | - spec: 591 | name: get_last_updated_time_of_entity 592 | description: > 593 | Use this function to get last updated time of entity 594 | parameters: 595 | type: object 596 | properties: 597 | entity_id: 598 | type: string 599 | description: The target entity 600 | function: 601 | type: sqlite 602 | query: >- 603 | {%- if is_exposed(entity_id) -%} 604 | SELECT datetime(s.last_updated_ts, 'unixepoch', 'localtime') as last_updated_ts 605 | FROM states s 606 | INNER JOIN states_meta sm ON s.metadata_id = sm.metadata_id 607 | INNER JOIN states old ON s.old_state_id = old.state_id 608 | WHERE sm.entity_id = '{{entity_id}}' AND s.state != old.state ORDER BY s.last_updated_ts DESC LIMIT 1 609 | {%- else -%} 610 | {{ raise("entity_id should be exposed.") }} 611 | {%- endif -%} 612 | ``` 613 | 614 | ## Practical Usage 615 | See more practical [examples](https://github.com/jekalmin/extended_openai_conversation/tree/main/examples). 616 | 617 | ## Logging 618 | In order to monitor logs of API requests and responses, add following config to `configuration.yaml` file 619 | 620 | ```yaml 621 | logger: 622 | logs: 623 | custom_components.extended_openai_conversation: info 624 | ``` 625 | -------------------------------------------------------------------------------- /custom_components/extended_openai_conversation/helpers.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from datetime import timedelta 3 | from functools import partial 4 | import logging 5 | import os 6 | import re 7 | import sqlite3 8 | import time 9 | from typing import Any 10 | from urllib import parse 11 | 12 | from bs4 import BeautifulSoup 13 | from openai import AsyncAzureOpenAI, AsyncOpenAI 14 | import voluptuous as vol 15 | import yaml 16 | 17 | from homeassistant.components import ( 18 | automation, 19 | conversation, 20 | energy, 21 | recorder, 22 | rest, 23 | scrape, 24 | ) 25 | from homeassistant.components.automation.config import _async_validate_config_item 26 | from homeassistant.components.script.config import SCRIPT_ENTITY_SCHEMA 27 | from homeassistant.config import AUTOMATION_CONFIG_PATH 28 | from homeassistant.const import ( 29 | CONF_ATTRIBUTE, 30 | CONF_METHOD, 31 | CONF_NAME, 32 | CONF_PAYLOAD, 33 | CONF_RESOURCE, 34 | CONF_RESOURCE_TEMPLATE, 35 | CONF_TIMEOUT, 36 | CONF_VALUE_TEMPLATE, 37 | CONF_VERIFY_SSL, 38 | SERVICE_RELOAD, 39 | ) 40 | from homeassistant.core import HomeAssistant, State 41 | from homeassistant.exceptions import HomeAssistantError, ServiceNotFound 42 | from homeassistant.helpers import config_validation as cv 43 | from homeassistant.helpers.httpx_client import get_async_client 44 | from homeassistant.helpers.script import Script 45 | from homeassistant.helpers.template import Template 46 | import homeassistant.util.dt as dt_util 47 | 48 | from .const import CONF_PAYLOAD_TEMPLATE, DOMAIN, EVENT_AUTOMATION_REGISTERED 49 | from .exceptions import ( 50 | CallServiceError, 51 | EntityNotExposed, 52 | EntityNotFound, 53 | FunctionNotFound, 54 | InvalidFunction, 55 | NativeNotFound, 56 | ) 57 | 58 | _LOGGER = logging.getLogger(__name__) 59 | 60 | 61 | AZURE_DOMAIN_PATTERN = r"\.(openai\.azure\.com|azure-api\.net|services\.ai\.azure\.com)" 62 | 63 | 64 | def get_function_executor(value: str): 65 | function_executor = FUNCTION_EXECUTORS.get(value) 66 | if function_executor is None: 67 | raise FunctionNotFound(value) 68 | return function_executor 69 | 70 | 71 | def is_azure(base_url: str): 72 | if base_url and re.search(AZURE_DOMAIN_PATTERN, base_url): 73 | return True 74 | return False 75 | 76 | 77 | def convert_to_template( 78 | settings, 79 | template_keys=["data", "event_data", "target", "service"], 80 | hass: HomeAssistant | None = None, 81 | ): 82 | _convert_to_template(settings, template_keys, hass, []) 83 | 84 | 85 | def _convert_to_template(settings, template_keys, hass, parents: list[str]): 86 | if isinstance(settings, dict): 87 | for key, value in settings.items(): 88 | if isinstance(value, str) and ( 89 | key in template_keys or set(parents).intersection(template_keys) 90 | ): 91 | settings[key] = Template(value, hass) 92 | if isinstance(value, dict): 93 | parents.append(key) 94 | _convert_to_template(value, template_keys, hass, parents) 95 | parents.pop() 96 | if isinstance(value, list): 97 | parents.append(key) 98 | for item in value: 99 | _convert_to_template(item, template_keys, hass, parents) 100 | parents.pop() 101 | if isinstance(settings, list): 102 | for setting in settings: 103 | _convert_to_template(setting, template_keys, hass, parents) 104 | 105 | 106 | def _get_rest_data(hass, rest_config, arguments): 107 | rest_config.setdefault(CONF_METHOD, rest.const.DEFAULT_METHOD) 108 | rest_config.setdefault(CONF_VERIFY_SSL, rest.const.DEFAULT_VERIFY_SSL) 109 | rest_config.setdefault(CONF_TIMEOUT, rest.data.DEFAULT_TIMEOUT) 110 | rest_config.setdefault(rest.const.CONF_ENCODING, rest.const.DEFAULT_ENCODING) 111 | 112 | resource_template: Template | None = rest_config.get(CONF_RESOURCE_TEMPLATE) 113 | if resource_template is not None: 114 | rest_config.pop(CONF_RESOURCE_TEMPLATE) 115 | rest_config[CONF_RESOURCE] = resource_template.async_render( 116 | arguments, parse_result=False 117 | ) 118 | 119 | payload_template: Template | None = rest_config.get(CONF_PAYLOAD_TEMPLATE) 120 | if payload_template is not None: 121 | rest_config.pop(CONF_PAYLOAD_TEMPLATE) 122 | rest_config[CONF_PAYLOAD] = payload_template.async_render( 123 | arguments, parse_result=False 124 | ) 125 | 126 | return rest.create_rest_data_from_config(hass, rest_config) 127 | 128 | 129 | async def validate_authentication( 130 | hass: HomeAssistant, 131 | api_key: str, 132 | base_url: str, 133 | api_version: str, 134 | organization: str = None, 135 | skip_authentication=False, 136 | ) -> None: 137 | if skip_authentication: 138 | return 139 | 140 | if is_azure(base_url): 141 | client = AsyncAzureOpenAI( 142 | api_key=api_key, 143 | azure_endpoint=base_url, 144 | api_version=api_version, 145 | organization=organization, 146 | http_client=get_async_client(hass), 147 | ) 148 | else: 149 | client = AsyncOpenAI( 150 | api_key=api_key, 151 | base_url=base_url, 152 | organization=organization, 153 | http_client=get_async_client(hass), 154 | ) 155 | 156 | await hass.async_add_executor_job(partial(client.models.list, timeout=10)) 157 | 158 | 159 | class FunctionExecutor(ABC): 160 | def __init__(self, data_schema=vol.Schema({})) -> None: 161 | """initialize function executor""" 162 | self.data_schema = data_schema.extend({vol.Required("type"): str}) 163 | 164 | def to_arguments(self, arguments): 165 | """to_arguments function""" 166 | try: 167 | return self.data_schema(arguments) 168 | except vol.error.Error as e: 169 | function_type = next( 170 | (key for key, value in FUNCTION_EXECUTORS.items() if value == self), 171 | None, 172 | ) 173 | raise InvalidFunction(function_type) from e 174 | 175 | def validate_entity_ids(self, hass: HomeAssistant, entity_ids, exposed_entities): 176 | if any(hass.states.get(entity_id) is None for entity_id in entity_ids): 177 | raise EntityNotFound(entity_ids) 178 | exposed_entity_ids = map(lambda e: e["entity_id"], exposed_entities) 179 | if not set(entity_ids).issubset(exposed_entity_ids): 180 | raise EntityNotExposed(entity_ids) 181 | 182 | @abstractmethod 183 | async def execute( 184 | self, 185 | hass: HomeAssistant, 186 | function, 187 | arguments, 188 | user_input: conversation.ConversationInput, 189 | exposed_entities, 190 | ): 191 | """execute function""" 192 | 193 | 194 | class NativeFunctionExecutor(FunctionExecutor): 195 | def __init__(self) -> None: 196 | """initialize native function""" 197 | super().__init__(vol.Schema({vol.Required("name"): str})) 198 | 199 | async def execute( 200 | self, 201 | hass: HomeAssistant, 202 | function, 203 | arguments, 204 | user_input: conversation.ConversationInput, 205 | exposed_entities, 206 | ): 207 | name = function["name"] 208 | if name == "execute_service": 209 | return await self.execute_service( 210 | hass, function, arguments, user_input, exposed_entities 211 | ) 212 | if name == "execute_service_single": 213 | return await self.execute_service_single( 214 | hass, function, arguments, user_input, exposed_entities 215 | ) 216 | if name == "add_automation": 217 | return await self.add_automation( 218 | hass, function, arguments, user_input, exposed_entities 219 | ) 220 | if name == "get_history": 221 | return await self.get_history( 222 | hass, function, arguments, user_input, exposed_entities 223 | ) 224 | if name == "get_energy": 225 | return await self.get_energy( 226 | hass, function, arguments, user_input, exposed_entities 227 | ) 228 | if name == "get_statistics": 229 | return await self.get_statistics( 230 | hass, function, arguments, user_input, exposed_entities 231 | ) 232 | if name == "get_user_from_user_id": 233 | return await self.get_user_from_user_id( 234 | hass, function, arguments, user_input, exposed_entities 235 | ) 236 | 237 | raise NativeNotFound(name) 238 | 239 | async def execute_service_single( 240 | self, 241 | hass: HomeAssistant, 242 | function, 243 | service_argument, 244 | user_input: conversation.ConversationInput, 245 | exposed_entities, 246 | ): 247 | domain = service_argument["domain"] 248 | service = service_argument["service"] 249 | service_data = service_argument.get( 250 | "service_data", service_argument.get("data", {}) 251 | ) 252 | entity_id = service_data.get("entity_id", service_argument.get("entity_id")) 253 | area_id = service_data.get("area_id") 254 | device_id = service_data.get("device_id") 255 | 256 | if isinstance(entity_id, str): 257 | entity_id = [e.strip() for e in entity_id.split(",")] 258 | service_data["entity_id"] = entity_id 259 | 260 | if entity_id is None and area_id is None and device_id is None: 261 | raise CallServiceError(domain, service, service_data) 262 | if not hass.services.has_service(domain, service): 263 | raise ServiceNotFound(domain, service) 264 | self.validate_entity_ids(hass, entity_id or [], exposed_entities) 265 | 266 | try: 267 | await hass.services.async_call( 268 | domain=domain, 269 | service=service, 270 | service_data=service_data, 271 | ) 272 | return {"success": True} 273 | except HomeAssistantError as e: 274 | _LOGGER.error(e) 275 | return {"error": str(e)} 276 | 277 | async def execute_service( 278 | self, 279 | hass: HomeAssistant, 280 | function, 281 | arguments, 282 | user_input: conversation.ConversationInput, 283 | exposed_entities, 284 | ): 285 | result = [] 286 | for service_argument in arguments.get("list", []): 287 | result.append( 288 | await self.execute_service_single( 289 | hass, function, service_argument, user_input, exposed_entities 290 | ) 291 | ) 292 | return result 293 | 294 | async def add_automation( 295 | self, 296 | hass: HomeAssistant, 297 | function, 298 | arguments, 299 | user_input: conversation.ConversationInput, 300 | exposed_entities, 301 | ): 302 | automation_config = yaml.safe_load(arguments["automation_config"]) 303 | config = {"id": str(round(time.time() * 1000))} 304 | if isinstance(automation_config, list): 305 | config.update(automation_config[0]) 306 | if isinstance(automation_config, dict): 307 | config.update(automation_config) 308 | 309 | await _async_validate_config_item(hass, config, True, False) 310 | 311 | automations = [config] 312 | with open( 313 | os.path.join(hass.config.config_dir, AUTOMATION_CONFIG_PATH), 314 | "r", 315 | encoding="utf-8", 316 | ) as f: 317 | current_automations = yaml.safe_load(f.read()) 318 | 319 | with open( 320 | os.path.join(hass.config.config_dir, AUTOMATION_CONFIG_PATH), 321 | "a" if current_automations else "w", 322 | encoding="utf-8", 323 | ) as f: 324 | raw_config = yaml.dump(automations, allow_unicode=True, sort_keys=False) 325 | f.write("\n" + raw_config) 326 | 327 | await hass.services.async_call(automation.config.DOMAIN, SERVICE_RELOAD) 328 | hass.bus.async_fire( 329 | EVENT_AUTOMATION_REGISTERED, 330 | {"automation_config": config, "raw_config": raw_config}, 331 | ) 332 | return "Success" 333 | 334 | async def get_history( 335 | self, 336 | hass: HomeAssistant, 337 | function, 338 | arguments, 339 | user_input: conversation.ConversationInput, 340 | exposed_entities, 341 | ): 342 | start_time = arguments.get("start_time") 343 | end_time = arguments.get("end_time") 344 | entity_ids = arguments.get("entity_ids", []) 345 | include_start_time_state = arguments.get("include_start_time_state", True) 346 | significant_changes_only = arguments.get("significant_changes_only", True) 347 | minimal_response = arguments.get("minimal_response", True) 348 | no_attributes = arguments.get("no_attributes", True) 349 | 350 | now = dt_util.utcnow() 351 | one_day = timedelta(days=1) 352 | start_time = self.as_utc(start_time, now - one_day, "start_time not valid") 353 | end_time = self.as_utc(end_time, start_time + one_day, "end_time not valid") 354 | 355 | self.validate_entity_ids(hass, entity_ids, exposed_entities) 356 | 357 | with recorder.util.session_scope(hass=hass, read_only=True) as session: 358 | result = await recorder.get_instance(hass).async_add_executor_job( 359 | recorder.history.get_significant_states_with_session, 360 | hass, 361 | session, 362 | start_time, 363 | end_time, 364 | entity_ids, 365 | None, 366 | include_start_time_state, 367 | significant_changes_only, 368 | minimal_response, 369 | no_attributes, 370 | ) 371 | 372 | return [[self.as_dict(item) for item in sublist] for sublist in result.values()] 373 | 374 | async def get_energy( 375 | self, 376 | hass: HomeAssistant, 377 | function, 378 | arguments, 379 | user_input: conversation.ConversationInput, 380 | exposed_entities, 381 | ): 382 | energy_manager: energy.data.EnergyManager = await energy.async_get_manager(hass) 383 | return energy_manager.data 384 | 385 | async def get_user_from_user_id( 386 | self, 387 | hass: HomeAssistant, 388 | function, 389 | arguments, 390 | user_input: conversation.ConversationInput, 391 | exposed_entities, 392 | ): 393 | user = await hass.auth.async_get_user(user_input.context.user_id) 394 | return {'name': user.name if user and hasattr(user, 'name') else 'Unknown'} 395 | 396 | async def get_statistics( 397 | self, 398 | hass: HomeAssistant, 399 | function, 400 | arguments, 401 | user_input: conversation.ConversationInput, 402 | exposed_entities, 403 | ): 404 | statistic_ids = arguments.get("statistic_ids", []) 405 | start_time = dt_util.as_utc(dt_util.parse_datetime(arguments["start_time"])) 406 | end_time = dt_util.as_utc(dt_util.parse_datetime(arguments["end_time"])) 407 | 408 | return await recorder.get_instance(hass).async_add_executor_job( 409 | recorder.statistics.statistics_during_period, 410 | hass, 411 | start_time, 412 | end_time, 413 | statistic_ids, 414 | arguments.get("period", "day"), 415 | arguments.get("units"), 416 | arguments.get("types", {"change"}), 417 | ) 418 | 419 | def as_utc(self, value: str, default_value, parse_error_message: str): 420 | if value is None: 421 | return default_value 422 | 423 | parsed_datetime = dt_util.parse_datetime(value) 424 | if parsed_datetime is None: 425 | raise HomeAssistantError(parse_error_message) 426 | 427 | return dt_util.as_utc(parsed_datetime) 428 | 429 | def as_dict(self, state: State | dict[str, Any]): 430 | if isinstance(state, State): 431 | return state.as_dict() 432 | return state 433 | 434 | 435 | class ScriptFunctionExecutor(FunctionExecutor): 436 | def __init__(self) -> None: 437 | """initialize script function""" 438 | super().__init__(SCRIPT_ENTITY_SCHEMA) 439 | 440 | async def execute( 441 | self, 442 | hass: HomeAssistant, 443 | function, 444 | arguments, 445 | user_input: conversation.ConversationInput, 446 | exposed_entities, 447 | ): 448 | script = Script( 449 | hass, 450 | function["sequence"], 451 | "extended_openai_conversation", 452 | DOMAIN, 453 | running_description="[extended_openai_conversation] function", 454 | logger=_LOGGER, 455 | ) 456 | 457 | result = await script.async_run( 458 | run_variables=arguments, context=user_input.context 459 | ) 460 | return result.variables.get("_function_result", "Success") 461 | 462 | 463 | class TemplateFunctionExecutor(FunctionExecutor): 464 | def __init__(self) -> None: 465 | """initialize template function""" 466 | super().__init__( 467 | vol.Schema( 468 | { 469 | vol.Required("value_template"): cv.template, 470 | vol.Optional("parse_result"): bool, 471 | } 472 | ) 473 | ) 474 | 475 | async def execute( 476 | self, 477 | hass: HomeAssistant, 478 | function, 479 | arguments, 480 | user_input: conversation.ConversationInput, 481 | exposed_entities, 482 | ): 483 | return function["value_template"].async_render( 484 | arguments, 485 | parse_result=function.get("parse_result", False), 486 | ) 487 | 488 | 489 | class RestFunctionExecutor(FunctionExecutor): 490 | def __init__(self) -> None: 491 | """initialize Rest function""" 492 | super().__init__( 493 | vol.Schema(rest.RESOURCE_SCHEMA).extend( 494 | { 495 | vol.Optional("value_template"): cv.template, 496 | vol.Optional("payload_template"): cv.template, 497 | } 498 | ) 499 | ) 500 | 501 | async def execute( 502 | self, 503 | hass: HomeAssistant, 504 | function, 505 | arguments, 506 | user_input: conversation.ConversationInput, 507 | exposed_entities, 508 | ): 509 | config = function 510 | rest_data = _get_rest_data(hass, config, arguments) 511 | 512 | await rest_data.async_update() 513 | value = rest_data.data_without_xml() 514 | value_template = config.get(CONF_VALUE_TEMPLATE) 515 | 516 | if value is not None and value_template is not None: 517 | value = value_template.async_render_with_possible_json_value( 518 | value, None, arguments 519 | ) 520 | 521 | return value 522 | 523 | 524 | class ScrapeFunctionExecutor(FunctionExecutor): 525 | def __init__(self) -> None: 526 | """initialize Scrape function""" 527 | super().__init__( 528 | scrape.COMBINED_SCHEMA.extend( 529 | { 530 | vol.Optional("value_template"): cv.template, 531 | vol.Optional("payload_template"): cv.template, 532 | } 533 | ) 534 | ) 535 | 536 | async def execute( 537 | self, 538 | hass: HomeAssistant, 539 | function, 540 | arguments, 541 | user_input: conversation.ConversationInput, 542 | exposed_entities, 543 | ): 544 | config = function 545 | rest_data = _get_rest_data(hass, config, arguments) 546 | coordinator = scrape.coordinator.ScrapeCoordinator( 547 | hass, 548 | rest_data, 549 | scrape.const.DEFAULT_SCAN_INTERVAL, 550 | ) 551 | await coordinator.async_config_entry_first_refresh() 552 | 553 | new_arguments = dict(arguments) 554 | 555 | for sensor_config in config["sensor"]: 556 | name: Template = sensor_config.get(CONF_NAME) 557 | value = self._async_update_from_rest_data( 558 | coordinator.data, sensor_config, arguments 559 | ) 560 | new_arguments["value"] = value 561 | if name: 562 | new_arguments[name.async_render()] = value 563 | 564 | result = new_arguments["value"] 565 | value_template = config.get(CONF_VALUE_TEMPLATE) 566 | 567 | if value_template is not None: 568 | result = value_template.async_render_with_possible_json_value( 569 | result, None, new_arguments 570 | ) 571 | 572 | return result 573 | 574 | def _async_update_from_rest_data( 575 | self, 576 | data: BeautifulSoup, 577 | sensor_config: dict[str, Any], 578 | arguments: dict[str, Any], 579 | ) -> None: 580 | """Update state from the rest data.""" 581 | value = self._extract_value(data, sensor_config) 582 | value_template = sensor_config.get(CONF_VALUE_TEMPLATE) 583 | 584 | if value_template is not None: 585 | value = value_template.async_render_with_possible_json_value( 586 | value, None, arguments 587 | ) 588 | 589 | return value 590 | 591 | def _extract_value(self, data: BeautifulSoup, sensor_config: dict[str, Any]) -> Any: 592 | """Parse the html extraction in the executor.""" 593 | value: str | list[str] | None 594 | select = sensor_config[scrape.const.CONF_SELECT] 595 | index = sensor_config.get(scrape.const.CONF_INDEX, 0) 596 | attr = sensor_config.get(CONF_ATTRIBUTE) 597 | try: 598 | if attr is not None: 599 | value = data.select(select)[index][attr] 600 | else: 601 | tag = data.select(select)[index] 602 | if tag.name in ("style", "script", "template"): 603 | value = tag.string 604 | else: 605 | value = tag.text 606 | except IndexError: 607 | _LOGGER.warning("Index '%s' not found", index) 608 | value = None 609 | except KeyError: 610 | _LOGGER.warning("Attribute '%s' not found", attr) 611 | value = None 612 | _LOGGER.debug("Parsed value: %s", value) 613 | return value 614 | 615 | 616 | class CompositeFunctionExecutor(FunctionExecutor): 617 | def __init__(self) -> None: 618 | """initialize composite function""" 619 | super().__init__( 620 | vol.Schema( 621 | { 622 | vol.Required("sequence"): vol.All( 623 | cv.ensure_list, [self.function_schema] 624 | ) 625 | } 626 | ) 627 | ) 628 | 629 | def function_schema(self, value: Any) -> dict: 630 | """Validate a composite function schema.""" 631 | if not isinstance(value, dict): 632 | raise vol.Invalid("expected dictionary") 633 | 634 | composite_schema = {vol.Optional("response_variable"): str} 635 | function_executor = get_function_executor(value["type"]) 636 | 637 | return function_executor.data_schema.extend(composite_schema)(value) 638 | 639 | async def execute( 640 | self, 641 | hass: HomeAssistant, 642 | function, 643 | arguments, 644 | user_input: conversation.ConversationInput, 645 | exposed_entities, 646 | ): 647 | config = function 648 | sequence = config["sequence"] 649 | 650 | for executor_config in sequence: 651 | function_executor = get_function_executor(executor_config["type"]) 652 | result = await function_executor.execute( 653 | hass, executor_config, arguments, user_input, exposed_entities 654 | ) 655 | 656 | response_variable = executor_config.get("response_variable") 657 | if response_variable: 658 | arguments[response_variable] = result 659 | 660 | return result 661 | 662 | 663 | class SqliteFunctionExecutor(FunctionExecutor): 664 | def __init__(self) -> None: 665 | """initialize sqlite function""" 666 | super().__init__( 667 | vol.Schema( 668 | { 669 | vol.Optional("query"): str, 670 | vol.Optional("db_url"): str, 671 | vol.Optional("single"): bool, 672 | } 673 | ) 674 | ) 675 | 676 | def is_exposed(self, entity_id, exposed_entities) -> bool: 677 | return any( 678 | exposed_entity["entity_id"] == entity_id 679 | for exposed_entity in exposed_entities 680 | ) 681 | 682 | def is_exposed_entity_in_query(self, query: str, exposed_entities) -> bool: 683 | exposed_entity_ids = list( 684 | map(lambda e: f"'{e['entity_id']}'", exposed_entities) 685 | ) 686 | return any( 687 | exposed_entity_id in query for exposed_entity_id in exposed_entity_ids 688 | ) 689 | 690 | def raise_error(self, msg="Unexpected error occurred."): 691 | raise HomeAssistantError(msg) 692 | 693 | def get_default_db_url(self, hass: HomeAssistant) -> str: 694 | db_file_path = os.path.join(hass.config.config_dir, recorder.DEFAULT_DB_FILE) 695 | return f"file:{db_file_path}?mode=ro" 696 | 697 | def set_url_read_only(self, url: str) -> str: 698 | scheme, netloc, path, query_string, fragment = parse.urlsplit(url) 699 | query_params = parse.parse_qs(query_string) 700 | 701 | query_params["mode"] = ["ro"] 702 | new_query_string = parse.urlencode(query_params, doseq=True) 703 | 704 | return parse.urlunsplit((scheme, netloc, path, new_query_string, fragment)) 705 | 706 | async def execute( 707 | self, 708 | hass: HomeAssistant, 709 | function, 710 | arguments, 711 | user_input: conversation.ConversationInput, 712 | exposed_entities, 713 | ): 714 | db_url = self.set_url_read_only( 715 | function.get("db_url", self.get_default_db_url(hass)) 716 | ) 717 | query = function.get("query", "{{query}}") 718 | 719 | template_arguments = { 720 | "is_exposed": lambda e: self.is_exposed(e, exposed_entities), 721 | "is_exposed_entity_in_query": lambda q: self.is_exposed_entity_in_query( 722 | q, exposed_entities 723 | ), 724 | "exposed_entities": exposed_entities, 725 | "raise": self.raise_error, 726 | } 727 | template_arguments.update(arguments) 728 | 729 | q = Template(query, hass).async_render(template_arguments) 730 | _LOGGER.info("Rendered query: %s", q) 731 | 732 | with sqlite3.connect(db_url, uri=True) as conn: 733 | cursor = conn.cursor().execute(q) 734 | names = [description[0] for description in cursor.description] 735 | 736 | if function.get("single") is True: 737 | row = cursor.fetchone() 738 | return {name: val for name, val in zip(names, row)} 739 | 740 | rows = cursor.fetchall() 741 | result = [] 742 | for row in rows: 743 | result.append({name: val for name, val in zip(names, row)}) 744 | return result 745 | 746 | 747 | FUNCTION_EXECUTORS: dict[str, FunctionExecutor] = { 748 | "native": NativeFunctionExecutor(), 749 | "script": ScriptFunctionExecutor(), 750 | "template": TemplateFunctionExecutor(), 751 | "rest": RestFunctionExecutor(), 752 | "scrape": ScrapeFunctionExecutor(), 753 | "composite": CompositeFunctionExecutor(), 754 | "sqlite": SqliteFunctionExecutor(), 755 | } 756 | --------------------------------------------------------------------------------