Add Open AI integration

README.md

@@ -82,6 +82,18 @@ MA requires a 64bit Operating System and a minimum of 2GB of RAM on the physical
 NOTE: You need to set-up the players and music sources within Music Assistant itself.
 If you are running Music Assistant in docker, you need to access the webinterface at http://youripaddress:8095, when running the Home Assistant add-on, you can access the webinterface from the add-on (and even show that in the sidebar).
 
+## OpenAI features
+
+During [Chapter 5 of "Year of the Voice"](https://www.youtube.com/live/djEkgoS5dDQ?si=pt8-qYH3PTpsnOq9&t=3699), [JLo](https://blog.jlpouffier.fr/chatgpt-powered-music-search-engine-on-a-local-voice-assistant/) showed something he had been working on to use the OpenAI integration along with Music Assistant. We now have this feature baked in to the integration code directly, although some extra setup is still required.
+- You need to create/add another OpenAI integration that is purely for Music Assistant.
+- Add the prompt found [here](https://github.com/jozefKruszynski/home-assistant-things/blob/main/blueprints/modified_prompt.txt) to the configuration of the the OpenAI integration.
+- Add a directory in your Home Assistant `config` dir name `custom_sentences/en`
+- Add the file found [here](https://github.com/jozefKruszynski/home-assistant-things/blob/main/custom_sentences/en/play_media_on_mass.yaml), to that dir.
+- When setting up the Music Assistant integration, make sure that you select the correct Conversation Agent and also
+allow the auto-exposure of Mass media players to Assist
+
+![Preview image](https://raw.githubusercontent.com/music-assistant/hass-music-assistant/main/screenshots/screen6.png)
+
 ## Usage and notes
 
 - Music from your music sources will be automatically loaded into the Music Assistant library. If you have multiple sources, they will be merged as one library.

custom_components/mass/config_flow.py

@@ -18,15 +18,17 @@
 from homeassistant.core import HomeAssistant
 from homeassistant.data_entry_flow import AbortFlow, FlowResult
 from homeassistant.exceptions import HomeAssistantError
-from homeassistant.helpers import aiohttp_client
+from homeassistant.helpers import aiohttp_client, selector
 from music_assistant.client import MusicAssistantClient
 from music_assistant.client.exceptions import CannotConnect, InvalidServerVersion
 from music_assistant.common.models.api import ServerInfoMessage
 
 from .addon import get_addon_manager, install_repository
 from .const import (
     ADDON_HOSTNAME,
+    CONF_ASSIST_AUTO_EXPOSE_PLAYERS,
     CONF_INTEGRATION_CREATED_ADDON,
+    CONF_OPENAI_AGENT_ID,
     CONF_USE_ADDON,
     DOMAIN,
     LOGGER,
@@ -37,13 +39,42 @@
 DEFAULT_URL = "http://mass.local:8095"
 ADDON_URL = f"http://{ADDON_HOSTNAME}:8095"
 DEFAULT_TITLE = "Music Assistant"
-ON_SUPERVISOR_SCHEMA = vol.Schema({vol.Optional(CONF_USE_ADDON, default=True): bool})
+
+ON_SUPERVISOR_SCHEMA = vol.Schema(
+    {
+        vol.Optional(CONF_USE_ADDON, default=True): bool,
+        vol.Optional(CONF_OPENAI_AGENT_ID, default=None): selector.ConversationAgentSelector(
+            selector.ConversationAgentSelectorConfig(language="en")
+        ),
+        vol.Optional(CONF_ASSIST_AUTO_EXPOSE_PLAYERS, default=False): bool,
+    }
+)
 
 
 def get_manual_schema(user_input: dict[str, Any]) -> vol.Schema:
     """Return a schema for the manual step."""
     default_url = user_input.get(CONF_URL, DEFAULT_URL)
-    return vol.Schema({vol.Required(CONF_URL, default=default_url): str})
+    return vol.Schema(
+        {
+            vol.Required(CONF_URL, default=default_url): str,
+            vol.Optional(CONF_OPENAI_AGENT_ID, default=None): selector.ConversationAgentSelector(
+                selector.ConversationAgentSelectorConfig(language="en")
+            ),
+            vol.Optional(CONF_ASSIST_AUTO_EXPOSE_PLAYERS, default=False): bool,
+        }
+    )
+
+
+def get_zeroconf_schema() -> vol.Schema:
+    """Return a schema for the zeroconf step."""
+    return vol.Schema(
+        {
+            vol.Optional(CONF_OPENAI_AGENT_ID, default=None): selector.ConversationAgentSelector(
+                selector.ConversationAgentSelectorConfig(language="en")
+            ),
+            vol.Optional(CONF_ASSIST_AUTO_EXPOSE_PLAYERS, default=False): bool,
+        }
+    )
 
 
 async def get_server_info(hass: HomeAssistant, url: str) -> ServerInfoMessage:
@@ -63,6 +94,8 @@ class ConfigFlow(config_entries.ConfigFlow, domain=DOMAIN):
     def __init__(self) -> None:
         """Set up flow instance."""
         self.server_info: ServerInfoMessage | None = None
+        self.openai_agent_id: str | None = None
+        self.expose_players_assist: bool | None = None
         # If we install the add-on we should uninstall it on entry remove.
         self.integration_created_addon = False
         self.install_task: asyncio.Task | None = None
@@ -182,6 +215,8 @@ async def async_step_manual(self, user_input: dict[str, Any] | None = None) -> F
 
         try:
             self.server_info = await get_server_info(self.hass, user_input[CONF_URL])
+            self.openai_agent_id = user_input[CONF_OPENAI_AGENT_ID]
+            self.expose_players_assist = user_input[CONF_ASSIST_AUTO_EXPOSE_PLAYERS]
             await self.async_set_unique_id(self.server_info.server_id)
         except CannotConnect:
             errors["base"] = "cannot_connect"
@@ -223,12 +258,15 @@ async def async_step_discovery_confirm(
         if user_input is not None:
             # Check that we can connect to the address.
             try:
+                self.openai_agent_id = user_input[CONF_OPENAI_AGENT_ID]
+                self.expose_players_assist = user_input[CONF_ASSIST_AUTO_EXPOSE_PLAYERS]
                 await get_server_info(self.hass, self.server_info.base_url)
             except CannotConnect:
                 return self.async_abort(reason="cannot_connect")
             return await self._async_create_entry_or_abort()
         return self.async_show_form(
             step_id="discovery_confirm",
+            data_schema=get_zeroconf_schema(),
             description_placeholders={"url": self.server_info.base_url},
         )
 
@@ -279,6 +317,8 @@ async def _async_create_entry_or_abort(self) -> FlowResult:
                     CONF_URL: self.server_info.base_url,
                     CONF_USE_ADDON: self.use_addon,
                     CONF_INTEGRATION_CREATED_ADDON: self.integration_created_addon,
+                    CONF_OPENAI_AGENT_ID: self.openai_agent_id,
+                    CONF_ASSIST_AUTO_EXPOSE_PLAYERS: self.expose_players_assist,
                 },
                 title=DEFAULT_TITLE,
             )
@@ -295,6 +335,8 @@ async def _async_create_entry_or_abort(self) -> FlowResult:
                 CONF_URL: self.server_info.base_url,
                 CONF_USE_ADDON: self.use_addon,
                 CONF_INTEGRATION_CREATED_ADDON: self.integration_created_addon,
+                CONF_OPENAI_AGENT_ID: self.openai_agent_id,
+                CONF_ASSIST_AUTO_EXPOSE_PLAYERS: self.expose_players_assist,
             },
         )
 

custom_components/mass/const.py

@@ -22,5 +22,7 @@
 
 CONF_INTEGRATION_CREATED_ADDON = "integration_created_addon"
 CONF_USE_ADDON = "use_addon"
+CONF_OPENAI_AGENT_ID = "openai_agent_id"
+CONF_ASSIST_AUTO_EXPOSE_PLAYERS = "expose_players_assist"
 
 LOGGER = logging.getLogger(__package__)

custom_components/mass/intent.py

@@ -0,0 +1,90 @@
+"""Intents for the client integration."""
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING, Any
+
+import voluptuous as vol
+from homeassistant.config_entries import ConfigEntryState
+from homeassistant.core import HomeAssistant
+from homeassistant.helpers import config_validation as cv
+from homeassistant.helpers import intent
+
+from . import DOMAIN
+from .media_player import MassPlayer
+
+if TYPE_CHECKING:
+    pass
+if TYPE_CHECKING:
+    from music_assistant.client import MusicAssistantClient
+
+
+INTENT_PLAY_MEDIA_ON_MEDIA_PLAYER = "MassPlayMediaOnMediaPlayerNameEn"
+CONVERSATION_DOMAIN = "conversation"
+CONVERSATION_SERVICE = "process"
+
+
+async def async_setup_intents(hass: HomeAssistant) -> None:
+    """Set up the climate intents."""
+    intent.async_register(hass, MassPlayMediaOnMediaPlayerNameEn())
+
+
+class MassPlayMediaOnMediaPlayerNameEn(intent.IntentHandler):
+    """Handle PlayMediaOnMediaPlayer intents."""
+
+    intent_type = INTENT_PLAY_MEDIA_ON_MEDIA_PLAYER
+    slot_schema = {vol.Optional("query"): cv.string, vol.Optional("name"): cv.string}
+
+    async def async_handle(self, intent_obj: intent.Intent) -> intent.IntentResponse:
+        """Handle the intent."""
+        hass = intent_obj.hass
+        service_data: dict[str, Any] = {}
+        slots = self.async_validate_slots(intent_obj.slots)
+
+        query: str = slots.get("query", {}).get("value")
+        if query is not None:
+            config_entry = await self.get_loaded_config_entry(hass)
+            service_data["agent_id"] = config_entry.data.get("openai_agent_id")
+            service_data["text"] = query
+
+        name: str = slots.get("name", {}).get("value")
+        if name is not None:
+            media_id, media_type = await self.get_media_id_and_media_type_from_query_result(
+                hass, service_data, intent_obj
+            )
+            mass: MusicAssistantClient = hass.data[DOMAIN][config_entry.entry_id].mass
+            players = await mass.players.get_players()
+            for player in players:
+                if player.name.casefold() == name.casefold():
+                    actual_player = MassPlayer(mass, player.player_id)
+                    actual_player.async_play_media(media_id=media_id, media_type=media_type)
+
+        response = intent_obj.create_response()
+        response.response_type = intent.IntentResponseType.ACTION_DONE
+        response.async_set_speech(f"Playing selection on {actual_player.name}")
+        return response
+
+    async def get_media_id_and_media_type_from_query_result(
+        self, hass: HomeAssistant, service_data: dict[str, Any], intent_obj: intent.Intent
+    ) -> str:
+        """Get  from the query."""
+        ai_response = await hass.services.async_call(
+            CONVERSATION_DOMAIN,
+            CONVERSATION_SERVICE,
+            {**service_data},
+            blocking=True,
+            context=intent_obj.context,
+            return_response=True,
+        )
+        json_payload = json.loads(ai_response["response"]["speech"]["plain"]["speech"])
+        media_id = json_payload.get("media_id")
+        media_type = json_payload.get("media_type")
+        return media_id, media_type
+
+    async def get_loaded_config_entry(self, hass: HomeAssistant) -> str:
+        """Get the correct config entry."""
+        config_entries = hass.config_entries.async_entries(DOMAIN)
+        for config_entry in config_entries:
+            if config_entry.state == ConfigEntryState.LOADED:
+                return config_entry
+        return None

custom_components/mass/media_player.py

@@ -8,6 +8,7 @@
 import homeassistant.helpers.config_validation as cv
 import voluptuous as vol
 from homeassistant.components import media_source
+from homeassistant.components.homeassistant.exposed_entities import async_expose_entity
 from homeassistant.components.media_player import (
     BrowseMedia,
     MediaPlayerDeviceClass,
@@ -21,7 +22,7 @@
     ATTR_MEDIA_EXTRA,
     MediaPlayerEntityFeature,
 )
-from homeassistant.config_entries import ConfigEntry
+from homeassistant.config_entries import ConfigEntry, ConfigEntryState
 from homeassistant.const import STATE_IDLE, STATE_OFF, STATE_PAUSED, STATE_PLAYING
 from homeassistant.core import HomeAssistant
 from homeassistant.helpers.entity_platform import AddEntitiesCallback, async_get_current_platform
@@ -165,6 +166,10 @@ def __init__(self, mass: MusicAssistantClient, player_id: str) -> None:
     async def async_added_to_hass(self) -> None:
         """Register callbacks."""
         await super().async_added_to_hass()
+        # we need to get the hass object in order to get our config entry
+        # and expose the player to the conversation component, assuming that
+        # the config entry has the option enabled.
+        await self._expose_players_assist()
 
         # we subscribe to player queue time update but we only
         # accept a state change on big time jumps (e.g. seeking)
@@ -512,3 +517,13 @@ async def _get_item_by_name(
                 # simply return the first item because search is already sorted by best match
                 return item
         return None
+
+    async def _expose_players_assist(self) -> None:
+        """Get the correct config entry."""
+        hass = self.hass
+        config_entries = hass.config_entries.async_entries(DOMAIN)
+        for config_entry in config_entries:
+            if config_entry.state == ConfigEntryState.SETUP_IN_PROGRESS and config_entry.data.get(
+                "expose_players_assist"
+            ):
+                async_expose_entity(hass, "conversation", self.entity_id, True)

custom_components/mass/strings.json

@@ -14,14 +14,21 @@
     "step": {
       "manual": {
         "data": {
-          "url": "URL of the Music Assistant server"
+          "url": "URL of the Music Assistant server",
+          "expose_players_assist": "Expose players to Assist"
+        }
+      },
+      "zeroconf": {
+        "data": {
+          "expose_players_assist": "Expose players to Assist"
         }
       },
       "on_supervisor": {
         "title": "Select connection method",
         "description": "Do you want to use the official Music Assistant Server add-on?\n\nIf you are already running the Music Assistant Server in another add-on, in a custom container, natively etc., then do not select this option.",
         "data": {
-          "use_addon": "Use the official Music Assistant Server add-on"
+          "use_addon": "Use the official Music Assistant Server add-on",
+          "expose_players_assist": "Expose players to Assist"
         }
       },
       "install_addon": {

custom_components/mass/translations/en.json

@@ -14,14 +14,21 @@
     "step": {
       "manual": {
         "data": {
-          "url": "URL of the Music Assistant server"
+          "url": "URL of the Music Assistant server",
+          "expose_players_assist": "Expose players to Assist"
+        }
+      },
+      "zeroconf": {
+        "data": {
+          "expose_players_assist": "Expose players to Assist"
         }
       },
       "on_supervisor": {
         "title": "Select connection method",
         "description": "Do you want to use the official Music Assistant Server add-on?\n\nIf you are already running the Music Assistant Server in another add-on, in a custom container, natively etc., then do not select this option.",
         "data": {
-          "use_addon": "Use the official Music Assistant Server add-on"
+          "use_addon": "Use the official Music Assistant Server add-on",
+          "expose_players_assist": "Expose players to Assist"
         }
       },
       "install_addon": {