Update multi-modal support and platform compatibility (#25)

Co-authored-by: wangyuxin <[email protected]>

README.md

@@ -19,11 +19,11 @@ Generate Package 允许用户通过统一的 api 访问跨平台的生成式模
 
 ## Features
 
-* **多模态**，支持文本生成，图像生成以及语音生成
-* **跨平台**，完整支持 OpenAI，Azure，Minimax 在内的国内外多家平台
-* **One API**，统一了不同平台的消息格式，推理参数，接口封装，返回解析
+* **多模态**，支持文本生成，多模态文本生成，结构体生成，图像生成，语音生成...
+* **跨平台**，完整支持 OpenAI，Azure，Minimax，智谱，文心一言 在内的国内外多家平台
+* **One API**，统一了不同平台的消息格式，推理参数，接口封装，返回解析，让用户无需关心不同平台的差异
 * **异步和流式**，提供流式调用，非流式调用，同步调用，异步调用，异步批量调用，适配不同的应用场景
-* **自带电池**，提供输入检查，参数检查，计费，*ChatEngine*, *Limit*, *Structure*, *function* 等功能
+* **自带电池**，提供输入检查，参数检查，计费，速率控制，*ChatEngine*, *function call* 等功能
 * **高质量代码**，100% typehints，pylance strict, ruff lint & format,  test coverage > 85% ...
 
 > 完整支持是指，只要是平台提供的功能和参数，`generate` 包都原生支持，不打折扣！比如，OpenAI 的 Function Call, Tool Calls，MinimaxPro 的 Plugins 等

generate/chat_completion/message/__init__.py

@@ -2,6 +2,7 @@
     AssistantMessage,
     FunctionCall,
     FunctionMessage,
+    ImagePart,
     ImageUrl,
     ImageUrlPart,
     Message,
@@ -32,6 +33,7 @@
     'ensure_messages',
     'FunctionCall',
     'FunctionMessage',
+    'ImagePart',
     'Message',
     'Messages',
     'Prompt',

generate/chat_completion/message/core.py

@@ -34,9 +34,14 @@ class ImageUrlPart(BaseModel):
     image_url: ImageUrl
 
 
+class ImagePart(BaseModel):
+    image: bytes
+    image_format: Optional[str] = None
+
+
 class UserMultiPartMessage(Message):
     role: Literal['user'] = 'user'
-    content: List[Union[TextPart, ImageUrlPart]]
+    content: List[Union[TextPart, ImageUrlPart, ImagePart]]
 
 
 class FunctionMessage(Message):

generate/chat_completion/models/dashscope.py

@@ -241,7 +241,7 @@ def _convert_message_to_multimodal_chat_message(message: Message) -> DashScopeMu
             elif isinstance(part, ImageUrlPart):
                 content.append({'image': part.image_url.url})
             else:
-                raise TypeError(f'Unknown part type: {part}')
+                raise TypeError(f'Unsupported part type: {part}')
         return {'role': 'user', 'content': content}
     allowed_message_type = (UserMessage, AssistantMessage, SystemMessage, UserMultiPartMessage)
     raise MessageTypeError(message, allowed_message_type=allowed_message_type)

generate/chat_completion/models/openai.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import base64
 import json
 from functools import partial
 from typing import Any, AsyncIterator, Callable, ClassVar, Dict, Iterator, List, Literal, Optional, Type, Union, cast
@@ -13,6 +14,7 @@
     AssistantMessage,
     FunctionCall,
     FunctionMessage,
+    ImagePart,
     Message,
     Messages,
     MessageTypeError,
@@ -123,14 +125,19 @@ def _to_user_multipart_message_dict(message: UserMultiPartMessage) -> OpenAIMess
         if isinstance(part, TextPart):
             content.append({'type': 'text', 'text': part.text})
         else:
+            if isinstance(part, ImagePart):
+                image_format = part.image_format or 'png'
+                url: str = f'data:image/{image_format};base64,{base64.b64encode(part.image).decode()}'
+                image_url_dict = {'url': url}
+            else:
+                image_url_dict = {}
+                image_url_dict['url'] = part.image_url.url
+                if part.image_url.detail:
+                    image_url_dict['detail'] = part.image_url.detail
             image_url_part_dict: dict[str, Any] = {
                 'type': 'image_url',
-                'image_url': {
-                    'url': part.image_url.url,
-                },
+                'image_url': image_url_dict,
             }
-            if part.image_url.detail:
-                image_url_part_dict['image_url']['detail'] = part.image_url.detail
             content.append(image_url_part_dict)
     return {
         'role': 'user',

generate/chat_completion/models/zhipu.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import base64
 import json
 from typing import Any, AsyncIterator, ClassVar, Dict, Iterator, List, Literal, Optional, Union
 
@@ -10,17 +11,20 @@
 from generate.chat_completion.message import (
     AssistantMessage,
     FunctionCall,
+    ImagePart,
+    ImageUrlPart,
     Message,
     Messages,
     MessageTypeError,
     Prompt,
     SystemMessage,
+    TextPart,
     ToolCall,
+    ToolMessage,
     UserMessage,
     UserMultiPartMessage,
     ensure_messages,
 )
-from generate.chat_completion.message.core import ImageUrlPart, TextPart, ToolMessage
 from generate.chat_completion.model_output import ChatCompletionOutput, ChatCompletionStreamOutput, Stream
 from generate.http import (
     HttpClient,
@@ -140,6 +144,15 @@ def convert_to_zhipu_message(message: Message) -> ZhipuMessage:
                         },
                     }
                 )
+            elif isinstance(part, ImagePart):
+                content.append(
+                    {
+                        'type': 'image_url',
+                        'image_url': {
+                            'url': base64.b64encode(part.image).decode(),
+                        },
+                    }
+                )
         return {'role': 'user', 'content': content}
 
     if isinstance(message, AssistantMessage):

pyproject.toml

@@ -55,6 +55,7 @@ ignore = [
     "PLR0913", # Too many arguments in function definition
     "TRY003", # Avoid specifying long messages outside the exception class
     "PLC0414", # reimport
+    'PLR0912', # too many branches
 ]
 exclude = ["playground", "api_docs"]
 target-version = "py38"