community[patch]: Update tongyi.py to support MultimodalConversation in dashscope. (#21249)

Add the support of multimodal conversation in dashscope,now we can use
multimodal language model "qwen-vl-v1", "qwen-vl-chat-v1",
"qwen-audio-turbo" to processing picture an audio. :)

- [ ] **PR title**: "community: add multimodal conversation support in
dashscope"



- [ ] **PR message**: ***Delete this entire checklist*** and replace
with
    - **Description:** add multimodal conversation support in dashscope
    - **Issue:** 
    - **Dependencies:** dashscope≥1.18.0
    - **Twitter handle:** none :)


- [ ] **How to use it?**:
   - ```python
     Tongyi_chat = ChatTongyi(
        top_p=0.5,
        dashscope_api_key=api_key,
        model="qwen-vl-v1"
     )
     response= Tongyi_chat.invoke(
        input = 
        [
        {
            "role": "user",
            "content": [
{"image":
"https://dashscope.oss-cn-beijing.aliyuncs.com/images/dog_and_girl.jpeg"},
                {"text": "这是什么?"}
            ]
        }
        ]
       )
      ```

---------

Co-authored-by: Bagatur <baskaryan@gmail.com>
This commit is contained in:
HuiyuanYan 2024-05-23 06:04:58 +08:00 committed by GitHub
parent 63284ffebf
commit bf3aefce93
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

View File

@ -241,8 +241,14 @@ class ChatTongyi(BaseChatModel):
client: Any #: :meta private:
model_name: str = Field(default="qwen-turbo", alias="model")
"""Model name to use."""
"""Model name to use.
callable multimodal model:
- qwen-vl-v1
- qwen-vl-chat-v1
- qwen-audio-turbo
- qwen-vl-plus
- qwen-vl-max
"""
model_kwargs: Dict[str, Any] = Field(default_factory=dict)
top_p: float = 0.8
@ -280,10 +286,27 @@ class ChatTongyi(BaseChatModel):
"Could not import dashscope python package. "
"Please install it with `pip install dashscope --upgrade`."
)
dashscope_multimodal_models = [
"qwen-vl-v1",
"qwen-vl-chat-v1",
"qwen-audio-turbo",
"qwen-vl-plus",
"qwen-vl-max",
]
if (
values["model_name"] in dashscope_multimodal_models
or "vl" in values["model_name"]
):
try:
if "vl" in values["model_name"]:
values["client"] = dashscope.MultiModalConversation
except AttributeError:
raise ValueError(
"`dashscope` has no `MultiModalConversation` attribute, this is "
"likely due to an old version of the dashscope package. Try "
"upgrading it with `pip install --upgrade dashscope`."
)
else:
try:
values["client"] = dashscope.Generation
except AttributeError:
raise ValueError(
@ -291,7 +314,6 @@ class ChatTongyi(BaseChatModel):
"due to an old version of the dashscope package. Try upgrading it "
"with `pip install --upgrade dashscope`."
)
return values
@property