From bf3aefce9369f117ae074599e21f3a17c3b254db Mon Sep 17 00:00:00 2001
From: HuiyuanYan <92134290+HuiyuanYan@users.noreply.github.com>
Date: Thu, 23 May 2024 06:04:58 +0800
Subject: [PATCH] community[patch]: Update tongyi.py to support
 MultimodalConversation in dashscope. (#21249)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add the support of multimodal conversation in dashscope,now we can use
multimodal language model "qwen-vl-v1", "qwen-vl-chat-v1",
"qwen-audio-turbo" to processing picture an audio. :)

- [ ] **PR title**: "community: add multimodal conversation support in
dashscope"



- [ ] **PR message**: ***Delete this entire checklist*** and replace
with
    - **Description:** add multimodal conversation support in dashscope
    - **Issue:**
    - **Dependencies:** dashscope≥1.18.0
    - **Twitter handle:** none :)


- [ ] **How to use it?**:
   - ```python
     Tongyi_chat = ChatTongyi(
        top_p=0.5,
        dashscope_api_key=api_key,
        model="qwen-vl-v1"
     )
     response= Tongyi_chat.invoke(
        input =
        [
        {
            "role": "user",
            "content": [
{"image":
"https://dashscope.oss-cn-beijing.aliyuncs.com/images/dog_and_girl.jpeg"},
                {"text": "这是什么?"}
            ]
        }
        ]
       )
      ```

---------

Co-authored-by: Bagatur <baskaryan@gmail.com>
---
 .../langchain_community/chat_models/tongyi.py | 46 ++++++++++++++-----
 1 file changed, 34 insertions(+), 12 deletions(-)

diff --git a/libs/community/langchain_community/chat_models/tongyi.py b/libs/community/langchain_community/chat_models/tongyi.py
index 10d50c0a373..225dfb7c9d4 100644
--- a/libs/community/langchain_community/chat_models/tongyi.py
+++ b/libs/community/langchain_community/chat_models/tongyi.py
@@ -241,8 +241,14 @@ class ChatTongyi(BaseChatModel):
 
     client: Any  #: :meta private:
     model_name: str = Field(default="qwen-turbo", alias="model")
-
-    """Model name to use."""
+    """Model name to use.
+    callable multimodal model:
+    - qwen-vl-v1
+    - qwen-vl-chat-v1
+    - qwen-audio-turbo
+    - qwen-vl-plus
+    - qwen-vl-max
+    """
     model_kwargs: Dict[str, Any] = Field(default_factory=dict)
 
     top_p: float = 0.8
@@ -280,18 +286,34 @@ class ChatTongyi(BaseChatModel):
                 "Could not import dashscope python package. "
                 "Please install it with `pip install dashscope --upgrade`."
             )
-        try:
-            if "vl" in values["model_name"]:
+        dashscope_multimodal_models = [
+            "qwen-vl-v1",
+            "qwen-vl-chat-v1",
+            "qwen-audio-turbo",
+            "qwen-vl-plus",
+            "qwen-vl-max",
+        ]
+        if (
+            values["model_name"] in dashscope_multimodal_models
+            or "vl" in values["model_name"]
+        ):
+            try:
                 values["client"] = dashscope.MultiModalConversation
-            else:
+            except AttributeError:
+                raise ValueError(
+                    "`dashscope` has no `MultiModalConversation` attribute, this is "
+                    "likely due to an old version of the dashscope package. Try "
+                    "upgrading it with `pip install --upgrade dashscope`."
+                )
+        else:
+            try:
                 values["client"] = dashscope.Generation
-        except AttributeError:
-            raise ValueError(
-                "`dashscope` has no `Generation` attribute, this is likely "
-                "due to an old version of the dashscope package. Try upgrading it "
-                "with `pip install --upgrade dashscope`."
-            )
-
+            except AttributeError:
+                raise ValueError(
+                    "`dashscope` has no `Generation` attribute, this is likely "
+                    "due to an old version of the dashscope package. Try upgrading it "
+                    "with `pip install --upgrade dashscope`."
+                )
         return values
 
     @property