mirror of
https://github.com/hwchase17/langchain.git
synced 2025-09-09 06:53:59 +00:00
Add support Vertex AI Gemini uses a public image URL (#14949)
## What Since `langchain_google_genai.ChatGoogleGenerativeAI` supported A public image URL, we add to support it in `langchain.chat_models.ChatVertexAI` as well. ### Example ```py from langchain.chat_models.vertexai import ChatVertexAI from langchain_core.messages import HumanMessage llm = ChatVertexAI(model_name="gemini-pro-vision") image_message = { "type": "image_url", "image_url": { "url": "https://python.langchain.com/assets/images/cell-18-output-1-0c7fb8b94ff032d51bfe1880d8370104.png", }, } text_message = { "type": "text", "text": "What is shown in this image?", } message = HumanMessage(content=[text_message, image_message]) output = llm([message]) print(output.content) ``` ## Refs - https://python.langchain.com/docs/integrations/llms/google_vertex_ai_palm - https://python.langchain.com/docs/integrations/chat/google_generative_ai
This commit is contained in:
@@ -6,7 +6,9 @@ import logging
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Union, cast
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import requests
|
||||
from langchain_core.callbacks import (
|
||||
AsyncCallbackManagerForLLMRun,
|
||||
CallbackManagerForLLMRun,
|
||||
@@ -87,6 +89,15 @@ def _parse_chat_history(history: List[BaseMessage]) -> _ChatHistory:
|
||||
return chat_history
|
||||
|
||||
|
||||
def _is_url(s: str) -> bool:
|
||||
try:
|
||||
result = urlparse(s)
|
||||
return all([result.scheme, result.netloc])
|
||||
except Exception as e:
|
||||
logger.debug(f"Unable to parse URL: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def _parse_chat_history_gemini(
|
||||
history: List[BaseMessage], project: Optional[str]
|
||||
) -> List["Content"]:
|
||||
@@ -118,6 +129,10 @@ def _parse_chat_history_gemini(
|
||||
"data:image/<image_type>;base64,<base64_encoded_image>."
|
||||
)
|
||||
image = Image.from_bytes(base64.b64decode(encoded))
|
||||
elif _is_url(path):
|
||||
response = requests.get(path)
|
||||
response.raise_for_status()
|
||||
image = Image.from_bytes(response.content)
|
||||
else:
|
||||
image = Image.load_from_file(path)
|
||||
else:
|
||||
|
Reference in New Issue
Block a user