github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-07-04 20:28:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Ollama vision support (#22734)

Browse Source 
**Description:** Ollama vision with messages in OpenAI-style support `{
"image_url": { "url": ... } }`
**Issue:** #22460 

Added flexible solution for ChatOllama to support chat messages with
images. Works when you provide either `image_url` as a string or as a
dict with "url" inside (like OpenAI does). So it makes available to use
tuples with `ChatPromptTemplate.from_messages()`

---------

Co-authored-by: Bagatur <baskaryan@gmail.com>
This commit is contained in:
Dmitry Stepanov 2024-06-11 19:10:19 +03:00 committed by GitHub
parent 0908b01cb2
commit 912751e268
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 19 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
libs/community/langchain_community/chat_models/ollama.py
View File

@ -137,18 +137,28 @@ class ChatOllama(BaseChatModel, _OllamaCommon):
if content_part.get("type") == "text": if content_part.get("type") == "text":
content += f"\n{content_part['text']}" content += f"\n{content_part['text']}"
elif content_part.get("type") == "image_url": elif content_part.get("type") == "image_url":
if isinstance(content_part.get("image_url"), str): image_url = None
image_url_components = content_part["image_url"].split(",") temp_image_url = content_part.get("image_url")
# Support data:image/jpeg;base64,<image> format if isinstance(temp_image_url, str):
# and base64 strings image_url = content_part["image_url"]
if len(image_url_components) > 1: elif (
images.append(image_url_components[1]) isinstance(temp_image_url, dict) and "url" in temp_image_url
else: ):
images.append(image_url_components[0]) image_url = temp_image_url
else: else:
raise ValueError( raise ValueError(
"Only string image_url " "content parts are supported." "Only string image_url or dict with string 'url' "
"inside content parts are supported."
) )
image_url_components = image_url.split(",")
# Support data:image/jpeg;base64,<image> format
# and base64 strings
if len(image_url_components) > 1:
images.append(image_url_components[1])
else:
images.append(image_url_components[0])
else: else:
raise ValueError( raise ValueError(
"Unsupported message content type. " "Unsupported message content type. "