github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2026-07-12 02:55:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(model-profiles): add input/output MIME type fields to ModelProfile

Browse Source 
Adds informational input_mime_types and output_mime_types dict fields
to ModelProfile, keyed by models.dev modality names ('image', 'audio',
'pdf', 'video'). Augments the CLI to dispatch provider-level vs
model-level overrides by ModelProfile field name so non-scalar
provider-level fields (like the new MIME maps) are routed correctly.

Backfills anthropic, openai, and perplexity profile_augmentations.toml
with documented MIME types and regenerates their _profiles.py.

Co-authored-by: Mason Daugherty <61371264+mdrxy@users.noreply.github.com>
This commit is contained in:
open-swe[bot]
2026-05-11 18:25:34 +00:00
parent 8b21400627
commit 988ee535d5
9 changed files with 1083 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
libs/core/langchain_core/language_models/model_profile.py
View File

@@ -50,7 +50,6 @@ class ModelProfile(TypedDict, total=False):
image_inputs: bool
"""Whether image inputs are supported."""
# TODO: add more detail about formats?
image_url_inputs: bool
"""Whether [image URL inputs](https://docs.langchain.com/oss/python/langchain/models#multimodal)
@@ -59,17 +58,36 @@ class ModelProfile(TypedDict, total=False):
pdf_inputs: bool
"""Whether [PDF inputs](https://docs.langchain.com/oss/python/langchain/models#multimodal)
are supported."""
# TODO: add more detail about formats? e.g. bytes or base64
audio_inputs: bool
"""Whether [audio inputs](https://docs.langchain.com/oss/python/langchain/models#multimodal)
are supported."""
# TODO: add more detail about formats? e.g. bytes or base64
video_inputs: bool
"""Whether [video inputs](https://docs.langchain.com/oss/python/langchain/models#multimodal)
are supported."""
# TODO: add more detail about formats? e.g. bytes or base64
input_mime_types: dict[str, list[str]]
"""MIME types accepted as input, grouped by modality.
Keys mirror the modality names used by [models.dev](https://models.dev) — for
example, `'image'`, `'audio'`, `'pdf'`, `'video'`. Values are lists of
[IANA media types](https://www.iana.org/assignments/media-types/) such as
`'image/png'` or `'audio/mpeg'`.
This field is *informational*: an entry means the provider is known to accept
that MIME type, but an empty or missing list does not necessarily mean a
type is rejected. Consumers that need hard validation should consult the
upstream provider's documentation.
Example:
```python
{
"image": ["image/png", "image/jpeg", "image/gif", "image/webp"],
"pdf": ["application/pdf"],
}
```
"""
image_tool_message: bool
"""Whether images can be included in tool messages."""
@@ -100,6 +118,25 @@ class ModelProfile(TypedDict, total=False):
"""Whether [video outputs](https://docs.langchain.com/oss/python/langchain/models#multimodal)
are supported."""
output_mime_types: dict[str, list[str]]
"""MIME types produced as output, grouped by modality.
Keys mirror the modality names used by [models.dev](https://models.dev) — for
example, `'image'`, `'audio'`, `'video'`. Values are lists of
[IANA media types](https://www.iana.org/assignments/media-types/) such as
`'image/png'` or `'audio/mpeg'`.
This field is *informational*: an entry means the provider is known to
return that MIME type, but absence does not imply a guarantee that no other
type can be returned. Consumers that need hard validation should consult the
upstream provider's documentation.
Example:
```python
{"image": ["image/png"], "audio": ["audio/mpeg"]}
```
"""
# --- Tool calling ---
tool_calling: bool
"""Whether the model supports [tool calling](https://docs.langchain.com/oss/python/langchain/models#tool-calling)"""

39
libs/model-profiles/langchain_model_profiles/cli.py
View File

@@ -55,11 +55,30 @@ def _validate_data_dir(data_dir: Path) -> Path:
return resolved
def _profile_field_names() -> frozenset[str]:
"""Return the set of keys declared on `ModelProfile`, or an empty set."""
try:
from langchain_core.language_models.model_profile import ModelProfile
except ImportError:
return frozenset()
try:
return frozenset(get_type_hints(ModelProfile).keys())
except (TypeError, NameError):
return frozenset()
def _load_augmentations(
data_dir: Path,
) -> tuple[dict[str, Any], dict[str, dict[str, Any]]]:
"""Load augmentations from `profile_augmentations.toml`.
Provider-level overrides are top-level keys under `[overrides]`. Model-level
overrides are keyed by model id under `[overrides."model-id"]`. Because TOML
subtables produce `dict` values, we distinguish the two by checking the key
against the declared `ModelProfile` field names. If `ModelProfile` cannot be
imported, we fall back to the legacy heuristic of "dict value ⇒ model id."
Args:
data_dir: Directory containing `profile_augmentations.toml`.
@@ -90,8 +109,26 @@ def _load_augmentations(
provider_aug: dict[str, Any] = {}
model_augs: dict[str, dict[str, Any]] = {}
profile_fields = _profile_field_names()
for key, value in overrides.items():
if isinstance(value, dict):
if profile_fields:
# Schema-driven: known profile field names are provider-level; all
# other keys are treated as model identifiers (whose values must be
# dict overrides).
if key in profile_fields:
provider_aug[key] = value
elif isinstance(value, dict):
model_augs[key] = value
else:
msg = (
f"Augmentation key '{key}' is not a declared ModelProfile "
f"field and its value is not a table of overrides."
)
print(f"❌ {msg}", file=sys.stderr)
sys.exit(1)
# Legacy fallback when ModelProfile is unavailable.
elif isinstance(value, dict):
model_augs[key] = value
else:
provider_aug[key] = value

130
libs/model-profiles/tests/unit_tests/test_cli.py
View File

@@ -472,3 +472,133 @@ class TestWarnUndeclaredProfileKeys:
side_effect=TypeError("broken"),
):
_warn_undeclared_profile_keys(profiles)
def test_refresh_merges_provider_level_mime_types(
tmp_path: Path, mock_models_dev_response: dict
) -> None:
"""Provider-level `input_mime_types` cascades to every model."""
data_dir = tmp_path / "data"
data_dir.mkdir()
aug_file = data_dir / "profile_augmentations.toml"
aug_file.write_text(
"""
provider = "anthropic"
[overrides]
image_url_inputs = true
[overrides.input_mime_types]
image = ["image/png", "image/jpeg"]
pdf = ["application/pdf"]
"""
)
mock_response = Mock()
mock_response.json.return_value = mock_models_dev_response
mock_response.raise_for_status = Mock()
with (
patch("langchain_model_profiles.cli.httpx.get", return_value=mock_response),
patch("builtins.input", return_value="y"),
):
refresh("anthropic", data_dir)
profiles_file = data_dir / "_profiles.py"
spec = importlib.util.spec_from_file_location("generated_mime", profiles_file)
assert spec
assert spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module) # type: ignore[union-attr]
for model_id in ("claude-3-opus", "claude-3-sonnet"):
profile = module._PROFILES[model_id] # type: ignore[attr-defined]
assert profile["input_mime_types"] == {
"image": ["image/png", "image/jpeg"],
"pdf": ["application/pdf"],
}
def test_refresh_model_level_mime_types_override_provider(
tmp_path: Path, mock_models_dev_response: dict
) -> None:
"""Model-level MIME-type overrides win over provider-level defaults."""
data_dir = tmp_path / "data"
data_dir.mkdir()
aug_file = data_dir / "profile_augmentations.toml"
aug_file.write_text(
"""
provider = "anthropic"
[overrides.input_mime_types]
image = ["image/png"]
[overrides."claude-3-opus".input_mime_types]
image = ["image/png", "image/jpeg", "image/webp"]
[overrides."claude-3-opus".output_mime_types]
image = ["image/png"]
"""
)
mock_response = Mock()
mock_response.json.return_value = mock_models_dev_response
mock_response.raise_for_status = Mock()
with (
patch("langchain_model_profiles.cli.httpx.get", return_value=mock_response),
patch("builtins.input", return_value="y"),
):
refresh("anthropic", data_dir)
profiles_file = data_dir / "_profiles.py"
spec = importlib.util.spec_from_file_location(
"generated_mime_override", profiles_file
)
assert spec
assert spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module) # type: ignore[union-attr]
opus = module._PROFILES["claude-3-opus"] # type: ignore[attr-defined]
sonnet = module._PROFILES["claude-3-sonnet"] # type: ignore[attr-defined]
assert opus["input_mime_types"] == {
"image": ["image/png", "image/jpeg", "image/webp"]
}
assert opus["output_mime_types"] == {"image": ["image/png"]}
assert sonnet["input_mime_types"] == {"image": ["image/png"]}
assert "output_mime_types" not in sonnet
def test_refresh_rejects_unknown_scalar_top_level_key(
tmp_path: Path, mock_models_dev_response: dict
) -> None:
"""Unknown scalar keys at the top level are no longer silently accepted."""
data_dir = tmp_path / "data"
data_dir.mkdir()
aug_file = data_dir / "profile_augmentations.toml"
aug_file.write_text(
"""
provider = "anthropic"
[overrides]
not_a_real_field = "oops"
"""
)
mock_response = Mock()
mock_response.json.return_value = mock_models_dev_response
mock_response.raise_for_status = Mock()
with (
patch("langchain_model_profiles.cli.httpx.get", return_value=mock_response),
patch("builtins.input", return_value="y"),
pytest.raises(SystemExit) as exc_info,
):
refresh("anthropic", data_dir)
assert exc_info.value.code == 1

253
libs/partners/anthropic/langchain_anthropic/data/_profiles.py
View File

@@ -40,6 +40,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"structured_output": False,
"input_mime_types": {
"image": [
"image/jpeg",
"image/png",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"claude-3-5-haiku-latest": {
"name": "Claude Haiku 3.5 (latest)",
@@ -65,6 +76,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"structured_output": False,
"input_mime_types": {
"image": [
"image/jpeg",
"image/png",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"claude-3-5-sonnet-20240620": {
"name": "Claude Sonnet 3.5",
@@ -90,6 +112,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"structured_output": False,
"input_mime_types": {
"image": [
"image/jpeg",
"image/png",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"claude-3-5-sonnet-20241022": {
"name": "Claude Sonnet 3.5 v2",
@@ -115,6 +148,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"structured_output": False,
"input_mime_types": {
"image": [
"image/jpeg",
"image/png",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"claude-3-7-sonnet-20250219": {
"name": "Claude Sonnet 3.7",
@@ -140,6 +184,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"structured_output": False,
"input_mime_types": {
"image": [
"image/jpeg",
"image/png",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"claude-3-haiku-20240307": {
"name": "Claude Haiku 3",
@@ -165,6 +220,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"structured_output": False,
"input_mime_types": {
"image": [
"image/jpeg",
"image/png",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"claude-3-opus-20240229": {
"name": "Claude Opus 3",
@@ -190,6 +256,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"structured_output": False,
"input_mime_types": {
"image": [
"image/jpeg",
"image/png",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"claude-3-sonnet-20240229": {
"name": "Claude Sonnet 3",
@@ -215,6 +292,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"structured_output": False,
"input_mime_types": {
"image": [
"image/jpeg",
"image/png",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"claude-haiku-4-5": {
"name": "Claude Haiku 4.5 (latest)",
@@ -240,6 +328,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"structured_output": True,
"input_mime_types": {
"image": [
"image/jpeg",
"image/png",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"claude-haiku-4-5-20251001": {
"name": "Claude Haiku 4.5",
@@ -265,6 +364,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"structured_output": False,
"input_mime_types": {
"image": [
"image/jpeg",
"image/png",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"claude-opus-4-0": {
"name": "Claude Opus 4 (latest)",
@@ -290,6 +400,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"structured_output": False,
"input_mime_types": {
"image": [
"image/jpeg",
"image/png",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"claude-opus-4-1": {
"name": "Claude Opus 4.1 (latest)",
@@ -315,6 +436,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"structured_output": True,
"input_mime_types": {
"image": [
"image/jpeg",
"image/png",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"claude-opus-4-1-20250805": {
"name": "Claude Opus 4.1",
@@ -340,6 +472,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"structured_output": False,
"input_mime_types": {
"image": [
"image/jpeg",
"image/png",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"claude-opus-4-20250514": {
"name": "Claude Opus 4",
@@ -365,6 +508,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"structured_output": False,
"input_mime_types": {
"image": [
"image/jpeg",
"image/png",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"claude-opus-4-5": {
"name": "Claude Opus 4.5 (latest)",
@@ -390,6 +544,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"structured_output": True,
"input_mime_types": {
"image": [
"image/jpeg",
"image/png",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"claude-opus-4-5-20251101": {
"name": "Claude Opus 4.5",
@@ -415,6 +580,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"structured_output": False,
"input_mime_types": {
"image": [
"image/jpeg",
"image/png",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"claude-opus-4-6": {
"name": "Claude Opus 4.6",
@@ -440,6 +616,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"structured_output": True,
"input_mime_types": {
"image": [
"image/jpeg",
"image/png",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"claude-opus-4-7": {
"name": "Claude Opus 4.7",
@@ -465,6 +652,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"structured_output": True,
"input_mime_types": {
"image": [
"image/jpeg",
"image/png",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"claude-sonnet-4-0": {
"name": "Claude Sonnet 4 (latest)",
@@ -490,6 +688,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"structured_output": False,
"input_mime_types": {
"image": [
"image/jpeg",
"image/png",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"claude-sonnet-4-20250514": {
"name": "Claude Sonnet 4",
@@ -515,6 +724,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"structured_output": False,
"input_mime_types": {
"image": [
"image/jpeg",
"image/png",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"claude-sonnet-4-5": {
"name": "Claude Sonnet 4.5 (latest)",
@@ -540,6 +760,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"structured_output": True,
"input_mime_types": {
"image": [
"image/jpeg",
"image/png",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"claude-sonnet-4-5-20250929": {
"name": "Claude Sonnet 4.5",
@@ -565,6 +796,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"structured_output": False,
"input_mime_types": {
"image": [
"image/jpeg",
"image/png",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"claude-sonnet-4-6": {
"name": "Claude Sonnet 4.6",
@@ -590,5 +832,16 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"structured_output": True,
"input_mime_types": {
"image": [
"image/jpeg",
"image/png",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
}

4
libs/partners/anthropic/langchain_anthropic/data/profile_augmentations.toml
View File

@@ -7,6 +7,10 @@ pdf_tool_message = true
image_tool_message = true
structured_output = false
[overrides.input_mime_types]
image = ["image/jpeg", "image/png", "image/gif", "image/webp"]
pdf = ["application/pdf"]
[overrides."claude-haiku-4-5"]
structured_output = true

593
libs/partners/openai/langchain_openai/data/_profiles.py
View File

@@ -40,6 +40,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-3.5-turbo": {
"name": "GPT-3.5-turbo",
@@ -66,6 +77,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": False,
"image_tool_message": False,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-4": {
"name": "GPT-4",
@@ -92,6 +114,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-4-turbo": {
"name": "GPT-4 Turbo",
@@ -118,6 +151,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-4.1": {
"name": "GPT-4.1",
@@ -144,6 +188,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-4.1-mini": {
"name": "GPT-4.1 mini",
@@ -170,6 +225,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-4.1-nano": {
"name": "GPT-4.1 nano",
@@ -196,6 +262,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-4o": {
"name": "GPT-4o",
@@ -222,6 +299,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-4o-2024-05-13": {
"name": "GPT-4o (2024-05-13)",
@@ -248,6 +336,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-4o-2024-08-06": {
"name": "GPT-4o (2024-08-06)",
@@ -274,6 +373,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-4o-2024-11-20": {
"name": "GPT-4o (2024-11-20)",
@@ -300,6 +410,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-4o-mini": {
"name": "GPT-4o mini",
@@ -326,6 +447,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-5": {
"name": "GPT-5",
@@ -352,6 +484,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-5-chat-latest": {
"name": "GPT-5 Chat (latest)",
@@ -378,6 +521,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-5-codex": {
"name": "GPT-5-Codex",
@@ -404,6 +558,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-5-mini": {
"name": "GPT-5 Mini",
@@ -430,6 +595,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-5-nano": {
"name": "GPT-5 Nano",
@@ -456,6 +632,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-5-pro": {
"name": "GPT-5 Pro",
@@ -482,6 +669,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-5.1": {
"name": "GPT-5.1",
@@ -508,6 +706,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-5.1-chat-latest": {
"name": "GPT-5.1 Chat",
@@ -534,6 +743,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-5.1-codex": {
"name": "GPT-5.1 Codex",
@@ -560,6 +780,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-5.1-codex-max": {
"name": "GPT-5.1 Codex Max",
@@ -586,6 +817,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-5.1-codex-mini": {
"name": "GPT-5.1 Codex mini",
@@ -612,6 +854,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-5.2": {
"name": "GPT-5.2",
@@ -638,6 +891,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-5.2-chat-latest": {
"name": "GPT-5.2 Chat",
@@ -664,6 +928,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-5.2-codex": {
"name": "GPT-5.2 Codex",
@@ -690,6 +965,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-5.2-pro": {
"name": "GPT-5.2 Pro",
@@ -716,6 +1002,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-5.3-chat-latest": {
"name": "GPT-5.3 Chat (latest)",
@@ -742,6 +1039,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-5.3-codex": {
"name": "GPT-5.3 Codex",
@@ -768,6 +1076,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-5.3-codex-spark": {
"name": "GPT-5.3 Codex Spark",
@@ -794,6 +1113,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-5.4": {
"name": "GPT-5.4",
@@ -820,6 +1150,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-5.4-mini": {
"name": "GPT-5.4 mini",
@@ -846,6 +1187,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-5.4-nano": {
"name": "GPT-5.4 nano",
@@ -872,6 +1224,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-5.4-pro": {
"name": "GPT-5.4 Pro",
@@ -898,6 +1261,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-5.5": {
"name": "GPT-5.5",
@@ -924,6 +1298,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-5.5-pro": {
"name": "GPT-5.5 Pro",
@@ -950,6 +1335,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"gpt-image-1": {
"name": "gpt-image-1",
@@ -975,6 +1371,24 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
"output_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/webp",
],
},
},
"gpt-image-1-mini": {
"name": "gpt-image-1-mini",
@@ -1000,6 +1414,24 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
"output_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/webp",
],
},
},
"gpt-image-1.5": {
"name": "gpt-image-1.5",
@@ -1025,6 +1457,24 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
"output_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/webp",
],
},
},
"o1": {
"name": "o1",
@@ -1051,6 +1501,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"o1-mini": {
"name": "o1-mini",
@@ -1077,6 +1538,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"o1-preview": {
"name": "o1-preview",
@@ -1102,6 +1574,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"o1-pro": {
"name": "o1-pro",
@@ -1128,6 +1611,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"o3": {
"name": "o3",
@@ -1154,6 +1648,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"o3-deep-research": {
"name": "o3-deep-research",
@@ -1179,6 +1684,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"o3-mini": {
"name": "o3-mini",
@@ -1205,6 +1721,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"o3-pro": {
"name": "o3-pro",
@@ -1231,6 +1758,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"o4-mini": {
"name": "o4-mini",
@@ -1257,6 +1795,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"o4-mini-deep-research": {
"name": "o4-mini-deep-research",
@@ -1282,6 +1831,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"text-embedding-3-large": {
"name": "text-embedding-3-large",
@@ -1307,6 +1867,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"text-embedding-3-small": {
"name": "text-embedding-3-small",
@@ -1332,6 +1903,17 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
"text-embedding-ada-002": {
"name": "text-embedding-ada-002",
@@ -1357,5 +1939,16 @@ _PROFILES: dict[str, dict[str, Any]] = {
"pdf_tool_message": True,
"image_tool_message": True,
"tool_choice": True,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
"pdf": [
"application/pdf",
],
},
},
}

13
libs/partners/openai/langchain_openai/data/profile_augmentations.toml
View File

@@ -7,12 +7,25 @@ pdf_tool_message = true
image_tool_message = true
tool_choice = true
[overrides.input_mime_types]
image = ["image/png", "image/jpeg", "image/gif", "image/webp"]
pdf = ["application/pdf"]
[overrides."gpt-3.5-turbo"]
image_url_inputs = false
pdf_inputs = false
pdf_tool_message = false
image_tool_message = false
[overrides."gpt-image-1".output_mime_types]
image = ["image/png", "image/jpeg", "image/webp"]
[overrides."gpt-image-1-mini".output_mime_types]
image = ["image/png", "image/jpeg", "image/webp"]
[overrides."gpt-image-1.5".output_mime_types]
image = ["image/png", "image/jpeg", "image/webp"]
[overrides."gpt-5.1-codex"]
max_input_tokens = 272000

8
libs/partners/perplexity/langchain_perplexity/data/_profiles.py
View File

@@ -55,6 +55,14 @@ _PROFILES: dict[str, dict[str, Any]] = {
"tool_calling": False,
"attachment": False,
"temperature": False,
"input_mime_types": {
"image": [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
],
},
},
"sonar-pro": {
"name": "Sonar Pro",

3
libs/partners/perplexity/langchain_perplexity/data/profile_augmentations.toml
View File

@@ -11,3 +11,6 @@ audio_outputs = false
video_outputs = false
reasoning_output = true
tool_calling = false
[overrides."sonar-deep-research".input_mime_types]
image = ["image/png", "image/jpeg", "image/gif", "image/webp"]