mirror of
https://github.com/hwchase17/langchain.git
synced 2026-07-11 18:18:51 +00:00
feat(groq): map context-length errors to ContextOverflowError (#37676)
Fixes #37533 --- `langchain-core` defines `ContextOverflowError` so that application code can catch an over-long prompt the same way regardless of which provider raised it. The Anthropic, OpenAI, and Fireworks integrations already promote their provider-specific context-length errors to a subclass of it, but `langchain-groq` did not: a context overflow there surfaced as a plain `groq.BadRequestError`, so anyone relying on the shared exception had to special-case Groq. This closes that gap for Groq. It adds a `GroqContextOverflowError` (a subclass of both `groq.BadRequestError` and `ContextOverflowError`) and a small promoter, `_handle_groq_invalid_request`, wired into the sync and async `generate` and `stream` paths. Because Groq's SDK mirrors OpenAI's, the implementation follows the same shape as the existing partners, and the promoted error keeps the original `response` and `body` so existing catchers that inspect `.response.status_code` keep working. Anything that already catches `groq.BadRequestError` is unaffected, since the new class is still a `BadRequestError`. One detail worth a reviewer's eye: Groq returns the overflow as a 400 whose JSON body carries `"code": "context_length_exceeded"`, but the SDK's `BadRequestError` does not expose that code as an attribute. The SDK does fold the full JSON body into the error message, so detection primarily matches `context_length_exceeded` against the stringified error, with `reduce the length` from the message as a secondary signal and an attribute check kept as defensive cover in case a future SDK adds `.code`. The unit tests construct the error exactly as the SDK does for a 4xx response and assert promotion across all four call paths, that an unrelated `BadRequestError` is left untouched, and that `response`/`body` are preserved. I scoped this to Groq and left Mistral as a follow-up: Mistral surfaces errors as raw `httpx.HTTPStatusError` rather than a typed SDK error, and I could not verify its exact context-overflow signal (status code plus body `code`/`message`) against an authoritative source well enough to assert it in a unit test without live API access, so I would rather not guess at the shape. --------- Co-authored-by: Mason Daugherty <github@mdrxy.com> Co-authored-by: Mason Daugherty <mason@langchain.dev>
This commit is contained in:
Aditya Singh
GitHub
@@ -6,12 +6,14 @@ import json
|
||||
import warnings
|
||||
from collections.abc import AsyncIterator, Callable, Iterator, Mapping, Sequence
|
||||
from operator import itemgetter
|
||||
from typing import Any, Literal, cast
|
||||
from typing import Any, Literal, NoReturn, cast
|
||||
|
||||
import groq
|
||||
from langchain_core.callbacks import (
|
||||
AsyncCallbackManagerForLLMRun,
|
||||
CallbackManagerForLLMRun,
|
||||
)
|
||||
from langchain_core.exceptions import ContextOverflowError
|
||||
from langchain_core.language_models import (
|
||||
LanguageModelInput,
|
||||
ModelProfile,
|
||||
@@ -88,6 +90,39 @@ def _get_default_model_profile(model_name: str) -> ModelProfile:
|
||||
return default.copy()
|
||||
|
||||
|
||||
class GroqContextOverflowError(groq.BadRequestError, ContextOverflowError):
|
||||
"""`BadRequestError` raised when input exceeds Groq's context limit."""
|
||||
|
||||
|
||||
def _handle_groq_invalid_request(e: groq.BadRequestError) -> NoReturn:
|
||||
"""Promote context-length errors to `GroqContextOverflowError`.
|
||||
|
||||
Groq surfaces an over-long prompt as a 400 `BadRequestError`, but the body
|
||||
shape varies by model and API version, so detection covers each observed
|
||||
form:
|
||||
|
||||
- The current API (verified live) returns no error code; the message reads
|
||||
"Please reduce the length of the messages or completion.", matched by
|
||||
`"reduce the length"`. This is the signal that actually fires today.
|
||||
- Some responses instead carry `"code": "context_length_exceeded"` in the
|
||||
body (see letta-ai/letta#1963), which the SDK folds into `str(e)` and is
|
||||
matched by the substring check.
|
||||
- The `getattr` check is forward-compatible defense in case a future SDK
|
||||
exposes the code as an attribute; no current version does.
|
||||
|
||||
Always raises: `GroqContextOverflowError` for a context overflow, otherwise
|
||||
the original error unchanged. Callers depend on this (`NoReturn`) so that
|
||||
code after the call site can assume the request did not overflow.
|
||||
"""
|
||||
if (
|
||||
getattr(e, "code", None) == "context_length_exceeded"
|
||||
or "context_length_exceeded" in str(e)
|
||||
or "reduce the length" in str(e)
|
||||
):
|
||||
raise GroqContextOverflowError(str(e), response=e.response, body=e.body) from e
|
||||
raise e
|
||||
|
||||
|
||||
class ChatGroq(BaseChatModel):
|
||||
r"""Groq Chat large language models API.
|
||||
|
||||
@@ -526,25 +561,14 @@ class ChatGroq(BaseChatModel):
|
||||
"default_query": self.default_query,
|
||||
}
|
||||
|
||||
try:
|
||||
import groq # noqa: PLC0415
|
||||
|
||||
sync_specific: dict[str, Any] = {"http_client": self.http_client}
|
||||
if not self.client:
|
||||
self.client = groq.Groq(
|
||||
**client_params, **sync_specific
|
||||
).chat.completions
|
||||
if not self.async_client:
|
||||
async_specific: dict[str, Any] = {"http_client": self.http_async_client}
|
||||
self.async_client = groq.AsyncGroq(
|
||||
**client_params, **async_specific
|
||||
).chat.completions
|
||||
except ImportError as exc:
|
||||
msg = (
|
||||
"Could not import groq python package. "
|
||||
"Please install it with `pip install groq`."
|
||||
)
|
||||
raise ImportError(msg) from exc
|
||||
sync_specific: dict[str, Any] = {"http_client": self.http_client}
|
||||
if not self.client:
|
||||
self.client = groq.Groq(**client_params, **sync_specific).chat.completions
|
||||
if not self.async_client:
|
||||
async_specific: dict[str, Any] = {"http_client": self.http_async_client}
|
||||
self.async_client = groq.AsyncGroq(
|
||||
**client_params, **async_specific
|
||||
).chat.completions
|
||||
return self
|
||||
|
||||
@model_validator(mode="after")
|
||||
@@ -634,7 +658,10 @@ class ChatGroq(BaseChatModel):
|
||||
**params,
|
||||
**kwargs,
|
||||
}
|
||||
response = self.client.create(messages=message_dicts, **params)
|
||||
try:
|
||||
response = self.client.create(messages=message_dicts, **params)
|
||||
except groq.BadRequestError as e:
|
||||
_handle_groq_invalid_request(e)
|
||||
return self._create_chat_result(response, params)
|
||||
|
||||
async def _agenerate(
|
||||
@@ -655,7 +682,10 @@ class ChatGroq(BaseChatModel):
|
||||
**params,
|
||||
**kwargs,
|
||||
}
|
||||
response = await self.async_client.create(messages=message_dicts, **params)
|
||||
try:
|
||||
response = await self.async_client.create(messages=message_dicts, **params)
|
||||
except groq.BadRequestError as e:
|
||||
_handle_groq_invalid_request(e)
|
||||
return self._create_chat_result(response, params)
|
||||
|
||||
def _stream(
|
||||
@@ -669,8 +699,12 @@ class ChatGroq(BaseChatModel):
|
||||
|
||||
params = {**params, **kwargs, "stream": True}
|
||||
|
||||
try:
|
||||
stream = self.client.create(messages=message_dicts, **params)
|
||||
except groq.BadRequestError as e:
|
||||
_handle_groq_invalid_request(e)
|
||||
default_chunk_class: type[BaseMessageChunk] = AIMessageChunk
|
||||
for chunk in self.client.create(messages=message_dicts, **params):
|
||||
for chunk in stream:
|
||||
if not isinstance(chunk, dict):
|
||||
chunk = chunk.model_dump() # noqa: PLW2901
|
||||
if len(chunk["choices"]) == 0:
|
||||
@@ -721,10 +755,12 @@ class ChatGroq(BaseChatModel):
|
||||
|
||||
params = {**params, **kwargs, "stream": True}
|
||||
|
||||
try:
|
||||
stream = await self.async_client.create(messages=message_dicts, **params)
|
||||
except groq.BadRequestError as e:
|
||||
_handle_groq_invalid_request(e)
|
||||
default_chunk_class: type[BaseMessageChunk] = AIMessageChunk
|
||||
async for chunk in await self.async_client.create(
|
||||
messages=message_dicts, **params
|
||||
):
|
||||
async for chunk in stream:
|
||||
if not isinstance(chunk, dict):
|
||||
chunk = chunk.model_dump() # noqa: PLW2901
|
||||
if len(chunk["choices"]) == 0:
|
||||
|
||||
@@ -5,8 +5,11 @@ import os
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import groq
|
||||
import httpx
|
||||
import langchain_core.load as lc_load
|
||||
import pytest
|
||||
from langchain_core.exceptions import ContextOverflowError
|
||||
from langchain_core.messages import (
|
||||
AIMessage,
|
||||
AIMessageChunk,
|
||||
@@ -21,10 +24,12 @@ from pydantic import BaseModel
|
||||
|
||||
from langchain_groq.chat_models import (
|
||||
ChatGroq,
|
||||
GroqContextOverflowError,
|
||||
_convert_chunk_to_message_chunk,
|
||||
_convert_dict_to_message,
|
||||
_create_usage_metadata,
|
||||
_format_message_content,
|
||||
_handle_groq_invalid_request,
|
||||
)
|
||||
|
||||
if "GROQ_API_KEY" not in os.environ:
|
||||
@@ -1104,3 +1109,216 @@ def test_metadata_versions() -> None:
|
||||
versions = llm.metadata["lc_versions"]
|
||||
assert "langchain-core" in versions
|
||||
assert "langchain-groq" in versions
|
||||
|
||||
|
||||
def _bad_request_error(
|
||||
body: dict[str, Any], status_code: int = 400
|
||||
) -> groq.BadRequestError:
|
||||
"""Build a `groq.BadRequestError` the way the SDK does for a 4xx response.
|
||||
|
||||
Mirrors `groq._base_client.BaseClient._make_status_error_from_response`: the
|
||||
message is formatted as ``Error code: <status> - <body>`` (so the JSON body
|
||||
ends up in ``str(e)``), and ``e.body`` is the full parsed JSON envelope.
|
||||
Both were verified against the live SDK.
|
||||
"""
|
||||
request = httpx.Request("POST", "https://api.groq.com/openai/v1/chat/completions")
|
||||
response = httpx.Response(status_code=status_code, request=request)
|
||||
message = f"Error code: {status_code} - {body}"
|
||||
return groq.BadRequestError(message, response=response, body=body)
|
||||
|
||||
|
||||
# Real Groq context-overflow error bodies. The codeless form is verified live
|
||||
# against the Groq API (llama-3.1-8b-instant); the code-bearing form is reported
|
||||
# in letta-ai/letta#1963. Detection must handle both.
|
||||
_CONTEXT_OVERFLOW_BODY = {
|
||||
"error": {
|
||||
"message": "Please reduce the length of the messages or completion.",
|
||||
"type": "invalid_request_error",
|
||||
"param": "messages",
|
||||
}
|
||||
}
|
||||
_CONTEXT_OVERFLOW_BODY_WITH_CODE = {
|
||||
"error": {
|
||||
"message": "This model's maximum context length was exceeded.",
|
||||
"type": "invalid_request_error",
|
||||
"param": "messages",
|
||||
"code": "context_length_exceeded",
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def test_context_overflow_error_invoke_sync() -> None:
|
||||
"""Context-length errors surface as `ContextOverflowError` on invoke."""
|
||||
llm = ChatGroq(model="foo", max_retries=0)
|
||||
mock_client = MagicMock()
|
||||
mock_client.create.side_effect = _bad_request_error(_CONTEXT_OVERFLOW_BODY)
|
||||
llm.client = mock_client
|
||||
|
||||
with pytest.raises(ContextOverflowError) as exc_info:
|
||||
llm.invoke([HumanMessage(content="test")])
|
||||
|
||||
assert "reduce the length" in str(exc_info.value)
|
||||
assert isinstance(exc_info.value, GroqContextOverflowError)
|
||||
|
||||
|
||||
async def test_context_overflow_error_invoke_async() -> None:
|
||||
"""Context-length errors surface as `ContextOverflowError` on ainvoke."""
|
||||
llm = ChatGroq(model="foo", max_retries=0)
|
||||
mock_async = MagicMock()
|
||||
|
||||
async def _create(**_kwargs: Any) -> dict[str, Any]:
|
||||
raise _bad_request_error(_CONTEXT_OVERFLOW_BODY)
|
||||
|
||||
mock_async.create = _create
|
||||
llm.async_client = mock_async
|
||||
|
||||
with pytest.raises(ContextOverflowError) as exc_info:
|
||||
await llm.ainvoke([HumanMessage(content="test")])
|
||||
|
||||
assert "reduce the length" in str(exc_info.value)
|
||||
assert isinstance(exc_info.value, GroqContextOverflowError)
|
||||
|
||||
|
||||
def test_context_overflow_error_stream_sync() -> None:
|
||||
"""Context-length errors surface as `ContextOverflowError` on stream."""
|
||||
llm = ChatGroq(model="foo", max_retries=0)
|
||||
mock_client = MagicMock()
|
||||
mock_client.create.side_effect = _bad_request_error(_CONTEXT_OVERFLOW_BODY)
|
||||
llm.client = mock_client
|
||||
|
||||
with pytest.raises(ContextOverflowError) as exc_info:
|
||||
list(llm.stream([HumanMessage(content="test")]))
|
||||
|
||||
assert "reduce the length" in str(exc_info.value)
|
||||
assert isinstance(exc_info.value, GroqContextOverflowError)
|
||||
|
||||
|
||||
async def test_context_overflow_error_stream_async() -> None:
|
||||
"""Context-length errors surface as `ContextOverflowError` on astream."""
|
||||
llm = ChatGroq(model="foo", max_retries=0)
|
||||
mock_async = MagicMock()
|
||||
|
||||
async def _create(**_kwargs: Any) -> Any:
|
||||
raise _bad_request_error(_CONTEXT_OVERFLOW_BODY)
|
||||
|
||||
mock_async.create = _create
|
||||
llm.async_client = mock_async
|
||||
|
||||
with pytest.raises(ContextOverflowError) as exc_info:
|
||||
async for _ in llm.astream([HumanMessage(content="test")]):
|
||||
pass
|
||||
|
||||
assert "reduce the length" in str(exc_info.value)
|
||||
assert isinstance(exc_info.value, GroqContextOverflowError)
|
||||
|
||||
|
||||
def test_context_overflow_error_backwards_compatibility() -> None:
|
||||
"""`ContextOverflowError` is also catchable as `groq.BadRequestError`."""
|
||||
llm = ChatGroq(model="foo", max_retries=0)
|
||||
mock_client = MagicMock()
|
||||
mock_client.create.side_effect = _bad_request_error(_CONTEXT_OVERFLOW_BODY)
|
||||
llm.client = mock_client
|
||||
|
||||
with pytest.raises(groq.BadRequestError) as exc_info:
|
||||
llm.invoke([HumanMessage(content="test")])
|
||||
|
||||
assert isinstance(exc_info.value, groq.BadRequestError)
|
||||
assert isinstance(exc_info.value, ContextOverflowError)
|
||||
|
||||
|
||||
def test_unrelated_invalid_request_error_not_promoted() -> None:
|
||||
"""Unrelated `BadRequestError`s should stay a plain `BadRequestError`."""
|
||||
llm = ChatGroq(model="foo", max_retries=0)
|
||||
other_error = {
|
||||
"error": {
|
||||
"message": "Invalid value for 'temperature'.",
|
||||
"type": "invalid_request_error",
|
||||
"code": "invalid_value",
|
||||
}
|
||||
}
|
||||
mock_client = MagicMock()
|
||||
mock_client.create.side_effect = _bad_request_error(other_error)
|
||||
llm.client = mock_client
|
||||
|
||||
with pytest.raises(groq.BadRequestError) as exc_info:
|
||||
llm.invoke([HumanMessage(content="test")])
|
||||
|
||||
assert not isinstance(exc_info.value, ContextOverflowError)
|
||||
|
||||
|
||||
def test_context_overflow_error_carries_response_metadata() -> None:
|
||||
"""Promoted `GroqContextOverflowError` preserves `response`/`body`.
|
||||
|
||||
Downstream catchers that introspect `.response.status_code` rely on this.
|
||||
"""
|
||||
llm = ChatGroq(model="foo", max_retries=0)
|
||||
mock_client = MagicMock()
|
||||
mock_client.create.side_effect = _bad_request_error(_CONTEXT_OVERFLOW_BODY)
|
||||
llm.client = mock_client
|
||||
|
||||
with pytest.raises(GroqContextOverflowError) as exc_info:
|
||||
llm.invoke([HumanMessage(content="test")])
|
||||
|
||||
assert exc_info.value.response.status_code == 400
|
||||
assert exc_info.value.body == _CONTEXT_OVERFLOW_BODY
|
||||
|
||||
|
||||
# The three detection branches in `_handle_groq_invalid_request` are OR'd, and
|
||||
# real overflow responses only ever satisfy one at a time, so each is exercised
|
||||
# in isolation below — otherwise a branch could be deleted without any test
|
||||
# noticing.
|
||||
|
||||
|
||||
def test_handle_invalid_request_promotes_codeless_message() -> None:
|
||||
"""Branch: live overflow has no code; `reduce the length` is the only signal."""
|
||||
err = _bad_request_error(_CONTEXT_OVERFLOW_BODY)
|
||||
# Guard: this shape must not accidentally satisfy the code-based branches.
|
||||
assert "context_length_exceeded" not in str(err)
|
||||
|
||||
with pytest.raises(GroqContextOverflowError):
|
||||
_handle_groq_invalid_request(err)
|
||||
|
||||
|
||||
def test_handle_invalid_request_promotes_code_in_body() -> None:
|
||||
"""Branch: some responses carry `code` but no `reduce the length` phrase."""
|
||||
err = _bad_request_error(_CONTEXT_OVERFLOW_BODY_WITH_CODE)
|
||||
# Guard: this shape must not accidentally satisfy the message-based branch.
|
||||
assert "reduce the length" not in str(err)
|
||||
|
||||
with pytest.raises(GroqContextOverflowError):
|
||||
_handle_groq_invalid_request(err)
|
||||
|
||||
|
||||
def test_handle_invalid_request_promotes_code_attribute() -> None:
|
||||
"""Branch: forward-compat guard for a future SDK that exposes `.code`."""
|
||||
err = _bad_request_error(
|
||||
{"error": {"message": "boom", "type": "invalid_request_error"}}
|
||||
)
|
||||
err.code = "context_length_exceeded" # type: ignore[attr-defined]
|
||||
# Guard: neither string-based branch should fire, so only `.code` promotes.
|
||||
assert "context_length_exceeded" not in str(err)
|
||||
assert "reduce the length" not in str(err)
|
||||
|
||||
with pytest.raises(GroqContextOverflowError):
|
||||
_handle_groq_invalid_request(err)
|
||||
|
||||
|
||||
def test_handle_invalid_request_ignores_max_tokens_error() -> None:
|
||||
"""A `max_tokens`-too-large 400 must not be promoted (verified live shape)."""
|
||||
max_tokens_body = {
|
||||
"error": {
|
||||
"message": (
|
||||
"`max_tokens` must be less than or equal to `131072`, the maximum "
|
||||
"value for `max_tokens` is less than the `context_window` for this "
|
||||
"model"
|
||||
),
|
||||
"type": "invalid_request_error",
|
||||
"param": "max_tokens",
|
||||
}
|
||||
}
|
||||
err = _bad_request_error(max_tokens_body)
|
||||
|
||||
with pytest.raises(groq.BadRequestError) as exc_info:
|
||||
_handle_groq_invalid_request(err)
|
||||
|
||||
assert not isinstance(exc_info.value, ContextOverflowError)
|
||||
|
||||
Reference in New Issue
Block a user