From bde894268b7acfd5213438914fbf76303160bea9 Mon Sep 17 00:00:00 2001 From: Aditya Singh <60082699+adityasingh2400@users.noreply.github.com> Date: Sat, 4 Jul 2026 21:27:44 -0700 Subject: [PATCH] feat(groq): map context-length errors to `ContextOverflowError` (#37676) Fixes #37533 --- `langchain-core` defines `ContextOverflowError` so that application code can catch an over-long prompt the same way regardless of which provider raised it. The Anthropic, OpenAI, and Fireworks integrations already promote their provider-specific context-length errors to a subclass of it, but `langchain-groq` did not: a context overflow there surfaced as a plain `groq.BadRequestError`, so anyone relying on the shared exception had to special-case Groq. This closes that gap for Groq. It adds a `GroqContextOverflowError` (a subclass of both `groq.BadRequestError` and `ContextOverflowError`) and a small promoter, `_handle_groq_invalid_request`, wired into the sync and async `generate` and `stream` paths. Because Groq's SDK mirrors OpenAI's, the implementation follows the same shape as the existing partners, and the promoted error keeps the original `response` and `body` so existing catchers that inspect `.response.status_code` keep working. Anything that already catches `groq.BadRequestError` is unaffected, since the new class is still a `BadRequestError`. One detail worth a reviewer's eye: Groq returns the overflow as a 400 whose JSON body carries `"code": "context_length_exceeded"`, but the SDK's `BadRequestError` does not expose that code as an attribute. The SDK does fold the full JSON body into the error message, so detection primarily matches `context_length_exceeded` against the stringified error, with `reduce the length` from the message as a secondary signal and an attribute check kept as defensive cover in case a future SDK adds `.code`. The unit tests construct the error exactly as the SDK does for a 4xx response and assert promotion across all four call paths, that an unrelated `BadRequestError` is left untouched, and that `response`/`body` are preserved. I scoped this to Groq and left Mistral as a follow-up: Mistral surfaces errors as raw `httpx.HTTPStatusError` rather than a typed SDK error, and I could not verify its exact context-overflow signal (status code plus body `code`/`message`) against an authoritative source well enough to assert it in a unit test without live API access, so I would rather not guess at the shape. --------- Co-authored-by: Mason Daugherty Co-authored-by: Mason Daugherty --- .../groq/langchain_groq/chat_models.py | 88 ++++--- .../groq/tests/unit_tests/test_chat_models.py | 218 ++++++++++++++++++ 2 files changed, 280 insertions(+), 26 deletions(-) diff --git a/libs/partners/groq/langchain_groq/chat_models.py b/libs/partners/groq/langchain_groq/chat_models.py index 95db3b86b55..7aabbfc1094 100644 --- a/libs/partners/groq/langchain_groq/chat_models.py +++ b/libs/partners/groq/langchain_groq/chat_models.py @@ -6,12 +6,14 @@ import json import warnings from collections.abc import AsyncIterator, Callable, Iterator, Mapping, Sequence from operator import itemgetter -from typing import Any, Literal, cast +from typing import Any, Literal, NoReturn, cast +import groq from langchain_core.callbacks import ( AsyncCallbackManagerForLLMRun, CallbackManagerForLLMRun, ) +from langchain_core.exceptions import ContextOverflowError from langchain_core.language_models import ( LanguageModelInput, ModelProfile, @@ -88,6 +90,39 @@ def _get_default_model_profile(model_name: str) -> ModelProfile: return default.copy() +class GroqContextOverflowError(groq.BadRequestError, ContextOverflowError): + """`BadRequestError` raised when input exceeds Groq's context limit.""" + + +def _handle_groq_invalid_request(e: groq.BadRequestError) -> NoReturn: + """Promote context-length errors to `GroqContextOverflowError`. + + Groq surfaces an over-long prompt as a 400 `BadRequestError`, but the body + shape varies by model and API version, so detection covers each observed + form: + + - The current API (verified live) returns no error code; the message reads + "Please reduce the length of the messages or completion.", matched by + `"reduce the length"`. This is the signal that actually fires today. + - Some responses instead carry `"code": "context_length_exceeded"` in the + body (see letta-ai/letta#1963), which the SDK folds into `str(e)` and is + matched by the substring check. + - The `getattr` check is forward-compatible defense in case a future SDK + exposes the code as an attribute; no current version does. + + Always raises: `GroqContextOverflowError` for a context overflow, otherwise + the original error unchanged. Callers depend on this (`NoReturn`) so that + code after the call site can assume the request did not overflow. + """ + if ( + getattr(e, "code", None) == "context_length_exceeded" + or "context_length_exceeded" in str(e) + or "reduce the length" in str(e) + ): + raise GroqContextOverflowError(str(e), response=e.response, body=e.body) from e + raise e + + class ChatGroq(BaseChatModel): r"""Groq Chat large language models API. @@ -526,25 +561,14 @@ class ChatGroq(BaseChatModel): "default_query": self.default_query, } - try: - import groq # noqa: PLC0415 - - sync_specific: dict[str, Any] = {"http_client": self.http_client} - if not self.client: - self.client = groq.Groq( - **client_params, **sync_specific - ).chat.completions - if not self.async_client: - async_specific: dict[str, Any] = {"http_client": self.http_async_client} - self.async_client = groq.AsyncGroq( - **client_params, **async_specific - ).chat.completions - except ImportError as exc: - msg = ( - "Could not import groq python package. " - "Please install it with `pip install groq`." - ) - raise ImportError(msg) from exc + sync_specific: dict[str, Any] = {"http_client": self.http_client} + if not self.client: + self.client = groq.Groq(**client_params, **sync_specific).chat.completions + if not self.async_client: + async_specific: dict[str, Any] = {"http_client": self.http_async_client} + self.async_client = groq.AsyncGroq( + **client_params, **async_specific + ).chat.completions return self @model_validator(mode="after") @@ -634,7 +658,10 @@ class ChatGroq(BaseChatModel): **params, **kwargs, } - response = self.client.create(messages=message_dicts, **params) + try: + response = self.client.create(messages=message_dicts, **params) + except groq.BadRequestError as e: + _handle_groq_invalid_request(e) return self._create_chat_result(response, params) async def _agenerate( @@ -655,7 +682,10 @@ class ChatGroq(BaseChatModel): **params, **kwargs, } - response = await self.async_client.create(messages=message_dicts, **params) + try: + response = await self.async_client.create(messages=message_dicts, **params) + except groq.BadRequestError as e: + _handle_groq_invalid_request(e) return self._create_chat_result(response, params) def _stream( @@ -669,8 +699,12 @@ class ChatGroq(BaseChatModel): params = {**params, **kwargs, "stream": True} + try: + stream = self.client.create(messages=message_dicts, **params) + except groq.BadRequestError as e: + _handle_groq_invalid_request(e) default_chunk_class: type[BaseMessageChunk] = AIMessageChunk - for chunk in self.client.create(messages=message_dicts, **params): + for chunk in stream: if not isinstance(chunk, dict): chunk = chunk.model_dump() # noqa: PLW2901 if len(chunk["choices"]) == 0: @@ -721,10 +755,12 @@ class ChatGroq(BaseChatModel): params = {**params, **kwargs, "stream": True} + try: + stream = await self.async_client.create(messages=message_dicts, **params) + except groq.BadRequestError as e: + _handle_groq_invalid_request(e) default_chunk_class: type[BaseMessageChunk] = AIMessageChunk - async for chunk in await self.async_client.create( - messages=message_dicts, **params - ): + async for chunk in stream: if not isinstance(chunk, dict): chunk = chunk.model_dump() # noqa: PLW2901 if len(chunk["choices"]) == 0: diff --git a/libs/partners/groq/tests/unit_tests/test_chat_models.py b/libs/partners/groq/tests/unit_tests/test_chat_models.py index de7d735c396..86fea8a21da 100644 --- a/libs/partners/groq/tests/unit_tests/test_chat_models.py +++ b/libs/partners/groq/tests/unit_tests/test_chat_models.py @@ -5,8 +5,11 @@ import os from typing import Any from unittest.mock import AsyncMock, MagicMock, patch +import groq +import httpx import langchain_core.load as lc_load import pytest +from langchain_core.exceptions import ContextOverflowError from langchain_core.messages import ( AIMessage, AIMessageChunk, @@ -21,10 +24,12 @@ from pydantic import BaseModel from langchain_groq.chat_models import ( ChatGroq, + GroqContextOverflowError, _convert_chunk_to_message_chunk, _convert_dict_to_message, _create_usage_metadata, _format_message_content, + _handle_groq_invalid_request, ) if "GROQ_API_KEY" not in os.environ: @@ -1104,3 +1109,216 @@ def test_metadata_versions() -> None: versions = llm.metadata["lc_versions"] assert "langchain-core" in versions assert "langchain-groq" in versions + + +def _bad_request_error( + body: dict[str, Any], status_code: int = 400 +) -> groq.BadRequestError: + """Build a `groq.BadRequestError` the way the SDK does for a 4xx response. + + Mirrors `groq._base_client.BaseClient._make_status_error_from_response`: the + message is formatted as ``Error code: - `` (so the JSON body + ends up in ``str(e)``), and ``e.body`` is the full parsed JSON envelope. + Both were verified against the live SDK. + """ + request = httpx.Request("POST", "https://api.groq.com/openai/v1/chat/completions") + response = httpx.Response(status_code=status_code, request=request) + message = f"Error code: {status_code} - {body}" + return groq.BadRequestError(message, response=response, body=body) + + +# Real Groq context-overflow error bodies. The codeless form is verified live +# against the Groq API (llama-3.1-8b-instant); the code-bearing form is reported +# in letta-ai/letta#1963. Detection must handle both. +_CONTEXT_OVERFLOW_BODY = { + "error": { + "message": "Please reduce the length of the messages or completion.", + "type": "invalid_request_error", + "param": "messages", + } +} +_CONTEXT_OVERFLOW_BODY_WITH_CODE = { + "error": { + "message": "This model's maximum context length was exceeded.", + "type": "invalid_request_error", + "param": "messages", + "code": "context_length_exceeded", + } +} + + +def test_context_overflow_error_invoke_sync() -> None: + """Context-length errors surface as `ContextOverflowError` on invoke.""" + llm = ChatGroq(model="foo", max_retries=0) + mock_client = MagicMock() + mock_client.create.side_effect = _bad_request_error(_CONTEXT_OVERFLOW_BODY) + llm.client = mock_client + + with pytest.raises(ContextOverflowError) as exc_info: + llm.invoke([HumanMessage(content="test")]) + + assert "reduce the length" in str(exc_info.value) + assert isinstance(exc_info.value, GroqContextOverflowError) + + +async def test_context_overflow_error_invoke_async() -> None: + """Context-length errors surface as `ContextOverflowError` on ainvoke.""" + llm = ChatGroq(model="foo", max_retries=0) + mock_async = MagicMock() + + async def _create(**_kwargs: Any) -> dict[str, Any]: + raise _bad_request_error(_CONTEXT_OVERFLOW_BODY) + + mock_async.create = _create + llm.async_client = mock_async + + with pytest.raises(ContextOverflowError) as exc_info: + await llm.ainvoke([HumanMessage(content="test")]) + + assert "reduce the length" in str(exc_info.value) + assert isinstance(exc_info.value, GroqContextOverflowError) + + +def test_context_overflow_error_stream_sync() -> None: + """Context-length errors surface as `ContextOverflowError` on stream.""" + llm = ChatGroq(model="foo", max_retries=0) + mock_client = MagicMock() + mock_client.create.side_effect = _bad_request_error(_CONTEXT_OVERFLOW_BODY) + llm.client = mock_client + + with pytest.raises(ContextOverflowError) as exc_info: + list(llm.stream([HumanMessage(content="test")])) + + assert "reduce the length" in str(exc_info.value) + assert isinstance(exc_info.value, GroqContextOverflowError) + + +async def test_context_overflow_error_stream_async() -> None: + """Context-length errors surface as `ContextOverflowError` on astream.""" + llm = ChatGroq(model="foo", max_retries=0) + mock_async = MagicMock() + + async def _create(**_kwargs: Any) -> Any: + raise _bad_request_error(_CONTEXT_OVERFLOW_BODY) + + mock_async.create = _create + llm.async_client = mock_async + + with pytest.raises(ContextOverflowError) as exc_info: + async for _ in llm.astream([HumanMessage(content="test")]): + pass + + assert "reduce the length" in str(exc_info.value) + assert isinstance(exc_info.value, GroqContextOverflowError) + + +def test_context_overflow_error_backwards_compatibility() -> None: + """`ContextOverflowError` is also catchable as `groq.BadRequestError`.""" + llm = ChatGroq(model="foo", max_retries=0) + mock_client = MagicMock() + mock_client.create.side_effect = _bad_request_error(_CONTEXT_OVERFLOW_BODY) + llm.client = mock_client + + with pytest.raises(groq.BadRequestError) as exc_info: + llm.invoke([HumanMessage(content="test")]) + + assert isinstance(exc_info.value, groq.BadRequestError) + assert isinstance(exc_info.value, ContextOverflowError) + + +def test_unrelated_invalid_request_error_not_promoted() -> None: + """Unrelated `BadRequestError`s should stay a plain `BadRequestError`.""" + llm = ChatGroq(model="foo", max_retries=0) + other_error = { + "error": { + "message": "Invalid value for 'temperature'.", + "type": "invalid_request_error", + "code": "invalid_value", + } + } + mock_client = MagicMock() + mock_client.create.side_effect = _bad_request_error(other_error) + llm.client = mock_client + + with pytest.raises(groq.BadRequestError) as exc_info: + llm.invoke([HumanMessage(content="test")]) + + assert not isinstance(exc_info.value, ContextOverflowError) + + +def test_context_overflow_error_carries_response_metadata() -> None: + """Promoted `GroqContextOverflowError` preserves `response`/`body`. + + Downstream catchers that introspect `.response.status_code` rely on this. + """ + llm = ChatGroq(model="foo", max_retries=0) + mock_client = MagicMock() + mock_client.create.side_effect = _bad_request_error(_CONTEXT_OVERFLOW_BODY) + llm.client = mock_client + + with pytest.raises(GroqContextOverflowError) as exc_info: + llm.invoke([HumanMessage(content="test")]) + + assert exc_info.value.response.status_code == 400 + assert exc_info.value.body == _CONTEXT_OVERFLOW_BODY + + +# The three detection branches in `_handle_groq_invalid_request` are OR'd, and +# real overflow responses only ever satisfy one at a time, so each is exercised +# in isolation below — otherwise a branch could be deleted without any test +# noticing. + + +def test_handle_invalid_request_promotes_codeless_message() -> None: + """Branch: live overflow has no code; `reduce the length` is the only signal.""" + err = _bad_request_error(_CONTEXT_OVERFLOW_BODY) + # Guard: this shape must not accidentally satisfy the code-based branches. + assert "context_length_exceeded" not in str(err) + + with pytest.raises(GroqContextOverflowError): + _handle_groq_invalid_request(err) + + +def test_handle_invalid_request_promotes_code_in_body() -> None: + """Branch: some responses carry `code` but no `reduce the length` phrase.""" + err = _bad_request_error(_CONTEXT_OVERFLOW_BODY_WITH_CODE) + # Guard: this shape must not accidentally satisfy the message-based branch. + assert "reduce the length" not in str(err) + + with pytest.raises(GroqContextOverflowError): + _handle_groq_invalid_request(err) + + +def test_handle_invalid_request_promotes_code_attribute() -> None: + """Branch: forward-compat guard for a future SDK that exposes `.code`.""" + err = _bad_request_error( + {"error": {"message": "boom", "type": "invalid_request_error"}} + ) + err.code = "context_length_exceeded" # type: ignore[attr-defined] + # Guard: neither string-based branch should fire, so only `.code` promotes. + assert "context_length_exceeded" not in str(err) + assert "reduce the length" not in str(err) + + with pytest.raises(GroqContextOverflowError): + _handle_groq_invalid_request(err) + + +def test_handle_invalid_request_ignores_max_tokens_error() -> None: + """A `max_tokens`-too-large 400 must not be promoted (verified live shape).""" + max_tokens_body = { + "error": { + "message": ( + "`max_tokens` must be less than or equal to `131072`, the maximum " + "value for `max_tokens` is less than the `context_window` for this " + "model" + ), + "type": "invalid_request_error", + "param": "max_tokens", + } + } + err = _bad_request_error(max_tokens_body) + + with pytest.raises(groq.BadRequestError) as exc_info: + _handle_groq_invalid_request(err) + + assert not isinstance(exc_info.value, ContextOverflowError)