fix: add buffer to the condensation

private_gpt/components/chat/models/chat_config_models.py

@@ -468,6 +468,14 @@ class CondensationConfig(BaseModel):
         default=True,
         description="Whether to enable condensation in the chat.",
     )
+    token_buffer: float = Field(
+        default=0.05,
+        ge=0.0,
+        lt=1.0,
+        description=(
+            "Fraction of the model token limit to reserve when condensing chat history."
+        ),
+    )
 
 
 class ThinkingConfig(BaseModel):

private_gpt/server/chat/interceptors/condensation_interceptor.py

@@ -37,6 +37,10 @@ logger = logging.getLogger(__name__)
 _SENTINEL: object = object()
 
 
+def _token_limit_with_buffer(token_limit: int, token_buffer: float) -> int:
+    return max(1, int(token_limit * (1 - token_buffer)))
+
+
 class _CondensationResult(BaseModel):
     chat_history: list[ChatMessage] | None = None
     condensed: bool = False
@@ -166,12 +170,17 @@ class CondensationRequestInterceptor(ChatRequestLoopInterceptor):
         if not self._enabled or not history:
             return
 
+        max_length = _token_limit_with_buffer(
+            context.state.runtime.effective_token_limit,
+            state.input.request.condensation.token_buffer,
+        )
+
         generator = condense_chat_history(
             **state.input.llm_kwargs,
             chat_history=history,
             tools=state.input.context_stack.all_tools(),
             strategy_type=self._strategy_type,
-            max_length=context.state.runtime.effective_token_limit,
+            max_length=max_length,
             tokenizer_fn=context.state.runtime.tokenizer_fn,
             message_to_input=context.llm.messages_to_prompt,
             condensation_timeout=self._condensation_timeout,