github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-08-09 13:00:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

[OpenAI]: Encoding Model (#31402)

Browse Source 
- **Description:** Small Fix for when getting the encoder in case of
KeyError and using the correct encoder for newer models
- **Issue:** #31390
This commit is contained in:
Mohammad Mohtashim 2025-06-11 01:00:00 +05:00 committed by GitHub
parent 40bd71caa5
commit 42eb356a44
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 6 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
libs/partners/openai/langchain_openai/chat_models/base.py
View File

@ -1259,8 +1259,12 @@ class BaseChatOpenAI(BaseChatModel):
try: try:
encoding = tiktoken.encoding_for_model(model) encoding = tiktoken.encoding_for_model(model)
except KeyError: except KeyError:
model = "cl100k_base" encoder = "cl100k_base"
encoding = tiktoken.get_encoding(model) if self.model_name.startswith("gpt-4o") or self.model_name.startswith(
"gpt-4.1"
):
encoder = "o200k_base"
encoding = tiktoken.get_encoding(encoder)
return model, encoding return model, encoding
def get_token_ids(self, text: str) -> list[int]: def get_token_ids(self, text: str) -> list[int]: