From a3c79b1909fe1cbe85394c353b0535117ef0cdf0 Mon Sep 17 00:00:00 2001
From: Bagatur <22008038+baskaryan@users.noreply.github.com>
Date: Wed, 16 Aug 2023 12:09:22 -0700
Subject: [PATCH] Add tiktoken integration dep (#9332)

---
 libs/langchain/poetry.lock    | 6 +++---
 libs/langchain/pyproject.toml | 1 +
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/libs/langchain/poetry.lock b/libs/langchain/poetry.lock
index 91505da4d04..3a1023763da 100644
--- a/libs/langchain/poetry.lock
+++ b/libs/langchain/poetry.lock
@@ -7878,7 +7878,7 @@ name = "regex"
 version = "2023.6.3"
 description = "Alternative regular expression module, to replace re."
 category = "main"
-optional = true
+optional = false
 python-versions = ">=3.6"
 files = [
     {file = "regex-2023.6.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:824bf3ac11001849aec3fa1d69abcb67aac3e150a933963fb12bda5151fe1bfd"},
@@ -9215,7 +9215,7 @@ name = "tiktoken"
 version = "0.3.3"
 description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models"
 category = "main"
-optional = true
+optional = false
 python-versions = ">=3.8"
 files = [
     {file = "tiktoken-0.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d1f37fa75ba70c1bc7806641e8ccea1fba667d23e6341a1591ea333914c226a9"},
@@ -10487,4 +10487,4 @@ text-helpers = ["chardet"]
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.8.1,<4.0"
-content-hash = "6e85bdaca0b4a62bace541dd914266b49a4d7f90c7be2030fab639bf7efc23c6"
+content-hash = "0c6f0b48849161928ab5df69f818779d6f5c58760cf967448a8785c938a23b57"
diff --git a/libs/langchain/pyproject.toml b/libs/langchain/pyproject.toml
index 22d78820674..6641184a5fc 100644
--- a/libs/langchain/pyproject.toml
+++ b/libs/langchain/pyproject.toml
@@ -172,6 +172,7 @@ wrapt = "^1.15.0"
 openai = "^0.27.4"
 python-dotenv = "^1.0.0"
 cassio = "^0.0.7"
+tiktoken = "^0.3.2"
 
 [tool.poetry.group.lint.dependencies]
 ruff = "^0.0.249"