github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-08-02 01:23:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

openai[patch]: unskip test and relax tolerance in embeddings comparison (#28262)

Browse Source 
From what I can tell response using SDK is not deterministic:
```python
import numpy as np
import openai

documents = ["disallowed special token '<|endoftext|>'"]
model = "text-embedding-ada-002"

direct_output_1 = (
    openai.OpenAI()
    .embeddings.create(input=documents, model=model)
    .data[0]
    .embedding
)

for i in range(10):
    direct_output_2 = (
        openai.OpenAI()
        .embeddings.create(input=documents, model=model)
        .data[0]
        .embedding
    )
    print(f"{i}: {np.isclose(direct_output_1, direct_output_2).all()}")
```
```
0: True
1: True
2: True
3: True
4: False
5: True
6: True
7: True
8: True
9: True
```

See related discussion here:
https://community.openai.com/t/can-text-embedding-ada-002-be-made-deterministic/318054

Found the same result using `"text-embedding-3-small"`.
This commit is contained in:
ccurme 2024-11-21 13:23:10 -05:00 committed by GitHub
parent f5f53d1101
commit 56499cf58b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 3 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
libs/partners/openai/tests/integration_tests/embeddings/test_azure.py
View File

@ -117,7 +117,7 @@ def test_azure_openai_embedding_with_empty_string() -> None:
.data[0]
.embedding
)
assert np.allclose(output[0], expected_output, atol=0.0001)
assert np.allclose(output[0], expected_output, atol=0.001)
assert len(output[1]) == 1536

7
libs/partners/openai/tests/integration_tests/embeddings/test_base.py
View File

@ -2,7 +2,6 @@
import numpy as np
import openai
import pytest
from langchain_openai.embeddings.base import OpenAIEmbeddings
@ -33,7 +32,6 @@ def test_langchain_openai_embeddings_dimensions() -> None:
assert len(output[0]) == 128
@pytest.mark.skip(reason="flaky")
def test_langchain_openai_embeddings_equivalent_to_raw() -> None:
documents = ["disallowed special token '<|endoftext|>'"]
embedding = OpenAIEmbeddings()
@ -45,10 +43,9 @@ def test_langchain_openai_embeddings_equivalent_to_raw() -> None:
.data[0]
.embedding
)
assert np.isclose(lc_output, direct_output).all()
assert np.allclose(lc_output, direct_output, atol=0.001)
@pytest.mark.skip(reason="flaky")
async def test_langchain_openai_embeddings_equivalent_to_raw_async() -> None:
documents = ["disallowed special token '<|endoftext|>'"]
embedding = OpenAIEmbeddings()
@ -60,7 +57,7 @@ async def test_langchain_openai_embeddings_equivalent_to_raw_async() -> None:
.data[0]
.embedding
)
assert np.isclose(lc_output, direct_output).all()
assert np.allclose(lc_output, direct_output, atol=0.001)
def test_langchain_openai_embeddings_dimensions_large_num() -> None: