Add String Distance and Embedding Evaluators (#7123)

Add a string evaluator and pairwise string evaluator implementation for:
- Embedding distance
- String distance

Update docs

tests/integration_tests/evaluation/embedding_distance/test_embedding.py

@@ -0,0 +1,123 @@
+from typing import Tuple
+
+import numpy as np
+import pytest
+
+from langchain.evaluation.embedding_distance import (
+    EmbeddingDistance,
+    PairwiseEmbeddingDistanceEvalChain,
+)
+
+
+@pytest.fixture
+def vectors() -> Tuple[np.ndarray, np.ndarray]:
+    """Create two random vectors."""
+    vector_a = np.array(
+        [
+            0.5488135,
+            0.71518937,
+            0.60276338,
+            0.54488318,
+            0.4236548,
+            0.64589411,
+            0.43758721,
+            0.891773,
+            0.96366276,
+            0.38344152,
+        ]
+    )
+    vector_b = np.array(
+        [
+            0.79172504,
+            0.52889492,
+            0.56804456,
+            0.92559664,
+            0.07103606,
+            0.0871293,
+            0.0202184,
+            0.83261985,
+            0.77815675,
+            0.87001215,
+        ]
+    )
+    return vector_a, vector_b
+
+
+@pytest.fixture
+def chain() -> PairwiseEmbeddingDistanceEvalChain:
+    """Create a PairwiseEmbeddingDistanceEvalChain."""
+    return PairwiseEmbeddingDistanceEvalChain()
+
+
+@pytest.mark.requires("scipy")
+def test_cosine_similarity(
+    chain: PairwiseEmbeddingDistanceEvalChain, vectors: Tuple[np.ndarray, np.ndarray]
+) -> None:
+    """Test the cosine similarity."""
+    chain.distance_metric = EmbeddingDistance.COSINE
+    result = chain._compute_score(np.array(vectors))
+    expected = 1.0 - np.dot(vectors[0], vectors[1]) / (
+        np.linalg.norm(vectors[0]) * np.linalg.norm(vectors[1])
+    )
+    assert np.isclose(result, expected)
+
+
+@pytest.mark.requires("scipy")
+def test_euclidean_distance(
+    chain: PairwiseEmbeddingDistanceEvalChain, vectors: Tuple[np.ndarray, np.ndarray]
+) -> None:
+    """Test the euclidean distance."""
+    from scipy.spatial.distance import euclidean
+
+    chain.distance_metric = EmbeddingDistance.EUCLIDEAN
+    result = chain._compute_score(np.array(vectors))
+    expected = euclidean(*vectors)
+    assert np.isclose(result, expected)
+
+
+@pytest.mark.requires("scipy")
+def test_manhattan_distance(
+    chain: PairwiseEmbeddingDistanceEvalChain, vectors: Tuple[np.ndarray, np.ndarray]
+) -> None:
+    """Test the manhattan distance."""
+    from scipy.spatial.distance import cityblock
+
+    chain.distance_metric = EmbeddingDistance.MANHATTAN
+    result = chain._compute_score(np.array(vectors))
+    expected = cityblock(*vectors)
+    assert np.isclose(result, expected)
+
+
+@pytest.mark.requires("scipy")
+def test_chebyshev_distance(
+    chain: PairwiseEmbeddingDistanceEvalChain, vectors: Tuple[np.ndarray, np.ndarray]
+) -> None:
+    """Test the chebyshev distance."""
+    from scipy.spatial.distance import chebyshev
+
+    chain.distance_metric = EmbeddingDistance.CHEBYSHEV
+    result = chain._compute_score(np.array(vectors))
+    expected = chebyshev(*vectors)
+    assert np.isclose(result, expected)
+
+
+@pytest.mark.requires("scipy")
+def test_hamming_distance(
+    chain: PairwiseEmbeddingDistanceEvalChain, vectors: Tuple[np.ndarray, np.ndarray]
+) -> None:
+    """Test the hamming distance."""
+    from scipy.spatial.distance import hamming
+
+    chain.distance_metric = EmbeddingDistance.HAMMING
+    result = chain._compute_score(np.array(vectors))
+    expected = hamming(*vectors)
+    assert np.isclose(result, expected)
+
+
+@pytest.mark.requires("openai", "tiktoken")
+def test_embedding_distance(chain: PairwiseEmbeddingDistanceEvalChain) -> None:
+    """Test the embedding distance."""
+    result = chain.evaluate_string_pairs(
+        prediction="A single cat", prediction_b="A single cat"
+    )
+    assert np.isclose(result["score"], 0.0)

tests/unit_tests/evaluation/string_distance/test_base.py

@@ -0,0 +1,51 @@
+import pytest
+
+from langchain.evaluation.string_distance import (
+    PairwiseStringDistanceEvalChain,
+    StringDistance,
+    StringDistanceEvalChain,
+)
+
+
+@pytest.mark.requires("rapidfuzz")
+@pytest.mark.parametrize("distance", list(StringDistance))
+def test_zero_distance(distance: StringDistance) -> None:
+    eval_chain = StringDistanceEvalChain(distance=distance)
+    string = "三人行则必有我师"
+    result = eval_chain.evaluate_strings(prediction=string, reference=string)
+    assert "score" in result
+    assert result["score"] == 0
+
+
+@pytest.mark.asyncio
+@pytest.mark.requires("rapidfuzz")
+@pytest.mark.parametrize("distance", list(StringDistance))
+async def test_zero_distance_async(distance: StringDistance) -> None:
+    eval_chain = StringDistanceEvalChain(distance=distance)
+    string = "三人行则必有我师"
+    result = await eval_chain.aevaluate_strings(prediction=string, reference=string)
+    assert "score" in result
+    assert result["score"] == 0
+
+
+@pytest.mark.requires("rapidfuzz")
+@pytest.mark.parametrize("distance", list(StringDistance))
+def test_zero_distance_pairwise(distance: StringDistance) -> None:
+    eval_chain = PairwiseStringDistanceEvalChain(distance=distance)
+    string = "三人行则必有我师"
+    result = eval_chain.evaluate_string_pairs(prediction=string, prediction_b=string)
+    assert "score" in result
+    assert result["score"] == 0
+
+
+@pytest.mark.asyncio
+@pytest.mark.requires("rapidfuzz")
+@pytest.mark.parametrize("distance", list(StringDistance))
+async def test_zero_distance_pairwise_async(distance: StringDistance) -> None:
+    eval_chain = PairwiseStringDistanceEvalChain(distance=distance)
+    string = "三人行则必有我师"
+    result = await eval_chain.aevaluate_string_pairs(
+        prediction=string, prediction_b=string
+    )
+    assert "score" in result
+    assert result["score"] == 0

tests/unit_tests/evaluation/test_loading.py

@@ -2,20 +2,27 @@
 
 import pytest
 
+from langchain.embeddings.fake import FakeEmbeddings
 from langchain.evaluation.loading import EvaluatorType, load_evaluators
 from langchain.evaluation.schema import StringEvaluator
 from tests.unit_tests.llms.fake_chat_model import FakeChatModel
 from tests.unit_tests.llms.fake_llm import FakeLLM
 
 
+@pytest.mark.requires("rapidfuzz")
 @pytest.mark.parametrize("evaluator_type", EvaluatorType)
 def test_load_evaluators(evaluator_type: EvaluatorType) -> None:
     """Test loading evaluators."""
     fake_llm = FakeChatModel()
-    load_evaluators([evaluator_type], llm=fake_llm)
+    embeddings = FakeEmbeddings(size=32)
+    load_evaluators([evaluator_type], llm=fake_llm, embeddings=embeddings)
 
     # Test as string
-    load_evaluators([evaluator_type.value], llm=fake_llm)  # type: ignore
+    load_evaluators(
+        [evaluator_type.value],  # type: ignore
+        llm=fake_llm,
+        embeddings=embeddings,
+    )
 
 
 def test_criteria_eval_chain_requires_reference() -> None: