community[minor]: add DashScope Rerank (#22403)

**Description:** this PR adds DashScope Rerank capability to Langchain,
you can find DashScope Rerank API from
[here](https://help.aliyun.com/document_detail/2780058.html?spm=a2c4g.2780059.0.0.6d995024FlrJ12)
&
[here](https://help.aliyun.com/document_detail/2780059.html?spm=a2c4g.2780058.0.0.63f75024cr11N9).
[DashScope](https://dashscope.aliyun.com/) is the generative AI service
from Alibaba Cloud (Aliyun). You can create DashScope API key from
[here](https://bailian.console.aliyun.com/?apiKey=1#/api-key).

**Dependencies:** DashScopeRerank depends on `dashscope` python package.

**Twitter handle:** my twitter/x account is https://x.com/LastMonopoly
and I'd like a mention, thanks you!


**Tests and docs**
  1. integration test: `test_dashscope_rerank.py`
  2. example notebook: `dashscope_rerank.ipynb`

**Lint and test**: I have run `make format`, `make lint` and `make test`
from the root of the package I've modified.

---------

Co-authored-by: Bagatur <baskaryan@gmail.com>
Co-authored-by: Bagatur <22008038+baskaryan@users.noreply.github.com>

libs/community/langchain_community/document_compressors/__init__.py

@@ -2,6 +2,9 @@ import importlib
 from typing import TYPE_CHECKING, Any
 
 if TYPE_CHECKING:
+    from langchain_community.document_compressors.dashscope_rerank import (
+        DashScopeRerank,
+    )
     from langchain_community.document_compressors.flashrank_rerank import (
         FlashrankRerank,
     )
@@ -25,6 +28,7 @@ _module_lookup = {
     "JinaRerank": "langchain_community.document_compressors.jina_rerank",
     "RankLLMRerank": "langchain_community.document_compressors.rankllm_rerank",
     "FlashrankRerank": "langchain_community.document_compressors.flashrank_rerank",
+    "DashScopeRerank": "langchain_community.document_compressors.dashscope_rerank",
 }
 
 
@@ -41,4 +45,5 @@ __all__ = [
     "FlashrankRerank",
     "JinaRerank",
     "RankLLMRerank",
+    "DashScopeRerank",
 ]

libs/community/langchain_community/document_compressors/dashscope_rerank.py

@@ -0,0 +1,119 @@
+from __future__ import annotations
+
+from copy import deepcopy
+from typing import Any, Dict, List, Optional, Sequence, Union
+
+from langchain_core.callbacks.base import Callbacks
+from langchain_core.documents import BaseDocumentCompressor, Document
+from langchain_core.pydantic_v1 import Extra, Field, root_validator
+from langchain_core.utils import get_from_dict_or_env
+
+
+class DashScopeRerank(BaseDocumentCompressor):
+    """Document compressor that uses `DashScope Rerank API`."""
+
+    client: Any = None
+    """DashScope client to use for compressing documents."""
+
+    model: Optional[str] = None
+    """Model to use for reranking."""
+
+    top_n: Optional[int] = 3
+    """Number of documents to return."""
+
+    dashscope_api_key: Optional[str] = Field(None, alias="api_key")
+    """DashScope API key. Must be specified directly or via environment variable 
+        DASHSCOPE_API_KEY."""
+
+    class Config:
+        """Configuration for this pydantic object."""
+
+        extra = Extra.forbid
+        arbitrary_types_allowed = True
+        allow_population_by_field_name = True
+
+    @root_validator()
+    def validate_environment(cls, values: Dict) -> Dict:
+        """Validate that api key and python package exists in environment."""
+
+        if not values.get("client"):
+            try:
+                import dashscope
+            except ImportError:
+                raise ImportError(
+                    "Could not import dashscope python package. "
+                    "Please install it with `pip install dashscope`."
+                )
+
+            values["client"] = dashscope.TextReRank
+            values["dashscope_api_key"] = get_from_dict_or_env(
+                values, "dashscope_api_key", "DASHSCOPE_API_KEY"
+            )
+            values["model"] = dashscope.TextReRank.Models.gte_rerank
+
+        return values
+
+    def rerank(
+        self,
+        documents: Sequence[Union[str, Document, dict]],
+        query: str,
+        *,
+        top_n: Optional[int] = -1,
+    ) -> List[Dict[str, Any]]:
+        """Returns an ordered list of documents ordered by their relevance to the provided query.
+
+        Args:
+            query: The query to use for reranking.
+            documents: A sequence of documents to rerank.
+            top_n : The number of results to return. If None returns all results.
+                Defaults to self.top_n.
+        """  # noqa: E501
+
+        if len(documents) == 0:  # to avoid empty api call
+            return []
+        docs = [
+            doc.page_content if isinstance(doc, Document) else doc for doc in documents
+        ]
+
+        top_n = top_n if (top_n is None or top_n > 0) else self.top_n
+
+        results = self.client.call(
+            model=self.model,
+            query=query,
+            documents=docs,
+            top_n=top_n,
+            return_documents=False,
+            api_key=self.dashscope_api_key,
+        )
+
+        result_dicts = []
+        for res in results.output.results:
+            result_dicts.append(
+                {"index": res.index, "relevance_score": res.relevance_score}
+            )
+        return result_dicts
+
+    def compress_documents(
+        self,
+        documents: Sequence[Document],
+        query: str,
+        callbacks: Optional[Callbacks] = None,
+    ) -> Sequence[Document]:
+        """
+        Compress documents using DashScope's rerank API.
+
+        Args:
+            documents: A sequence of documents to compress.
+            query: The query to use for compressing the documents.
+            callbacks: Callbacks to run during the compression process.
+
+        Returns:
+            A sequence of compressed documents.
+        """
+        compressed = []
+        for res in self.rerank(documents, query):
+            doc = documents[res["index"]]
+            doc_copy = Document(doc.page_content, metadata=deepcopy(doc.metadata))
+            doc_copy.metadata["relevance_score"] = res["relevance_score"]
+            compressed.append(doc_copy)
+        return compressed

libs/community/tests/integration_tests/document_compressors/test_dashscope_rerank.py

@@ -0,0 +1,24 @@
+from langchain_core.documents import Document
+
+from langchain_community.document_compressors.dashscope_rerank import (
+    DashScopeRerank,
+)
+
+
+def test_rerank() -> None:
+    reranker = DashScopeRerank(api_key=None)
+    docs = [
+        Document(page_content="量子计算是计算科学的一个前沿领域"),
+        Document(page_content="预训练语言模型的发展给文本排序模型带来了新的进展"),
+        Document(
+            page_content="文本排序模型广泛用于搜索引擎和推荐系统中，它们根据文本相关性对候选文本进行排序"
+        ),
+        Document(page_content="random text for nothing"),
+    ]
+    compressed = reranker.compress_documents(
+        query="什么是文本排序模型",
+        documents=docs,
+    )
+
+    assert len(compressed) == 3, "default top_n is 3"
+    assert compressed[0].page_content == docs[2].page_content, "rerank works"

libs/community/tests/unit_tests/document_compressors/test_imports.py

@@ -6,6 +6,7 @@ EXPECTED_ALL = [
     "JinaRerank",
     "RankLLMRerank",
     "FlashrankRerank",
+    "DashScopeRerank",
 ]