multiple: langchain 0.2 in master (#21191)

0.2rc 

migrations

- [x] Move memory
- [x] Move remaining retrievers
- [x] graph_qa chains
- [x] some dependency from evaluation code potentially on math utils
- [x] Move openapi chain from `langchain.chains.api.openapi` to
`langchain_community.chains.openapi`
- [x] Migrate `langchain.chains.ernie_functions` to
`langchain_community.chains.ernie_functions`
- [x] migrate `langchain/chains/llm_requests.py` to
`langchain_community.chains.llm_requests`
- [x] Moving `langchain_community.cross_enoders.base:BaseCrossEncoder`
->
`langchain_community.retrievers.document_compressors.cross_encoder:BaseCrossEncoder`
(namespace not ideal, but it needs to be moved to `langchain` to avoid
circular deps)
- [x] unit tests langchain -- add pytest.mark.community to some unit
tests that will stay in langchain
- [x] unit tests community -- move unit tests that depend on community
to community
- [x] mv integration tests that depend on community to community
- [x] mypy checks

Other todo

- [x] Make deprecation warnings not noisy (need to use warn deprecated
and check that things are implemented properly)
- [x] Update deprecation messages with timeline for code removal (likely
we actually won't be removing things until 0.4 release) -- will give
people more time to transition their code.
- [ ] Add information to deprecation warning to show users how to
migrate their code base using langchain-cli
- [ ] Remove any unnecessary requirements in langchain (e.g., is
SQLALchemy required?)

---------

Co-authored-by: Erick Friis <erick@langchain.dev>

libs/community/tests/integration_tests/document_transformers/test_embeddings_filter.py

@@ -0,0 +1,72 @@
+"""Integration test for embedding-based redundant doc filtering."""
+from langchain_core.documents import Document
+
+from langchain_community.document_transformers.embeddings_redundant_filter import (
+    EmbeddingsClusteringFilter,
+    EmbeddingsRedundantFilter,
+    _DocumentWithState,
+)
+from langchain_community.embeddings import OpenAIEmbeddings
+
+
+def test_embeddings_redundant_filter() -> None:
+    texts = [
+        "What happened to all of my cookies?",
+        "Where did all of my cookies go?",
+        "I wish there were better Italian restaurants in my neighborhood.",
+    ]
+    docs = [Document(page_content=t) for t in texts]
+    embeddings = OpenAIEmbeddings()
+    redundant_filter = EmbeddingsRedundantFilter(embeddings=embeddings)
+    actual = redundant_filter.transform_documents(docs)
+    assert len(actual) == 2
+    assert set(texts[:2]).intersection([d.page_content for d in actual])
+
+
+def test_embeddings_redundant_filter_with_state() -> None:
+    texts = ["What happened to all of my cookies?", "foo bar baz"]
+    state = {"embedded_doc": [0.5] * 10}
+    docs = [_DocumentWithState(page_content=t, state=state) for t in texts]
+    embeddings = OpenAIEmbeddings()
+    redundant_filter = EmbeddingsRedundantFilter(embeddings=embeddings)
+    actual = redundant_filter.transform_documents(docs)
+    assert len(actual) == 1
+
+
+def test_embeddings_clustering_filter() -> None:
+    texts = [
+        "What happened to all of my cookies?",
+        "A cookie is a small, baked sweet treat and you can find it in the cookie",
+        "monsters' jar.",
+        "Cookies are good.",
+        "I have nightmares about the cookie monster.",
+        "The most popular pizza styles are: Neapolitan, New York-style and",
+        "Chicago-style. You can find them on iconic restaurants in major cities.",
+        "Neapolitan pizza: This is the original pizza style,hailing from Naples,",
+        "Italy.",
+        "I wish there were better Italian Pizza restaurants in my neighborhood.",
+        "New York-style pizza: This is characterized by its large, thin crust, and",
+        "generous toppings.",
+        "The first movie to feature a robot was 'A Trip to the Moon' (1902).",
+        "The first movie to feature a robot that could pass for a human was",
+        "'Blade Runner' (1982)",
+        "The first movie to feature a robot that could fall in love with a human",
+        "was 'Her' (2013)",
+        "A robot is a machine capable of carrying out complex actions automatically.",
+        "There are certainly hundreds, if not thousands movies about robots like:",
+        "'Blade Runner', 'Her' and 'A Trip to the Moon'",
+    ]
+
+    docs = [Document(page_content=t) for t in texts]
+    embeddings = OpenAIEmbeddings()
+    redundant_filter = EmbeddingsClusteringFilter(
+        embeddings=embeddings,
+        num_clusters=3,
+        num_closest=1,
+        sorted=True,
+    )
+    actual = redundant_filter.transform_documents(docs)
+    assert len(actual) == 3
+    assert texts[1] in [d.page_content for d in actual]
+    assert texts[4] in [d.page_content for d in actual]
+    assert texts[11] in [d.page_content for d in actual]