make doc utils public (#12394)

This commit is contained in:
Bagatur
2023-10-27 12:08:08 -07:00
committed by GitHub
parent d6888a90d0
commit c2a0a6b6df
5 changed files with 73 additions and 20 deletions

View File

@@ -32,7 +32,7 @@
"from functools import partial\n",
"\n",
"from langchain.callbacks.manager import CallbackManagerForChainRun\n",
"from langchain.chains.combine_documents.reduce import _collapse_docs, _split_list_of_docs\n",
"from langchain.chains.combine_documents import collapse_docs, split_list_of_docs\n",
"from langchain.chat_models import ChatAnthropic\n",
"from langchain.prompts import PromptTemplate\n",
"from langchain.schema import StrOutputParser\n",
@@ -109,8 +109,8 @@
" while get_num_tokens(docs) > token_max:\n",
" config[\"run_name\"] = f\"Collapse {collapse_ct}\"\n",
" invoke = partial(collapse_chain.invoke, config=config)\n",
" split_docs = _split_list_of_docs(docs, get_num_tokens, token_max)\n",
" docs = [_collapse_docs(_docs, invoke) for _docs in split_docs]\n",
" split_docs = split_list_of_docs(docs, get_num_tokens, token_max)\n",
" docs = [collapse_docs(_docs, invoke) for _docs in split_docs]\n",
" collapse_ct += 1\n",
" return docs"
]