mirror of
https://github.com/hwchase17/langchain.git
synced 2025-09-06 13:33:37 +00:00
community[minor]: Add Baichuan Text Embedding Model and Baichuan Inc introduction (#16568)
- **Description:** Adding Baichuan Text Embedding Model and Baichuan Inc introduction. Baichuan Text Embedding ranks #1 in C-MTEB leaderboard: https://huggingface.co/spaces/mteb/leaderboard Co-authored-by: BaiChuanHelper <wintergyc@WinterGYCs-MacBook-Pro.local>
This commit is contained in:
baichuan-assistant
committed by
GitHub
GitHub
parent
5b5115c408
commit
70ff54eace
13
docs/docs/integrations/providers/baichuan.mdx
Normal file
13
docs/docs/integrations/providers/baichuan.mdx
Normal file
@@ -0,0 +1,13 @@
|
||||
# Baichuan
|
||||
|
||||
>[Baichuan Inc.](https://www.baichuan-ai.com/) is a Chinese startup in the era of AGI, dedicated to addressing fundamental human needs: Efficiency, Health, and Happiness.
|
||||
|
||||
## Visit Us
|
||||
Visit us at https://www.baichuan-ai.com/.
|
||||
Register and get an API key if you are trying out our APIs.
|
||||
|
||||
## Baichuan Chat Model
|
||||
An example is available at [example](/docs/integrations/chat/baichuan).
|
||||
|
||||
## Baichuan Text Embedding Model
|
||||
An example is available at [example] (/docs/integrations/text_embedding/baichuan)
|
||||
75
docs/docs/integrations/text_embedding/baichuan.ipynb
Normal file
75
docs/docs/integrations/text_embedding/baichuan.ipynb
Normal file
@@ -0,0 +1,75 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Baichuan Text Embeddings\n",
|
||||
"\n",
|
||||
"As of today (Jan 25th, 2024) BaichuanTextEmbeddings ranks #1 in C-MTEB (Chinese Multi-Task Embedding Benchmark) leaderboard.\n",
|
||||
"\n",
|
||||
"Leaderboard (Under Overall -> Chinese section): https://huggingface.co/spaces/mteb/leaderboard\n",
|
||||
"\n",
|
||||
"Official Website: https://platform.baichuan-ai.com/docs/text-Embedding\n",
|
||||
"An API-key is required to use this embedding model. You can get one by registering at https://platform.baichuan-ai.com/docs/text-Embedding.\n",
|
||||
"BaichuanTextEmbeddings support 512 token window and preduces vectors with 1024 dimensions. \n",
|
||||
"\n",
|
||||
"Please NOTE that BaichuanTextEmbeddings only supports Chinese text embedding. Multi-language support is coming soon.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"vscode": {
|
||||
"languageId": "plaintext"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain_community.embeddings import BaichuanTextEmbeddings\n",
|
||||
"\n",
|
||||
"# Place your Baichuan API-key here.\n",
|
||||
"embeddings = BaichuanTextEmbeddings(baichuan_api_key=\"sk-*\")\n",
|
||||
"\n",
|
||||
"text_1 = \"今天天气不错\"\n",
|
||||
"text_2 = \"今天阳光很好\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"vscode": {
|
||||
"languageId": "plaintext"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"query_result = embeddings.embed_query(text_1)\n",
|
||||
"query_result"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"vscode": {
|
||||
"languageId": "plaintext"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"doc_result = embeddings.embed_documents([text_1, text_2])\n",
|
||||
"doc_result"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@@ -167,9 +167,9 @@
|
||||
],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"URL = 'https://www.conseil-constitutionnel.fr/node/3850/pdf'\n",
|
||||
"PDF = 'Déclaration_des_droits_de_l_homme_et_du_citoyen.pdf'\n",
|
||||
"open(PDF, 'wb').write(requests.get(URL).content)"
|
||||
"URL = \"https://www.conseil-constitutionnel.fr/node/3850/pdf\"\n",
|
||||
"PDF = \"Déclaration_des_droits_de_l_homme_et_du_citoyen.pdf\"\n",
|
||||
"open(PDF, \"wb\").write(requests.get(URL).content)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -208,7 +208,7 @@
|
||||
],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"print('Read a PDF...')\n",
|
||||
"print(\"Read a PDF...\")\n",
|
||||
"loader = PyPDFLoader(PDF)\n",
|
||||
"pages = loader.load_and_split()\n",
|
||||
"len(pages)"
|
||||
@@ -252,12 +252,14 @@
|
||||
],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"print('Create a Vector Database from PDF text...')\n",
|
||||
"embeddings = OpenAIEmbeddings(model='text-embedding-ada-002')\n",
|
||||
"print(\"Create a Vector Database from PDF text...\")\n",
|
||||
"embeddings = OpenAIEmbeddings(model=\"text-embedding-ada-002\")\n",
|
||||
"texts = [p.page_content for p in pages]\n",
|
||||
"metadata = pd.DataFrame(index=list(range(len(texts))))\n",
|
||||
"metadata['tag'] = 'law'\n",
|
||||
"metadata['title'] = 'Déclaration des Droits de l\\'Homme et du Citoyen de 1789'.encode('utf-8')\n",
|
||||
"metadata[\"tag\"] = \"law\"\n",
|
||||
"metadata[\"title\"] = \"Déclaration des Droits de l'Homme et du Citoyen de 1789\".encode(\n",
|
||||
" \"utf-8\"\n",
|
||||
")\n",
|
||||
"vectordb = KDBAI(table, embeddings)\n",
|
||||
"vectordb.add_texts(texts=texts, metadatas=metadata)"
|
||||
]
|
||||
@@ -288,11 +290,13 @@
|
||||
],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"print('Create LangChain Pipeline...')\n",
|
||||
"qabot = RetrievalQA.from_chain_type(chain_type='stuff',\n",
|
||||
" llm=ChatOpenAI(model='gpt-3.5-turbo-16k', temperature=TEMP), \n",
|
||||
" retriever=vectordb.as_retriever(search_kwargs=dict(k=K)),\n",
|
||||
" return_source_documents=True)"
|
||||
"print(\"Create LangChain Pipeline...\")\n",
|
||||
"qabot = RetrievalQA.from_chain_type(\n",
|
||||
" chain_type=\"stuff\",\n",
|
||||
" llm=ChatOpenAI(model=\"gpt-3.5-turbo-16k\", temperature=TEMP),\n",
|
||||
" retriever=vectordb.as_retriever(search_kwargs=dict(k=K)),\n",
|
||||
" return_source_documents=True,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -325,9 +329,9 @@
|
||||
],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"Q = 'Summarize the document in English:'\n",
|
||||
"print(f'\\n\\n{Q}\\n')\n",
|
||||
"print(qabot.invoke(dict(query=Q))['result'])"
|
||||
"Q = \"Summarize the document in English:\"\n",
|
||||
"print(f\"\\n\\n{Q}\\n\")\n",
|
||||
"print(qabot.invoke(dict(query=Q))[\"result\"])"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -362,9 +366,9 @@
|
||||
],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"Q = 'Is it a fair law and why ?'\n",
|
||||
"print(f'\\n\\n{Q}\\n')\n",
|
||||
"print(qabot.invoke(dict(query=Q))['result'])"
|
||||
"Q = \"Is it a fair law and why ?\"\n",
|
||||
"print(f\"\\n\\n{Q}\\n\")\n",
|
||||
"print(qabot.invoke(dict(query=Q))[\"result\"])"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -414,9 +418,9 @@
|
||||
],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"Q = 'What are the rights and duties of the man, the citizen and the society ?'\n",
|
||||
"print(f'\\n\\n{Q}\\n')\n",
|
||||
"print(qabot.invoke(dict(query=Q))['result'])"
|
||||
"Q = \"What are the rights and duties of the man, the citizen and the society ?\"\n",
|
||||
"print(f\"\\n\\n{Q}\\n\")\n",
|
||||
"print(qabot.invoke(dict(query=Q))[\"result\"])"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -441,9 +445,9 @@
|
||||
],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"Q = 'Is this law practical ?'\n",
|
||||
"print(f'\\n\\n{Q}\\n')\n",
|
||||
"print(qabot.invoke(dict(query=Q))['result'])"
|
||||
"Q = \"Is this law practical ?\"\n",
|
||||
"print(f\"\\n\\n{Q}\\n\")\n",
|
||||
"print(qabot.invoke(dict(query=Q))[\"result\"])"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user