feat:knowledge space and document delete (#403)

1.add knowledge space delete and document delete
2.knowledge chat route
3.docs, readme, star history
4.fix csv loader bug, add new csv_loader
Close #396

README.md

@@ -60,6 +60,27 @@ https://github.com/csunny/DB-GPT/assets/13723926/55f31781-1d49-4757-b96e-7ef6d3d
       <source id="mp4" src="https://github.com/csunny/DB-GPT/assets/17919400/654b5a49-5ea4-4c02-b5b2-72d089dcc1f0" type="video/mp4">
 </videos> -->
 
+#### Chat with data, and figure charts.
+
+<p align="left">
+  <img src="./assets/dashboard.png" width="800px" />
+</p>
+
+#### Text2SQL, generate SQL from chat
+<p align="left">
+  <img src="./assets/chatdata.png" width="800px" />
+</p>
+
+#### Knowledge space to manage docs.
+<p align="left">
+  <img src="./assets/ks.png" width="800px" />
+</p>
+
+#### Chat with knowledge, such as url, pdf, csv, word. etc
+<p align="left">
+  <img src="./assets/chat_knowledge.png" width="800px" />
+</p>
+
 ## Features
 
 Currently, we have released multiple key features, which are listed below to demonstrate our current capabilities:
@@ -92,6 +113,9 @@ Currently, we have released multiple key features, which are listed below to dem
   - Gorilla(7b,13b)
   - baichuan(7b,13b)
 
+[![Star History Chart](https://api.star-history.com/svg?repos=csunny/DB-GPT)](https://star-history.com/#csunny/DB-GPT)
+
+
 ## Introduction 
 DB-GPT creates a vast model operating system using [FastChat](https://github.com/lm-sys/FastChat) and offers a large language model powered by [Vicuna](https://huggingface.co/Tribbiani/vicuna-7b). In addition, we provide private domain knowledge base question-answering capability. Furthermore, we also provide support for additional plugins, and our design natively supports the Auto-GPT plugin.Our vision is to make it easier and more convenient to build  applications around databases and llm.
 

pilot/embedding_engine/csv_embedding.py

@@ -1,6 +1,5 @@
-from typing import Dict, List, Optional
+from typing import List, Optional
 
-from langchain.document_loaders import CSVLoader
 from langchain.schema import Document
 from langchain.text_splitter import (
     TextSplitter,
@@ -9,6 +8,7 @@ from langchain.text_splitter import (
 )
 
 from pilot.embedding_engine import SourceEmbedding, register
+from pilot.embedding_engine.loader.csv_loader import NewCSVLoader
 
 
 class CSVEmbedding(SourceEmbedding):
@@ -34,7 +34,7 @@ class CSVEmbedding(SourceEmbedding):
     def read(self):
         """Load from csv path."""
         if self.source_reader is None:
-            self.source_reader = CSVLoader(self.file_path)
+            self.source_reader = NewCSVLoader(self.file_path)
         if self.text_splitter is None:
             try:
                 self.text_splitter = SpacyTextSplitter(

pilot/embedding_engine/loader/csv_loader.py

@@ -0,0 +1,76 @@
+"""Loads a CSV file into a list of documents.
+
+Each document represents one row of the CSV file. Every row is converted into a
+key/value pair and outputted to a new line in the document's page_content.
+
+The source for each document loaded from csv is set to the value of the
+`file_path` argument for all doucments by default.
+You can override this by setting the `source_column` argument to the
+name of a column in the CSV file.
+The source of each document will then be set to the value of the column
+with the name specified in `source_column`.
+
+Output Example:
+    .. code-block:: txt
+
+        column1: value1
+        column2: value2
+        column3: value3
+"""
+from typing import Optional, Dict, List
+import csv
+from langchain.document_loaders.base import BaseLoader
+from langchain.schema import Document
+
+
+class NewCSVLoader(BaseLoader):
+    def __init__(
+        self,
+        file_path: str,
+        source_column: Optional[str] = None,
+        csv_args: Optional[Dict] = None,
+        encoding: Optional[str] = None,
+    ):
+        """
+
+        Args:
+            file_path: The path to the CSV file.
+            source_column: The name of the column in the CSV file to use as the source.
+              Optional. Defaults to None.
+            csv_args: A dictionary of arguments to pass to the csv.DictReader.
+              Optional. Defaults to None.
+            encoding: The encoding of the CSV file. Optional. Defaults to None.
+        """
+        self.file_path = file_path
+        self.source_column = source_column
+        self.encoding = encoding
+        self.csv_args = csv_args or {}
+
+    def load(self) -> List[Document]:
+        """Load data into document objects."""
+
+        docs = []
+        with open(self.file_path, newline="", encoding=self.encoding) as csvfile:
+            csv_reader = csv.DictReader(csvfile, **self.csv_args)  # type: ignore
+            for i, row in enumerate(csv_reader):
+                strs = []
+                for k, v in row.items():
+                    if k is None or v is None:
+                        continue
+                    strs.append(f"{k.strip()}: {v.strip()}")
+                content = "\n".join(strs)
+                try:
+                    source = (
+                        row[self.source_column]
+                        if self.source_column is not None
+                        else self.file_path
+                    )
+                except KeyError:
+                    raise ValueError(
+                        f"Source column '{self.source_column}' not found in CSV file."
+                    )
+                metadata = {"source": source, "row": i}
+                doc = Document(page_content=content, metadata=metadata)
+                docs.append(doc)
+
+        return docs

pilot/embedding_engine/markdown_embedding.py

@@ -14,7 +14,7 @@ from langchain.text_splitter import (
 )
 
 from pilot.embedding_engine import SourceEmbedding, register
-from pilot.embedding_engine.EncodeTextLoader import EncodeTextLoader
+from pilot.embedding_engine.encode_text_loader import EncodeTextLoader
 
 
 class MarkdownEmbedding(SourceEmbedding):

pilot/embedding_engine/ppt_embedding.py

@@ -10,7 +10,7 @@ from langchain.text_splitter import (
 )
 
 from pilot.embedding_engine import SourceEmbedding, register
-from pilot.embedding_engine.ppt_loader import PPTLoader
+from pilot.embedding_engine.loader.ppt_loader import PPTLoader
 
 
 class PPTEmbedding(SourceEmbedding):

pilot/embedding_engine/word_embedding.py

@@ -10,7 +10,7 @@ from langchain.text_splitter import (
 )
 
 from pilot.embedding_engine import SourceEmbedding, register
-from pilot.embedding_engine.docx_loader import DocxLoader
+from pilot.embedding_engine.loader.docx_loader import DocxLoader
 
 
 class WordEmbedding(SourceEmbedding):

pilot/server/knowledge/api.py

@@ -52,6 +52,15 @@ def space_list(request: KnowledgeSpaceRequest):
         return Result.faild(code="E000X", msg=f"space list error {e}")
 
 
+@router.post("/knowledge/space/delete")
+def space_delete(request: KnowledgeSpaceRequest):
+    print(f"/space/list params:")
+    try:
+        return Result.succ(knowledge_space_service.delete_space(request.name))
+    except Exception as e:
+        return Result.faild(code="E000X", msg=f"space list error {e}")
+
+
 @router.post("/knowledge/{space_name}/document/add")
 def document_add(space_name: str, request: KnowledgeDocumentRequest):
     print(f"/document/add params: {space_name}, {request}")
@@ -77,6 +86,17 @@ def document_list(space_name: str, query_request: DocumentQueryRequest):
         return Result.faild(code="E000X", msg=f"document list error {e}")
 
 
+@router.post("/knowledge/{space_name}/document/delete")
+def document_delete(space_name: str, query_request: DocumentQueryRequest):
+    print(f"/document/list params: {space_name}, {query_request}")
+    try:
+        return Result.succ(
+            knowledge_space_service.delete_document(space_name, query_request.doc_name)
+        )
+    except Exception as e:
+        return Result.faild(code="E000X", msg=f"document list error {e}")
+
+
 @router.post("/knowledge/{space_name}/document/upload")
 async def document_upload(
     space_name: str,

pilot/server/knowledge/chunk_db.py

@@ -81,6 +81,7 @@ class DocumentChunkDao:
             page_size
         )
         result = document_chunks.all()
+        session.close()
         return result
 
     def get_document_chunks_count(self, query: DocumentChunkEntity):
@@ -105,6 +106,7 @@ class DocumentChunkDao:
                 DocumentChunkEntity.meta_info == query.meta_info
             )
         count = document_chunks.scalar()
+        session.close()
         return count
 
     # def update_knowledge_document(self, document:KnowledgeDocumentEntity):
@@ -113,9 +115,16 @@ class DocumentChunkDao:
     #     session.commit()
     #     return updated_space.id
 
-    # def delete_knowledge_document(self, document_id:int):
-    #     cursor = self.conn.cursor()
-    #     query = "DELETE FROM knowledge_document WHERE id = %s"
-    #     cursor.execute(query, (document_id,))
-    #     self.conn.commit()
-    #     cursor.close()
+    def delete(self, document_id: int):
+        session = self.Session()
+        if document_id is None:
+            raise Exception("document_id is None")
+        query = DocumentChunkEntity(document_id=document_id)
+        knowledge_documents = session.query(DocumentChunkEntity)
+        if query.document_id is not None:
+            chunks = knowledge_documents.filter(
+                DocumentChunkEntity.document_id == query.document_id
+            )
+        chunks.delete()
+        session.commit()
+        session.close()

pilot/server/knowledge/document_db.py

@@ -91,6 +91,38 @@ class KnowledgeDocumentDao:
             page_size
         )
         result = knowledge_documents.all()
+        session.close()
+        return result
+
+    def get_documents(self, query):
+        session = self.Session()
+        knowledge_documents = session.query(KnowledgeDocumentEntity)
+        if query.id is not None:
+            knowledge_documents = knowledge_documents.filter(
+                KnowledgeDocumentEntity.id == query.id
+            )
+        if query.doc_name is not None:
+            knowledge_documents = knowledge_documents.filter(
+                KnowledgeDocumentEntity.doc_name == query.doc_name
+            )
+        if query.doc_type is not None:
+            knowledge_documents = knowledge_documents.filter(
+                KnowledgeDocumentEntity.doc_type == query.doc_type
+            )
+        if query.space is not None:
+            knowledge_documents = knowledge_documents.filter(
+                KnowledgeDocumentEntity.space == query.space
+            )
+        if query.status is not None:
+            knowledge_documents = knowledge_documents.filter(
+                KnowledgeDocumentEntity.status == query.status
+            )
+
+        knowledge_documents = knowledge_documents.order_by(
+            KnowledgeDocumentEntity.id.desc()
+        )
+        result = knowledge_documents.all()
+        session.close()
         return result
 
     def get_knowledge_documents_count(self, query):
@@ -117,6 +149,7 @@ class KnowledgeDocumentDao:
                 KnowledgeDocumentEntity.status == query.status
             )
         count = knowledge_documents.scalar()
+        session.close()
         return count
 
     def update_knowledge_document(self, document: KnowledgeDocumentEntity):
@@ -126,9 +159,21 @@ class KnowledgeDocumentDao:
         return updated_space.id
 
     #
-    # def delete_knowledge_document(self, document_id: int):
-    #     cursor = self.conn.cursor()
-    #     query = "DELETE FROM knowledge_document WHERE id = %s"
-    #     cursor.execute(query, (document_id,))
-    #     self.conn.commit()
-    #     cursor.close()
+    def delete(self, query: KnowledgeDocumentEntity):
+        session = self.Session()
+        knowledge_documents = session.query(KnowledgeDocumentEntity)
+        if query.id is not None:
+            knowledge_documents = knowledge_documents.filter(
+                KnowledgeDocumentEntity.id == query.id
+            )
+        if query.doc_name is not None:
+            knowledge_documents = knowledge_documents.filter(
+                KnowledgeDocumentEntity.doc_name == query.doc_name
+            )
+        if query.space is not None:
+            knowledge_documents = knowledge_documents.filter(
+                KnowledgeDocumentEntity.doc_name == query.doc_name
+            )
+        knowledge_documents.delete()
+        session.commit()
+        session.close()

pilot/server/knowledge/request/response.py

@@ -26,6 +26,7 @@ class DocumentQueryResponse(BaseModel):
 class SpaceQueryResponse(BaseModel):
     """data: data"""
 
+    id: int = None
     name: str = None
     """vector_type: vector type"""
     vector_type: str = None
@@ -33,5 +34,7 @@ class SpaceQueryResponse(BaseModel):
     desc: str = None
     """owner: owner"""
     owner: str = None
+    gmt_created: str = None
+    gmt_modified: str = None
     """doc_count: doc_count"""
-    doc_count: int = None
+    docs: int = None

pilot/server/knowledge/service.py

@@ -2,6 +2,7 @@ import threading
 from datetime import datetime
 
 from langchain.text_splitter import RecursiveCharacterTextSplitter, SpacyTextSplitter
+from pilot.vector_store.connector import VectorStoreConnector
 
 from pilot.configs.config import Config
 from pilot.configs.model_config import LLM_MODEL_CONFIG, KNOWLEDGE_UPLOAD_ROOT_PATH
@@ -89,7 +90,23 @@ class KnowledgeService:
         query = KnowledgeSpaceEntity(
             name=request.name, vector_type=request.vector_type, owner=request.owner
         )
-        return knowledge_space_dao.get_knowledge_space(query)
+        responses = []
+        spaces = knowledge_space_dao.get_knowledge_space(query)
+        for space in spaces:
+            res = SpaceQueryResponse()
+            res.id = space.id
+            res.name = space.name
+            res.vector_type = space.vector_type
+            res.desc = space.desc
+            res.owner = space.owner
+            res.gmt_created = space.gmt_created
+            res.gmt_modified = space.gmt_modified
+            res.owner = space.owner
+            query = KnowledgeDocumentEntity(space=space.name)
+            doc_count = knowledge_document_dao.get_knowledge_documents_count(query)
+            res.docs = doc_count
+            responses.append(res)
+        return responses
 
     """get knowledge get_knowledge_documents"""
 
@@ -191,8 +208,51 @@ class KnowledgeService:
 
     """delete knowledge space"""
 
-    def delete_knowledge_space(self, space_id: int):
-        return knowledge_space_dao.delete_knowledge_space(space_id)
+    def delete_space(self, space_name: str):
+        query = KnowledgeSpaceEntity(name=space_name)
+        spaces = knowledge_space_dao.get_knowledge_space(query)
+        if len(spaces) == 0:
+            raise Exception(f"delete error, no space name:{space_name} in database")
+        space = spaces[0]
+        vector_config = {}
+        vector_config["vector_store_name"] = space.name
+        vector_config["vector_store_type"] = CFG.VECTOR_STORE_TYPE
+        vector_config["chroma_persist_path"] = KNOWLEDGE_UPLOAD_ROOT_PATH
+        vector_client = VectorStoreConnector(
+            vector_store_type=CFG.VECTOR_STORE_TYPE, ctx=vector_config
+        )
+        # delete vectors
+        vector_client.delete_vector_name(space.name)
+        document_query = KnowledgeDocumentEntity(space=space.name)
+        # delete chunks
+        documents = knowledge_document_dao.get_documents(document_query)
+        for document in documents:
+            document_chunk_dao.delete(document.id)
+        # delete documents
+        knowledge_document_dao.delete(document_query)
+        # delete space
+        return knowledge_space_dao.delete_knowledge_space(space)
+
+    def delete_document(self, space_name: str, doc_name: str):
+        document_query = KnowledgeDocumentEntity(doc_name=doc_name, space=space_name)
+        documents = knowledge_document_dao.get_documents(document_query)
+        if len(documents) != 1:
+            raise Exception(f"there are no or more than one document called {doc_name}")
+        vector_ids = documents[0].vector_ids
+        if vector_ids is not None:
+            vector_config = {}
+            vector_config["vector_store_name"] = space_name
+            vector_config["vector_store_type"] = CFG.VECTOR_STORE_TYPE
+            vector_config["chroma_persist_path"] = KNOWLEDGE_UPLOAD_ROOT_PATH
+            vector_client = VectorStoreConnector(
+                vector_store_type=CFG.VECTOR_STORE_TYPE, ctx=vector_config
+            )
+            # delete vector by ids
+            vector_client.delete_by_ids(vector_ids)
+        # delete chunks
+        document_chunk_dao.delete(documents[0].id)
+        # delete document
+        return knowledge_document_dao.delete(document_query)
 
     """get document chunks"""
 

pilot/server/knowledge/space_db.py

@@ -39,7 +39,7 @@ class KnowledgeSpaceDao:
         session = self.Session()
         knowledge_space = KnowledgeSpaceEntity(
             name=space.name,
-            vector_type=space.vector_type,
+            vector_type=CFG.VECTOR_STORE_TYPE,
             desc=space.desc,
             owner=space.owner,
             gmt_created=datetime.now(),
@@ -47,7 +47,6 @@ class KnowledgeSpaceDao:
         )
         session.add(knowledge_space)
         session.commit()
-
         session.close()
 
     def get_knowledge_space(self, query: KnowledgeSpaceEntity):
@@ -86,6 +85,7 @@ class KnowledgeSpaceDao:
             KnowledgeSpaceEntity.gmt_created.desc()
         )
         result = knowledge_spaces.all()
+        session.close()
         return result
 
     def update_knowledge_space(self, space_id: int, space: KnowledgeSpaceEntity):
@@ -97,9 +97,9 @@ class KnowledgeSpaceDao:
         self.conn.commit()
         cursor.close()
 
-    def delete_knowledge_space(self, space_id: int):
-        cursor = self.conn.cursor()
-        query = "DELETE FROM knowledge_space WHERE id = %s"
-        cursor.execute(query, (space_id,))
-        self.conn.commit()
-        cursor.close()
+    def delete_knowledge_space(self, space: KnowledgeSpaceEntity):
+        session = self.Session()
+        if space:
+            session.delete(space)
+            session.commit()
+        session.close()

pilot/server/static/404.html

@@ -1 +1 @@
-<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><title>404: This page could not be found</title><meta name="next-head-count" content="3"/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/_next/static/chunks/webpack-8be0750561cfcccd.js" defer=""></script><script src="/_next/static/chunks/framework-43665103d101a22d.js" defer=""></script><script src="/_next/static/chunks/main-c6e90425c3eeb90a.js" defer=""></script><script src="/_next/static/chunks/pages/_app-1f2755172264764d.js" defer=""></script><script src="/_next/static/chunks/pages/_error-f5357f382422dd96.js" defer=""></script><script src="/_next/static/Iwzr9XFjkUv9028K-ieh2/_buildManifest.js" defer=""></script><script src="/_next/static/Iwzr9XFjkUv9028K-ieh2/_ssgManifest.js" defer=""></script></head><body><div id="__next"><div style="font-family:system-ui,&quot;Segoe UI&quot;,Roboto,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot;;height:100vh;text-align:center;display:flex;flex-direction:column;align-items:center;justify-content:center"><div style="line-height:48px"><style>body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}</style><h1 class="next-error-h1" style="display:inline-block;margin:0 20px 0 0;padding-right:23px;font-size:24px;font-weight:500;vertical-align:top">404</h1><div style="display:inline-block"><h2 style="font-size:14px;font-weight:400;line-height:28px">This page could not be found<!-- -->.</h2></div></div></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"statusCode":404}},"page":"/_error","query":{},"buildId":"Iwzr9XFjkUv9028K-ieh2","nextExport":true,"isFallback":false,"gip":true,"scriptLoader":[]}</script></body></html>
+<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><title>404: This page could not be found</title><meta name="next-head-count" content="3"/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/_next/static/chunks/webpack-8be0750561cfcccd.js" defer=""></script><script src="/_next/static/chunks/framework-43665103d101a22d.js" defer=""></script><script src="/_next/static/chunks/main-c6e90425c3eeb90a.js" defer=""></script><script src="/_next/static/chunks/pages/_app-1f2755172264764d.js" defer=""></script><script src="/_next/static/chunks/pages/_error-f5357f382422dd96.js" defer=""></script><script src="/_next/static/8wN6ARX3c5zahVupdkg9Y/_buildManifest.js" defer=""></script><script src="/_next/static/8wN6ARX3c5zahVupdkg9Y/_ssgManifest.js" defer=""></script></head><body><div id="__next"><div style="font-family:system-ui,&quot;Segoe UI&quot;,Roboto,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot;;height:100vh;text-align:center;display:flex;flex-direction:column;align-items:center;justify-content:center"><div style="line-height:48px"><style>body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}</style><h1 class="next-error-h1" style="display:inline-block;margin:0 20px 0 0;padding-right:23px;font-size:24px;font-weight:500;vertical-align:top">404</h1><div style="display:inline-block"><h2 style="font-size:14px;font-weight:400;line-height:28px">This page could not be found<!-- -->.</h2></div></div></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"statusCode":404}},"page":"/_error","query":{},"buildId":"8wN6ARX3c5zahVupdkg9Y","nextExport":true,"isFallback":false,"gip":true,"scriptLoader":[]}</script></body></html>

pilot/server/static/404/index.html

@@ -1 +1 @@
-<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><title>404: This page could not be found</title><meta name="next-head-count" content="3"/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/_next/static/chunks/webpack-8be0750561cfcccd.js" defer=""></script><script src="/_next/static/chunks/framework-43665103d101a22d.js" defer=""></script><script src="/_next/static/chunks/main-c6e90425c3eeb90a.js" defer=""></script><script src="/_next/static/chunks/pages/_app-1f2755172264764d.js" defer=""></script><script src="/_next/static/chunks/pages/_error-f5357f382422dd96.js" defer=""></script><script src="/_next/static/Iwzr9XFjkUv9028K-ieh2/_buildManifest.js" defer=""></script><script src="/_next/static/Iwzr9XFjkUv9028K-ieh2/_ssgManifest.js" defer=""></script></head><body><div id="__next"><div style="font-family:system-ui,&quot;Segoe UI&quot;,Roboto,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot;;height:100vh;text-align:center;display:flex;flex-direction:column;align-items:center;justify-content:center"><div style="line-height:48px"><style>body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}</style><h1 class="next-error-h1" style="display:inline-block;margin:0 20px 0 0;padding-right:23px;font-size:24px;font-weight:500;vertical-align:top">404</h1><div style="display:inline-block"><h2 style="font-size:14px;font-weight:400;line-height:28px">This page could not be found<!-- -->.</h2></div></div></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"statusCode":404}},"page":"/_error","query":{},"buildId":"Iwzr9XFjkUv9028K-ieh2","nextExport":true,"isFallback":false,"gip":true,"scriptLoader":[]}</script></body></html>
+<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><title>404: This page could not be found</title><meta name="next-head-count" content="3"/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/_next/static/chunks/webpack-8be0750561cfcccd.js" defer=""></script><script src="/_next/static/chunks/framework-43665103d101a22d.js" defer=""></script><script src="/_next/static/chunks/main-c6e90425c3eeb90a.js" defer=""></script><script src="/_next/static/chunks/pages/_app-1f2755172264764d.js" defer=""></script><script src="/_next/static/chunks/pages/_error-f5357f382422dd96.js" defer=""></script><script src="/_next/static/8wN6ARX3c5zahVupdkg9Y/_buildManifest.js" defer=""></script><script src="/_next/static/8wN6ARX3c5zahVupdkg9Y/_ssgManifest.js" defer=""></script></head><body><div id="__next"><div style="font-family:system-ui,&quot;Segoe UI&quot;,Roboto,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot;;height:100vh;text-align:center;display:flex;flex-direction:column;align-items:center;justify-content:center"><div style="line-height:48px"><style>body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}</style><h1 class="next-error-h1" style="display:inline-block;margin:0 20px 0 0;padding-right:23px;font-size:24px;font-weight:500;vertical-align:top">404</h1><div style="display:inline-block"><h2 style="font-size:14px;font-weight:400;line-height:28px">This page could not be found<!-- -->.</h2></div></div></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"statusCode":404}},"page":"/_error","query":{},"buildId":"8wN6ARX3c5zahVupdkg9Y","nextExport":true,"isFallback":false,"gip":true,"scriptLoader":[]}</script></body></html>

pilot/server/static/_next/static/chunks/app/datastores/documents/chunklist/page-5817243840a3f0d3.js

@@ -1 +1 @@
-(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[538],{30976:function(e,t,n){Promise.resolve().then(n.bind(n,26257))},26257:function(e,t,n){"use strict";n.r(t);var r=n(9268),a=n(56008),i=n(86006),c=n(78635),s=n(80937),o=n(44334),l=n(311),d=n(22046),h=n(83192),u=n(23910),g=n(1031),f=n(78915);t.default=()=>{let e=(0,a.useRouter)(),{mode:t}=(0,c.tv)(),n=(0,a.useSearchParams)().get("spacename"),j=(0,a.useSearchParams)().get("documentid"),[m,p]=(0,i.useState)(0),[x,P]=(0,i.useState)(0),[S,_]=(0,i.useState)([]);return(0,i.useEffect)(()=>{(async function(){let e=await (0,f.PR)("/knowledge/".concat(n,"/chunk/list"),{document_id:j,page:1,page_size:20});e.success&&(_(e.data.data),p(e.data.total),P(e.data.page))})()},[]),(0,r.jsxs)("div",{className:"p-4",children:[(0,r.jsx)(s.Z,{direction:"row",justifyContent:"flex-start",alignItems:"center",sx:{marginBottom:"20px"},children:(0,r.jsxs)(o.Z,{"aria-label":"breadcrumbs",children:[(0,r.jsx)(l.Z,{onClick:()=>{e.push("/datastores")},underline:"hover",color:"neutral",fontSize:"inherit",children:"Knowledge Space"},"Knowledge Space"),(0,r.jsx)(l.Z,{onClick:()=>{e.push("/datastores/documents?name=".concat(n))},underline:"hover",color:"neutral",fontSize:"inherit",children:"Documents"},"Knowledge Space"),(0,r.jsx)(d.ZP,{fontSize:"inherit",children:"Chunks"})]})}),(0,r.jsx)("div",{className:"p-4",children:S.length?(0,r.jsxs)(r.Fragment,{children:[(0,r.jsxs)(h.Z,{color:"primary",variant:"plain",size:"lg",sx:{"& tbody tr: hover":{backgroundColor:"light"===t?"rgb(246, 246, 246)":"rgb(33, 33, 40)"},"& tbody tr: hover a":{textDecoration:"underline"}},children:[(0,r.jsx)("thead",{children:(0,r.jsxs)("tr",{children:[(0,r.jsx)("th",{children:"Name"}),(0,r.jsx)("th",{children:"Content"}),(0,r.jsx)("th",{children:"Meta Data"})]})}),(0,r.jsx)("tbody",{children:S.map(e=>(0,r.jsxs)("tr",{children:[(0,r.jsx)("td",{children:e.doc_name}),(0,r.jsx)("td",{children:(0,r.jsx)(u.Z,{content:e.content,trigger:"hover",children:e.content.length>10?"".concat(e.content.slice(0,10),"..."):e.content})}),(0,r.jsx)("td",{children:(0,r.jsx)(u.Z,{content:JSON.stringify(e.meta_info||"{}",null,2),trigger:"hover",children:e.meta_info.length>10?"".concat(e.meta_info.slice(0,10),"..."):e.meta_info})})]},e.id))})]}),(0,r.jsx)(s.Z,{direction:"row",justifyContent:"flex-end",sx:{marginTop:"20px"},children:(0,r.jsx)(g.Z,{defaultPageSize:20,showSizeChanger:!1,current:x,total:m,onChange:async e=>{let t=await (0,f.PR)("/knowledge/".concat(n,"/chunk/list"),{document_id:j,page:e,page_size:20});t.success&&(_(t.data.data),p(t.data.total),P(t.data.page))},hideOnSinglePage:!0})})]}):(0,r.jsx)(r.Fragment,{})})]})}},78915:function(e,t,n){"use strict";n.d(t,{Tk:function(){return d},Kw:function(){return h},PR:function(){return u},Ej:function(){return g}});var r=n(21628),a=n(24214),i=n(52040);let c=a.Z.create({baseURL:i.env.API_BASE_URL});c.defaults.timeout=1e4,c.interceptors.response.use(e=>e.data,e=>Promise.reject(e));var s=n(84835);let o={"content-type":"application/json"},l=e=>{if(!(0,s.isPlainObject)(e))return JSON.stringify(e);let t={...e};for(let e in t){let n=t[e];"string"==typeof n&&(t[e]=n.trim())}return JSON.stringify(t)},d=(e,t)=>{if(t){let n=Object.keys(t).filter(e=>void 0!==t[e]&&""!==t[e]).map(e=>"".concat(e,"=").concat(t[e])).join("&");n&&(e+="?".concat(n))}return c.get("/api"+e,{headers:o}).then(e=>e).catch(e=>{r.ZP.error(e),Promise.reject(e)})},h=(e,t)=>{let n=l(t);return c.post("/api"+e,{body:n,headers:o}).then(e=>e).catch(e=>{r.ZP.error(e),Promise.reject(e)})},u=(e,t)=>(l(t),c.post(e,t,{headers:o}).then(e=>e).catch(e=>{r.ZP.error(e),Promise.reject(e)})),g=(e,t)=>c.post(e,t).then(e=>e).catch(e=>{r.ZP.error(e),Promise.reject(e)})}},function(e){e.O(0,[180,838,759,192,409,767,957,253,769,744],function(){return e(e.s=30976)}),_N_E=e.O()}]);
+(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[538],{40687:function(e,t,n){Promise.resolve().then(n.bind(n,26257))},26257:function(e,t,n){"use strict";n.r(t);var r=n(9268),a=n(56008),i=n(86006),c=n(78635),s=n(80937),o=n(44334),l=n(311),d=n(22046),h=n(83192),u=n(23910),g=n(1031),f=n(78915);t.default=()=>{let e=(0,a.useRouter)(),{mode:t}=(0,c.tv)(),n=(0,a.useSearchParams)().get("spacename"),j=(0,a.useSearchParams)().get("documentid"),[m,p]=(0,i.useState)(0),[x,P]=(0,i.useState)(0),[S,_]=(0,i.useState)([]);return(0,i.useEffect)(()=>{(async function(){let e=await (0,f.PR)("/knowledge/".concat(n,"/chunk/list"),{document_id:j,page:1,page_size:20});e.success&&(_(e.data.data),p(e.data.total),P(e.data.page))})()},[]),(0,r.jsxs)("div",{className:"p-4",children:[(0,r.jsx)(s.Z,{direction:"row",justifyContent:"flex-start",alignItems:"center",sx:{marginBottom:"20px"},children:(0,r.jsxs)(o.Z,{"aria-label":"breadcrumbs",children:[(0,r.jsx)(l.Z,{onClick:()=>{e.push("/datastores")},underline:"hover",color:"neutral",fontSize:"inherit",children:"Knowledge Space"},"Knowledge Space"),(0,r.jsx)(l.Z,{onClick:()=>{e.push("/datastores/documents?name=".concat(n))},underline:"hover",color:"neutral",fontSize:"inherit",children:"Documents"},"Knowledge Space"),(0,r.jsx)(d.ZP,{fontSize:"inherit",children:"Chunks"})]})}),(0,r.jsx)("div",{className:"p-4",children:S.length?(0,r.jsxs)(r.Fragment,{children:[(0,r.jsxs)(h.Z,{color:"primary",variant:"plain",size:"lg",sx:{"& tbody tr: hover":{backgroundColor:"light"===t?"rgb(246, 246, 246)":"rgb(33, 33, 40)"},"& tbody tr: hover a":{textDecoration:"underline"}},children:[(0,r.jsx)("thead",{children:(0,r.jsxs)("tr",{children:[(0,r.jsx)("th",{children:"Name"}),(0,r.jsx)("th",{children:"Content"}),(0,r.jsx)("th",{children:"Meta Data"})]})}),(0,r.jsx)("tbody",{children:S.map(e=>(0,r.jsxs)("tr",{children:[(0,r.jsx)("td",{children:e.doc_name}),(0,r.jsx)("td",{children:(0,r.jsx)(u.Z,{content:e.content,trigger:"hover",children:e.content.length>10?"".concat(e.content.slice(0,10),"..."):e.content})}),(0,r.jsx)("td",{children:(0,r.jsx)(u.Z,{content:JSON.stringify(e.meta_info||"{}",null,2),trigger:"hover",children:e.meta_info.length>10?"".concat(e.meta_info.slice(0,10),"..."):e.meta_info})})]},e.id))})]}),(0,r.jsx)(s.Z,{direction:"row",justifyContent:"flex-end",sx:{marginTop:"20px"},children:(0,r.jsx)(g.Z,{defaultPageSize:20,showSizeChanger:!1,current:x,total:m,onChange:async e=>{let t=await (0,f.PR)("/knowledge/".concat(n,"/chunk/list"),{document_id:j,page:e,page_size:20});t.success&&(_(t.data.data),p(t.data.total),P(t.data.page))},hideOnSinglePage:!0})})]}):(0,r.jsx)(r.Fragment,{})})]})}},78915:function(e,t,n){"use strict";n.d(t,{Tk:function(){return d},Kw:function(){return h},PR:function(){return u},Ej:function(){return g}});var r=n(21628),a=n(24214),i=n(52040);let c=a.Z.create({baseURL:i.env.API_BASE_URL});c.defaults.timeout=1e4,c.interceptors.response.use(e=>e.data,e=>Promise.reject(e));var s=n(84835);let o={"content-type":"application/json"},l=e=>{if(!(0,s.isPlainObject)(e))return JSON.stringify(e);let t={...e};for(let e in t){let n=t[e];"string"==typeof n&&(t[e]=n.trim())}return JSON.stringify(t)},d=(e,t)=>{if(t){let n=Object.keys(t).filter(e=>void 0!==t[e]&&""!==t[e]).map(e=>"".concat(e,"=").concat(t[e])).join("&");n&&(e+="?".concat(n))}return c.get("/api"+e,{headers:o}).then(e=>e).catch(e=>{r.ZP.error(e),Promise.reject(e)})},h=(e,t)=>{let n=l(t);return c.post("/api"+e,{body:n,headers:o}).then(e=>e).catch(e=>{r.ZP.error(e),Promise.reject(e)})},u=(e,t)=>(l(t),c.post(e,t,{headers:o}).then(e=>e).catch(e=>{r.ZP.error(e),Promise.reject(e)})),g=(e,t)=>c.post(e,t).then(e=>e).catch(e=>{r.ZP.error(e),Promise.reject(e)})}},function(e){e.O(0,[180,838,759,192,409,767,957,253,769,744],function(){return e(e.s=40687)}),_N_E=e.O()}]);

pilot/server/static/_next/static/chunks/main-app-75c197595e152149.js

@@ -1 +1 @@
-(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[744],{59617:function(e,n,t){Promise.resolve().then(t.t.bind(t,68802,23)),Promise.resolve().then(t.t.bind(t,13211,23)),Promise.resolve().then(t.t.bind(t,5767,23)),Promise.resolve().then(t.t.bind(t,14299,23)),Promise.resolve().then(t.t.bind(t,37396,23))}},function(e){var n=function(n){return e(e.s=n)};e.O(0,[253,769],function(){return n(29070),n(59617)}),_N_E=e.O()}]);
+(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[744],{72656:function(e,n,t){Promise.resolve().then(t.t.bind(t,68802,23)),Promise.resolve().then(t.t.bind(t,13211,23)),Promise.resolve().then(t.t.bind(t,5767,23)),Promise.resolve().then(t.t.bind(t,14299,23)),Promise.resolve().then(t.t.bind(t,37396,23))}},function(e){var n=function(n){return e(e.s=n)};e.O(0,[253,769],function(){return n(29070),n(72656)}),_N_E=e.O()}]);

pilot/server/static/chat/index.txt

@@ -1,9 +1,9 @@
 1:HL["/_next/static/css/f7886471966370ed.css",{"as":"style"}]
-0:["Iwzr9XFjkUv9028K-ieh2",[[["",{"children":["chat",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],"$L2",[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/f7886471966370ed.css","precedence":"next"}]],["$L3",null]]]]]
-4:I{"id":"50902","chunks":["180:static/chunks/0e02fca3-615d0d51fa074d92.js","838:static/chunks/838-25c9b71d449c8910.js","60:static/chunks/60-8ef99caef9fdf742.js","759:static/chunks/759-22c7defc3aa943a7.js","409:static/chunks/409-4b199bf070fd70fc.js","316:static/chunks/316-370750739484dff7.js","946:static/chunks/946-3a66ddfd20b8ad3d.js","394:static/chunks/394-0ffa189aa535d3eb.js","751:static/chunks/751-30fee9a32c6e64a2.js","256:static/chunks/256-f82130fbef33c4d6.js","185:static/chunks/app/layout-055e926a104869ef.js"],"name":"","async":false}
+0:["8wN6ARX3c5zahVupdkg9Y",[[["",{"children":["chat",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],"$L2",[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/f7886471966370ed.css","precedence":"next"}]],["$L3",null]]]]]
+4:I{"id":"50902","chunks":["180:static/chunks/0e02fca3-615d0d51fa074d92.js","838:static/chunks/838-25c9b71d449c8910.js","60:static/chunks/60-8ef99caef9fdf742.js","759:static/chunks/759-22c7defc3aa943a7.js","409:static/chunks/409-4b199bf070fd70fc.js","316:static/chunks/316-370750739484dff7.js","946:static/chunks/946-3a66ddfd20b8ad3d.js","394:static/chunks/394-0ffa189aa535d3eb.js","751:static/chunks/751-30fee9a32c6e64a2.js","256:static/chunks/256-f82130fbef33c4d6.js","185:static/chunks/app/layout-4991b4ac844c585c.js"],"name":"","async":false}
 5:I{"id":"13211","chunks":["272:static/chunks/webpack-8be0750561cfcccd.js","253:static/chunks/bce60fc1-18c9f145b45d8f36.js","769:static/chunks/769-76f7aafd375fdd6b.js"],"name":"","async":false}
 6:I{"id":"5767","chunks":["272:static/chunks/webpack-8be0750561cfcccd.js","253:static/chunks/bce60fc1-18c9f145b45d8f36.js","769:static/chunks/769-76f7aafd375fdd6b.js"],"name":"","async":false}
 7:I{"id":"37396","chunks":["272:static/chunks/webpack-8be0750561cfcccd.js","253:static/chunks/bce60fc1-18c9f145b45d8f36.js","769:static/chunks/769-76f7aafd375fdd6b.js"],"name":"","async":false}
-8:I{"id":"65641","chunks":["180:static/chunks/0e02fca3-615d0d51fa074d92.js","757:static/chunks/f60284a2-6891068c9ea7ce77.js","282:static/chunks/7e4358a0-8f10c290d655cdf1.js","838:static/chunks/838-25c9b71d449c8910.js","60:static/chunks/60-8ef99caef9fdf742.js","759:static/chunks/759-22c7defc3aa943a7.js","192:static/chunks/192-e9f419fb9f5bc502.js","86:static/chunks/86-6193a530bd8e3ef4.js","316:static/chunks/316-370750739484dff7.js","790:static/chunks/790-97e6b769f5c791cb.js","767:static/chunks/767-b93280f4b5b5e975.js","259:static/chunks/259-2c3490a9eca2f411.js","751:static/chunks/751-30fee9a32c6e64a2.js","320:static/chunks/320-63dc542e9a7120d1.js","929:static/chunks/app/chat/page-f0d8fd24469f64f9.js"],"name":"","async":false}
+8:I{"id":"65641","chunks":["180:static/chunks/0e02fca3-615d0d51fa074d92.js","757:static/chunks/f60284a2-6891068c9ea7ce77.js","282:static/chunks/7e4358a0-8f10c290d655cdf1.js","838:static/chunks/838-25c9b71d449c8910.js","60:static/chunks/60-8ef99caef9fdf742.js","759:static/chunks/759-22c7defc3aa943a7.js","192:static/chunks/192-e9f419fb9f5bc502.js","86:static/chunks/86-6193a530bd8e3ef4.js","316:static/chunks/316-370750739484dff7.js","790:static/chunks/790-97e6b769f5c791cb.js","767:static/chunks/767-b93280f4b5b5e975.js","259:static/chunks/259-2c3490a9eca2f411.js","751:static/chunks/751-30fee9a32c6e64a2.js","320:static/chunks/320-63dc542e9a7120d1.js","929:static/chunks/app/chat/page-614438239adc40fc.js"],"name":"","async":false}
 2:[["$","$L4",null,{"children":["$","$L5",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","loading":"$undefined","loadingStyles":"$undefined","hasLoading":false,"template":["$","$L6",null,{}],"templateStyles":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined","childProp":{"current":["$","$L5",null,{"parallelRouterKey":"children","segmentPath":["children","chat","children"],"error":"$undefined","errorStyles":"$undefined","loading":"$undefined","loadingStyles":"$undefined","hasLoading":false,"template":["$","$L6",null,{}],"templateStyles":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined","childProp":{"current":[["$","$L7",null,{"propsForComponent":{"params":{}},"Component":"$8"}],null],"segment":"__PAGE__"},"styles":[]}],"segment":"chat"},"styles":[]}],"params":{}}],null]
 3:[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","link","2",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"any"}]]

pilot/server/static/datastores/documents/chunklist/index.txt

@@ -1,9 +1,9 @@
 1:HL["/_next/static/css/f7886471966370ed.css",{"as":"style"}]
-0:["Iwzr9XFjkUv9028K-ieh2",[[["",{"children":["datastores",{"children":["documents",{"children":["chunklist",{"children":["__PAGE__",{}]}]}]}]},"$undefined","$undefined",true],"$L2",[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/f7886471966370ed.css","precedence":"next"}]],["$L3",null]]]]]
-4:I{"id":"50902","chunks":["180:static/chunks/0e02fca3-615d0d51fa074d92.js","838:static/chunks/838-25c9b71d449c8910.js","60:static/chunks/60-8ef99caef9fdf742.js","759:static/chunks/759-22c7defc3aa943a7.js","409:static/chunks/409-4b199bf070fd70fc.js","316:static/chunks/316-370750739484dff7.js","946:static/chunks/946-3a66ddfd20b8ad3d.js","394:static/chunks/394-0ffa189aa535d3eb.js","751:static/chunks/751-30fee9a32c6e64a2.js","256:static/chunks/256-f82130fbef33c4d6.js","185:static/chunks/app/layout-055e926a104869ef.js"],"name":"","async":false}
+0:["8wN6ARX3c5zahVupdkg9Y",[[["",{"children":["datastores",{"children":["documents",{"children":["chunklist",{"children":["__PAGE__",{}]}]}]}]},"$undefined","$undefined",true],"$L2",[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/f7886471966370ed.css","precedence":"next"}]],["$L3",null]]]]]
+4:I{"id":"50902","chunks":["180:static/chunks/0e02fca3-615d0d51fa074d92.js","838:static/chunks/838-25c9b71d449c8910.js","60:static/chunks/60-8ef99caef9fdf742.js","759:static/chunks/759-22c7defc3aa943a7.js","409:static/chunks/409-4b199bf070fd70fc.js","316:static/chunks/316-370750739484dff7.js","946:static/chunks/946-3a66ddfd20b8ad3d.js","394:static/chunks/394-0ffa189aa535d3eb.js","751:static/chunks/751-30fee9a32c6e64a2.js","256:static/chunks/256-f82130fbef33c4d6.js","185:static/chunks/app/layout-4991b4ac844c585c.js"],"name":"","async":false}
 5:I{"id":"13211","chunks":["272:static/chunks/webpack-8be0750561cfcccd.js","253:static/chunks/bce60fc1-18c9f145b45d8f36.js","769:static/chunks/769-76f7aafd375fdd6b.js"],"name":"","async":false}
 6:I{"id":"5767","chunks":["272:static/chunks/webpack-8be0750561cfcccd.js","253:static/chunks/bce60fc1-18c9f145b45d8f36.js","769:static/chunks/769-76f7aafd375fdd6b.js"],"name":"","async":false}
 7:I{"id":"37396","chunks":["272:static/chunks/webpack-8be0750561cfcccd.js","253:static/chunks/bce60fc1-18c9f145b45d8f36.js","769:static/chunks/769-76f7aafd375fdd6b.js"],"name":"","async":false}
-8:I{"id":"26257","chunks":["180:static/chunks/0e02fca3-615d0d51fa074d92.js","838:static/chunks/838-25c9b71d449c8910.js","759:static/chunks/759-22c7defc3aa943a7.js","192:static/chunks/192-e9f419fb9f5bc502.js","409:static/chunks/409-4b199bf070fd70fc.js","767:static/chunks/767-b93280f4b5b5e975.js","957:static/chunks/957-c96ff4de6e80d713.js","538:static/chunks/app/datastores/documents/chunklist/page-583473409d3bf959.js"],"name":"","async":false}
+8:I{"id":"26257","chunks":["180:static/chunks/0e02fca3-615d0d51fa074d92.js","838:static/chunks/838-25c9b71d449c8910.js","759:static/chunks/759-22c7defc3aa943a7.js","192:static/chunks/192-e9f419fb9f5bc502.js","409:static/chunks/409-4b199bf070fd70fc.js","767:static/chunks/767-b93280f4b5b5e975.js","957:static/chunks/957-c96ff4de6e80d713.js","538:static/chunks/app/datastores/documents/chunklist/page-5817243840a3f0d3.js"],"name":"","async":false}
 2:[["$","$L4",null,{"children":["$","$L5",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","loading":"$undefined","loadingStyles":"$undefined","hasLoading":false,"template":["$","$L6",null,{}],"templateStyles":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined","childProp":{"current":["$","$L5",null,{"parallelRouterKey":"children","segmentPath":["children","datastores","children"],"error":"$undefined","errorStyles":"$undefined","loading":"$undefined","loadingStyles":"$undefined","hasLoading":false,"template":["$","$L6",null,{}],"templateStyles":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined","childProp":{"current":["$","$L5",null,{"parallelRouterKey":"children","segmentPath":["children","datastores","children","documents","children"],"error":"$undefined","errorStyles":"$undefined","loading":"$undefined","loadingStyles":"$undefined","hasLoading":false,"template":["$","$L6",null,{}],"templateStyles":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined","childProp":{"current":["$","$L5",null,{"parallelRouterKey":"children","segmentPath":["children","datastores","children","documents","children","chunklist","children"],"error":"$undefined","errorStyles":"$undefined","loading":"$undefined","loadingStyles":"$undefined","hasLoading":false,"template":["$","$L6",null,{}],"templateStyles":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined","childProp":{"current":[["$","$L7",null,{"propsForComponent":{"params":{}},"Component":"$8"}],null],"segment":"__PAGE__"},"styles":[]}],"segment":"chunklist"},"styles":[]}],"segment":"documents"},"styles":[]}],"segment":"datastores"},"styles":[]}],"params":{}}],null]
 3:[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","link","2",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"any"}]]

pilot/server/static/datastores/documents/index.txt

@@ -1,9 +1,9 @@
 1:HL["/_next/static/css/f7886471966370ed.css",{"as":"style"}]
-0:["Iwzr9XFjkUv9028K-ieh2",[[["",{"children":["datastores",{"children":["documents",{"children":["__PAGE__",{}]}]}]},"$undefined","$undefined",true],"$L2",[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/f7886471966370ed.css","precedence":"next"}]],["$L3",null]]]]]
-4:I{"id":"50902","chunks":["180:static/chunks/0e02fca3-615d0d51fa074d92.js","838:static/chunks/838-25c9b71d449c8910.js","60:static/chunks/60-8ef99caef9fdf742.js","759:static/chunks/759-22c7defc3aa943a7.js","409:static/chunks/409-4b199bf070fd70fc.js","316:static/chunks/316-370750739484dff7.js","946:static/chunks/946-3a66ddfd20b8ad3d.js","394:static/chunks/394-0ffa189aa535d3eb.js","751:static/chunks/751-30fee9a32c6e64a2.js","256:static/chunks/256-f82130fbef33c4d6.js","185:static/chunks/app/layout-055e926a104869ef.js"],"name":"","async":false}
+0:["8wN6ARX3c5zahVupdkg9Y",[[["",{"children":["datastores",{"children":["documents",{"children":["__PAGE__",{}]}]}]},"$undefined","$undefined",true],"$L2",[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/f7886471966370ed.css","precedence":"next"}]],["$L3",null]]]]]
+4:I{"id":"50902","chunks":["180:static/chunks/0e02fca3-615d0d51fa074d92.js","838:static/chunks/838-25c9b71d449c8910.js","60:static/chunks/60-8ef99caef9fdf742.js","759:static/chunks/759-22c7defc3aa943a7.js","409:static/chunks/409-4b199bf070fd70fc.js","316:static/chunks/316-370750739484dff7.js","946:static/chunks/946-3a66ddfd20b8ad3d.js","394:static/chunks/394-0ffa189aa535d3eb.js","751:static/chunks/751-30fee9a32c6e64a2.js","256:static/chunks/256-f82130fbef33c4d6.js","185:static/chunks/app/layout-4991b4ac844c585c.js"],"name":"","async":false}
 5:I{"id":"13211","chunks":["272:static/chunks/webpack-8be0750561cfcccd.js","253:static/chunks/bce60fc1-18c9f145b45d8f36.js","769:static/chunks/769-76f7aafd375fdd6b.js"],"name":"","async":false}
 6:I{"id":"5767","chunks":["272:static/chunks/webpack-8be0750561cfcccd.js","253:static/chunks/bce60fc1-18c9f145b45d8f36.js","769:static/chunks/769-76f7aafd375fdd6b.js"],"name":"","async":false}
 7:I{"id":"37396","chunks":["272:static/chunks/webpack-8be0750561cfcccd.js","253:static/chunks/bce60fc1-18c9f145b45d8f36.js","769:static/chunks/769-76f7aafd375fdd6b.js"],"name":"","async":false}
-8:I{"id":"16692","chunks":["180:static/chunks/0e02fca3-615d0d51fa074d92.js","550:static/chunks/925f3d25-1af7259455ef26bd.js","838:static/chunks/838-25c9b71d449c8910.js","60:static/chunks/60-8ef99caef9fdf742.js","759:static/chunks/759-22c7defc3aa943a7.js","192:static/chunks/192-e9f419fb9f5bc502.js","86:static/chunks/86-6193a530bd8e3ef4.js","409:static/chunks/409-4b199bf070fd70fc.js","790:static/chunks/790-97e6b769f5c791cb.js","946:static/chunks/946-3a66ddfd20b8ad3d.js","767:static/chunks/767-b93280f4b5b5e975.js","957:static/chunks/957-c96ff4de6e80d713.js","775:static/chunks/775-224c8c8f5ee3fd65.js","470:static/chunks/app/datastores/documents/page-d94518835f877352.js"],"name":"","async":false}
+8:I{"id":"16692","chunks":["180:static/chunks/0e02fca3-615d0d51fa074d92.js","550:static/chunks/925f3d25-1af7259455ef26bd.js","838:static/chunks/838-25c9b71d449c8910.js","60:static/chunks/60-8ef99caef9fdf742.js","759:static/chunks/759-22c7defc3aa943a7.js","192:static/chunks/192-e9f419fb9f5bc502.js","86:static/chunks/86-6193a530bd8e3ef4.js","409:static/chunks/409-4b199bf070fd70fc.js","790:static/chunks/790-97e6b769f5c791cb.js","946:static/chunks/946-3a66ddfd20b8ad3d.js","767:static/chunks/767-b93280f4b5b5e975.js","957:static/chunks/957-c96ff4de6e80d713.js","872:static/chunks/872-4a145d8028102d89.js","470:static/chunks/app/datastores/documents/page-da4704cece0cf28a.js"],"name":"","async":false}
 2:[["$","$L4",null,{"children":["$","$L5",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","loading":"$undefined","loadingStyles":"$undefined","hasLoading":false,"template":["$","$L6",null,{}],"templateStyles":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined","childProp":{"current":["$","$L5",null,{"parallelRouterKey":"children","segmentPath":["children","datastores","children"],"error":"$undefined","errorStyles":"$undefined","loading":"$undefined","loadingStyles":"$undefined","hasLoading":false,"template":["$","$L6",null,{}],"templateStyles":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined","childProp":{"current":["$","$L5",null,{"parallelRouterKey":"children","segmentPath":["children","datastores","children","documents","children"],"error":"$undefined","errorStyles":"$undefined","loading":"$undefined","loadingStyles":"$undefined","hasLoading":false,"template":["$","$L6",null,{}],"templateStyles":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined","childProp":{"current":[["$","$L7",null,{"propsForComponent":{"params":{}},"Component":"$8"}],null],"segment":"__PAGE__"},"styles":[]}],"segment":"documents"},"styles":[]}],"segment":"datastores"},"styles":[]}],"params":{}}],null]
 3:[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","link","2",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"any"}]]

pilot/server/static/datastores/index.txt

@@ -1,9 +1,9 @@
 1:HL["/_next/static/css/f7886471966370ed.css",{"as":"style"}]
-0:["Iwzr9XFjkUv9028K-ieh2",[[["",{"children":["datastores",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],"$L2",[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/f7886471966370ed.css","precedence":"next"}]],["$L3",null]]]]]
-4:I{"id":"50902","chunks":["180:static/chunks/0e02fca3-615d0d51fa074d92.js","838:static/chunks/838-25c9b71d449c8910.js","60:static/chunks/60-8ef99caef9fdf742.js","759:static/chunks/759-22c7defc3aa943a7.js","409:static/chunks/409-4b199bf070fd70fc.js","316:static/chunks/316-370750739484dff7.js","946:static/chunks/946-3a66ddfd20b8ad3d.js","394:static/chunks/394-0ffa189aa535d3eb.js","751:static/chunks/751-30fee9a32c6e64a2.js","256:static/chunks/256-f82130fbef33c4d6.js","185:static/chunks/app/layout-055e926a104869ef.js"],"name":"","async":false}
+0:["8wN6ARX3c5zahVupdkg9Y",[[["",{"children":["datastores",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],"$L2",[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/f7886471966370ed.css","precedence":"next"}]],["$L3",null]]]]]
+4:I{"id":"50902","chunks":["180:static/chunks/0e02fca3-615d0d51fa074d92.js","838:static/chunks/838-25c9b71d449c8910.js","60:static/chunks/60-8ef99caef9fdf742.js","759:static/chunks/759-22c7defc3aa943a7.js","409:static/chunks/409-4b199bf070fd70fc.js","316:static/chunks/316-370750739484dff7.js","946:static/chunks/946-3a66ddfd20b8ad3d.js","394:static/chunks/394-0ffa189aa535d3eb.js","751:static/chunks/751-30fee9a32c6e64a2.js","256:static/chunks/256-f82130fbef33c4d6.js","185:static/chunks/app/layout-4991b4ac844c585c.js"],"name":"","async":false}
 5:I{"id":"13211","chunks":["272:static/chunks/webpack-8be0750561cfcccd.js","253:static/chunks/bce60fc1-18c9f145b45d8f36.js","769:static/chunks/769-76f7aafd375fdd6b.js"],"name":"","async":false}
 6:I{"id":"5767","chunks":["272:static/chunks/webpack-8be0750561cfcccd.js","253:static/chunks/bce60fc1-18c9f145b45d8f36.js","769:static/chunks/769-76f7aafd375fdd6b.js"],"name":"","async":false}
 7:I{"id":"37396","chunks":["272:static/chunks/webpack-8be0750561cfcccd.js","253:static/chunks/bce60fc1-18c9f145b45d8f36.js","769:static/chunks/769-76f7aafd375fdd6b.js"],"name":"","async":false}
-8:I{"id":"44323","chunks":["180:static/chunks/0e02fca3-615d0d51fa074d92.js","838:static/chunks/838-25c9b71d449c8910.js","60:static/chunks/60-8ef99caef9fdf742.js","759:static/chunks/759-22c7defc3aa943a7.js","192:static/chunks/192-e9f419fb9f5bc502.js","86:static/chunks/86-6193a530bd8e3ef4.js","790:static/chunks/790-97e6b769f5c791cb.js","946:static/chunks/946-3a66ddfd20b8ad3d.js","775:static/chunks/775-224c8c8f5ee3fd65.js","43:static/chunks/app/datastores/page-71ca37d05b729f1d.js"],"name":"","async":false}
+8:I{"id":"44323","chunks":["180:static/chunks/0e02fca3-615d0d51fa074d92.js","838:static/chunks/838-25c9b71d449c8910.js","60:static/chunks/60-8ef99caef9fdf742.js","759:static/chunks/759-22c7defc3aa943a7.js","192:static/chunks/192-e9f419fb9f5bc502.js","86:static/chunks/86-6193a530bd8e3ef4.js","790:static/chunks/790-97e6b769f5c791cb.js","946:static/chunks/946-3a66ddfd20b8ad3d.js","872:static/chunks/872-4a145d8028102d89.js","2:static/chunks/2-a60cf38d8ab305bb.js","43:static/chunks/app/datastores/page-9d798355f6d9e339.js"],"name":"","async":false}
 2:[["$","$L4",null,{"children":["$","$L5",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","loading":"$undefined","loadingStyles":"$undefined","hasLoading":false,"template":["$","$L6",null,{}],"templateStyles":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined","childProp":{"current":["$","$L5",null,{"parallelRouterKey":"children","segmentPath":["children","datastores","children"],"error":"$undefined","errorStyles":"$undefined","loading":"$undefined","loadingStyles":"$undefined","hasLoading":false,"template":["$","$L6",null,{}],"templateStyles":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined","childProp":{"current":[["$","$L7",null,{"propsForComponent":{"params":{}},"Component":"$8"}],null],"segment":"__PAGE__"},"styles":[]}],"segment":"datastores"},"styles":[]}],"params":{}}],null]
 3:[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","link","2",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"any"}]]

pilot/server/static/index.txt

@@ -1,9 +1,9 @@
 1:HL["/_next/static/css/f7886471966370ed.css",{"as":"style"}]
-0:["Iwzr9XFjkUv9028K-ieh2",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],"$L2",[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/f7886471966370ed.css","precedence":"next"}]],["$L3",null]]]]]
-4:I{"id":"50902","chunks":["180:static/chunks/0e02fca3-615d0d51fa074d92.js","838:static/chunks/838-25c9b71d449c8910.js","60:static/chunks/60-8ef99caef9fdf742.js","759:static/chunks/759-22c7defc3aa943a7.js","409:static/chunks/409-4b199bf070fd70fc.js","316:static/chunks/316-370750739484dff7.js","946:static/chunks/946-3a66ddfd20b8ad3d.js","394:static/chunks/394-0ffa189aa535d3eb.js","751:static/chunks/751-30fee9a32c6e64a2.js","256:static/chunks/256-f82130fbef33c4d6.js","185:static/chunks/app/layout-055e926a104869ef.js"],"name":"","async":false}
+0:["8wN6ARX3c5zahVupdkg9Y",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],"$L2",[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/f7886471966370ed.css","precedence":"next"}]],["$L3",null]]]]]
+4:I{"id":"50902","chunks":["180:static/chunks/0e02fca3-615d0d51fa074d92.js","838:static/chunks/838-25c9b71d449c8910.js","60:static/chunks/60-8ef99caef9fdf742.js","759:static/chunks/759-22c7defc3aa943a7.js","409:static/chunks/409-4b199bf070fd70fc.js","316:static/chunks/316-370750739484dff7.js","946:static/chunks/946-3a66ddfd20b8ad3d.js","394:static/chunks/394-0ffa189aa535d3eb.js","751:static/chunks/751-30fee9a32c6e64a2.js","256:static/chunks/256-f82130fbef33c4d6.js","185:static/chunks/app/layout-4991b4ac844c585c.js"],"name":"","async":false}
 5:I{"id":"13211","chunks":["272:static/chunks/webpack-8be0750561cfcccd.js","253:static/chunks/bce60fc1-18c9f145b45d8f36.js","769:static/chunks/769-76f7aafd375fdd6b.js"],"name":"","async":false}
 6:I{"id":"5767","chunks":["272:static/chunks/webpack-8be0750561cfcccd.js","253:static/chunks/bce60fc1-18c9f145b45d8f36.js","769:static/chunks/769-76f7aafd375fdd6b.js"],"name":"","async":false}
 7:I{"id":"37396","chunks":["272:static/chunks/webpack-8be0750561cfcccd.js","253:static/chunks/bce60fc1-18c9f145b45d8f36.js","769:static/chunks/769-76f7aafd375fdd6b.js"],"name":"","async":false}
-8:I{"id":"26925","chunks":["180:static/chunks/0e02fca3-615d0d51fa074d92.js","838:static/chunks/838-25c9b71d449c8910.js","60:static/chunks/60-8ef99caef9fdf742.js","86:static/chunks/86-6193a530bd8e3ef4.js","316:static/chunks/316-370750739484dff7.js","259:static/chunks/259-2c3490a9eca2f411.js","394:static/chunks/394-0ffa189aa535d3eb.js","931:static/chunks/app/page-ba37471c26b3ed90.js"],"name":"","async":false}
+8:I{"id":"93768","chunks":["180:static/chunks/0e02fca3-615d0d51fa074d92.js","838:static/chunks/838-25c9b71d449c8910.js","60:static/chunks/60-8ef99caef9fdf742.js","86:static/chunks/86-6193a530bd8e3ef4.js","316:static/chunks/316-370750739484dff7.js","259:static/chunks/259-2c3490a9eca2f411.js","394:static/chunks/394-0ffa189aa535d3eb.js","931:static/chunks/app/page-1a66758966b07f9b.js"],"name":"","async":false}
 2:[["$","$L4",null,{"children":["$","$L5",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","loading":"$undefined","loadingStyles":"$undefined","hasLoading":false,"template":["$","$L6",null,{}],"templateStyles":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined","childProp":{"current":[["$","$L7",null,{"propsForComponent":{"params":{}},"Component":"$8"}],null],"segment":"__PAGE__"},"styles":[]}],"params":{}}],null]
 3:[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","link","2",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"any"}]]

pilot/vector_store/chroma_store.py

@@ -1,8 +1,9 @@
 import os
 
+from chromadb.config import Settings
 from langchain.vectorstores import Chroma
 from pilot.logs import logger
-from pilot.vector_store.vector_store_base import VectorStoreBase
+from pilot.vector_store.base import VectorStoreBase
 
 
 class ChromaStore(VectorStoreBase):
@@ -10,12 +11,19 @@ class ChromaStore(VectorStoreBase):
 
     def __init__(self, ctx: {}) -> None:
         self.ctx = ctx
-        self.embeddings = ctx["embeddings"]
+        self.embeddings = ctx.get("embeddings", None)
         self.persist_dir = os.path.join(
             ctx["chroma_persist_path"], ctx["vector_store_name"] + ".vectordb"
         )
+        chroma_settings = Settings(
+            chroma_db_impl="duckdb+parquet",
+            persist_directory=self.persist_dir,
+            anonymized_telemetry=False,
+        )
         self.vector_store_client = Chroma(
-            persist_directory=self.persist_dir, embedding_function=self.embeddings
+            persist_directory=self.persist_dir,
+            embedding_function=self.embeddings,
+            client_settings=chroma_settings,
         )
 
     def similar_search(self, text, topk) -> None:
@@ -35,6 +43,21 @@ class ChromaStore(VectorStoreBase):
         self.vector_store_client.persist()
         return ids
 
+    def delete_vector_name(self, vector_name):
+        logger.info(f"chroma vector_name:{vector_name} begin delete...")
+        self.vector_store_client.delete_collection()
+        self._clean_persist_folder()
+        return True
+
     def delete_by_ids(self, ids):
+        logger.info(f"begin delete chroma ids...")
         collection = self.vector_store_client._collection
         collection.delete(ids=ids)
+
+    def _clean_persist_folder(self):
+        for root, dirs, files in os.walk(self.persist_dir, topdown=False):
+            for name in files:
+                os.remove(os.path.join(root, name))
+            for name in dirs:
+                os.rmdir(os.path.join(root, name))
+        os.rmdir(self.persist_dir)

pilot/vector_store/connector.py

@@ -33,5 +33,10 @@ class VectorStoreConnector:
         """is vector store name exist."""
         return self.client.vector_name_exists()
 
+    def delete_vector_name(self, vector_name):
+        """vector store delete"""
+        return self.client.delete_vector_name(vector_name)
+
     def delete_by_ids(self, ids):
-        self.client.delete_by_ids(ids=ids)
+        """vector store delete by ids."""
+        return self.client.delete_by_ids(ids=ids)

pilot/vector_store/milvus_store.py

@@ -3,7 +3,8 @@ from typing import Any, Iterable, List, Optional, Tuple
 from langchain.docstore.document import Document
 from pymilvus import Collection, DataType, connections, utility
 
-from pilot.vector_store.vector_store_base import VectorStoreBase
+from pilot.logs import logger
+from pilot.vector_store.base import VectorStoreBase
 
 
 class MilvusStore(VectorStoreBase):
@@ -319,5 +320,10 @@ class MilvusStore(VectorStoreBase):
         """is vector store name exist."""
         return utility.has_collection(self.collection_name)
 
+    def delete_vector_name(self, vector_name):
+        logger.info(f"milvus vector_name:{vector_name} begin delete...")
+        self.vector_store_client.drop()
+        return True
+
     def close(self):
         connections.disconnect()

pilot/vector_store/weaviate_store.py

@@ -8,7 +8,7 @@ from weaviate.exceptions import WeaviateBaseError
 from pilot.configs.config import Config
 from pilot.configs.model_config import KNOWLEDGE_UPLOAD_ROOT_PATH
 from pilot.logs import logger
-from pilot.vector_store.vector_store_base import VectorStoreBase
+from pilot.vector_store.base import VectorStoreBase
 
 CFG = Config()
 

tests/unit/embedding_engine/document_test.py

@@ -0,0 +1,21 @@
+from pilot import EmbeddingEngine, KnowledgeType
+
+embedding_model = "your_embedding_model"
+vector_store_type = "Chroma"
+chroma_persist_path = "your_persist_path"
+vector_store_config = {
+    "vector_store_name": "document_test",
+    "vector_store_type": vector_store_type,
+    "chroma_persist_path": chroma_persist_path,
+}
+
+# it can be .md,.pdf,.docx, .csv, .html
+document_path = "your_path/test.md"
+embedding_engine = EmbeddingEngine(
+    knowledge_source=document_path,
+    knowledge_type=KnowledgeType.DOCUMENT.value,
+    model_name=embedding_model,
+    vector_store_config=vector_store_config,
+)
+# embedding document content to vector store
+embedding_engine.knowledge_embedding()

tests/unit/embedding_engine/url_test.py

@@ -1,7 +1,7 @@
 from pilot import EmbeddingEngine, KnowledgeType
 
 url = "https://db-gpt.readthedocs.io/en/latest/getting_started/getting_started.html"
-embedding_model = "text2vec"
+embedding_model = "your_embedding_model"
 vector_store_type = "Chroma"
 chroma_persist_path = "your_persist_path"
 vector_store_config = {