"""Markdown Knowledge.""" from typing import Any, Dict, List, Optional, Union from dbgpt.core import Document from dbgpt.rag.knowledge.base import ( ChunkStrategy, DocumentType, Knowledge, KnowledgeType, ) class MarkdownKnowledge(Knowledge): """Markdown Knowledge.""" def __init__( self, file_path: Optional[str] = None, knowledge_type: KnowledgeType = KnowledgeType.DOCUMENT, encoding: Optional[str] = "utf-8", loader: Optional[Any] = None, metadata: Optional[Dict[str, Union[str, List[str]]]] = None, **kwargs: Any, ) -> None: """Create Markdown Knowledge with Knowledge arguments. Args: file_path(str, optional): file path knowledge_type(KnowledgeType, optional): knowledge type encoding(str, optional): csv encoding loader(Any, optional): loader """ super().__init__( path=file_path, knowledge_type=knowledge_type, data_loader=loader, metadata=metadata, **kwargs, ) self._encoding = encoding def _load(self) -> List[Document]: """Load markdown document from loader.""" if self._loader: documents = self._loader.load() else: if not self._path: raise ValueError("file path is required") with open(self._path, encoding=self._encoding, errors="ignore") as f: markdown_text = f.read() metadata = {"source": self._path} if self._metadata: metadata.update(self._metadata) # type: ignore documents = [Document(content=markdown_text, metadata=metadata)] return documents return [Document.langchain2doc(lc_document) for lc_document in documents] @classmethod def support_chunk_strategy(cls) -> List[ChunkStrategy]: """Return support chunk strategy.""" return [ ChunkStrategy.CHUNK_BY_SIZE, ChunkStrategy.CHUNK_BY_MARKDOWN_HEADER, ChunkStrategy.CHUNK_BY_SEPARATOR, ] @classmethod def default_chunk_strategy(cls) -> ChunkStrategy: """Return default chunk strategy.""" return ChunkStrategy.CHUNK_BY_MARKDOWN_HEADER @classmethod def type(cls) -> KnowledgeType: """Return knowledge type.""" return KnowledgeType.DOCUMENT @classmethod def document_type(cls) -> DocumentType: """Return document type.""" return DocumentType.MARKDOWN