feat(storage): Support oss and s3

packages/dbgpt-serve/src/dbgpt_serve/file/api/endpoints.py

@@ -120,7 +120,6 @@ async def upload_files(
         global_system_app,
         service.upload_files,
         bucket,
-        "distributed",
         files,
         user_name,
         sys_code,

packages/dbgpt-serve/src/dbgpt_serve/file/config.py

@@ -1,12 +1,14 @@
 from dataclasses import dataclass, field
-from typing import Optional
+from typing import List, Optional
 
 from dbgpt.core.awel.flow import (
     TAGS_ORDER_HIGH,
     ResourceCategory,
     auto_register_resource,
 )
+from dbgpt.core.interface.file import StorageBackendConfig
 from dbgpt.util.i18n_utils import _
+from dbgpt.util.module_utils import ScannerConfig
 from dbgpt_serve.core import BaseServeConfig
 
 APP_NAME = "file"
@@ -27,6 +29,7 @@ SERVER_APP_TABLE_NAME = "dbgpt_serve_file"
         "files in the file server."
     ),
     show_in_ui=False,
+    skip_fields=["backends"],
 )
 @dataclass
 class ServeConfig(BaseServeConfig):
@@ -34,6 +37,13 @@ class ServeConfig(BaseServeConfig):
 
     __type__ = APP_NAME
 
+    __scan_config__ = ScannerConfig(
+        module_path="dbgpt_ext.storage.file",
+        base_class=StorageBackendConfig,
+        recursive=True,
+        specific_files=["config"],
+    )
+
     check_hash: Optional[bool] = field(
         default=True,
         metadata={"help": _("Check the hash of the file when downloading")},
@@ -62,6 +72,14 @@ class ServeConfig(BaseServeConfig):
     local_storage_path: Optional[str] = field(
         default=None, metadata={"help": _("The local storage path")}
     )
+    default_backend: Optional[str] = field(
+        default=None,
+        metadata={"help": _("The default storage backend")},
+    )
+    backends: List[StorageBackendConfig] = field(
+        default_factory=list,
+        metadata={"help": _("The storage backend configurations")},
+    )
 
     def get_node_address(self) -> str:
         """Get the node address"""

packages/dbgpt-serve/src/dbgpt_serve/file/serve.py

@@ -88,6 +88,7 @@ class Serve(BaseServe):
             FileMetadataAdapter(),
             serializer,
         )
+        default_backend = self._serve_config.default_backend
         simple_distributed_storage = SimpleDistributedStorage(
             node_address=self._serve_config.get_node_address(),
             local_storage_path=self._serve_config.get_local_storage_path(),
@@ -98,6 +99,15 @@ class Serve(BaseServe):
         storage_backends = {
             simple_distributed_storage.storage_type: simple_distributed_storage,
         }
+        for backend_config in self._serve_config.backends:
+            storage_backend = backend_config.create_storage()
+            storage_backends[storage_backend.storage_type] = storage_backend
+            if not default_backend:
+                # First backend is the default backend
+                default_backend = storage_backend.storage_type
+        if not default_backend:
+            default_backend = simple_distributed_storage.storage_type
+
         fs = FileStorageSystem(
             storage_backends,
             metadata_storage=storage,
@@ -107,6 +117,7 @@ class Serve(BaseServe):
             system_app=self._system_app,
             storage_system=fs,
             save_chunk_size=self._serve_config.save_chunk_size,
+            default_storage_type=default_backend,
         )
         self._system_app.register_instance(self._file_storage_client)
 

packages/dbgpt-serve/src/dbgpt_serve/file/service/service.py

@@ -79,7 +79,6 @@ class Service(BaseService[ServeEntity, ServeRequest, ServerResponse]):
     def upload_files(
         self,
         bucket: str,
-        storage_type: str,
         files: List[UploadFile],
         user_name: Optional[str] = None,
         sys_code: Optional[str] = None,
@@ -97,7 +96,6 @@ class Service(BaseService[ServeEntity, ServeRequest, ServerResponse]):
                 bucket,
                 file_name,
                 file_data=file.file,
-                storage_type=storage_type,
                 custom_metadata=custom_metadata,
             )
             parsed_uri = FileStorageURI.parse(uri)

packages/dbgpt-serve/src/dbgpt_serve/rag/service/service.py

@@ -187,7 +187,6 @@ class Service(BaseService[KnowledgeSpaceEntity, SpaceServeRequest, SpaceServeRes
                 bucket,
                 safe_filename,
                 doc_file.file,
-                storage_type="distributed",
                 custom_metadata=custom_metadata,
             )
             request.content = file_uri