mirror of
https://github.com/imartinez/privateGPT.git
synced 2026-07-17 01:26:48 +00:00
* feat: add bundle to remove * fix: spaces * feat: add xml render as skill spec * feat: add skill volume root to cache * fix: deduplicate values * fix: change the mount path to skill_id * fix: use different paths * feat: add principal (cherry picked from commit 5db64fe721d5706440ce9f342b1388ffe742bc16) # Conflicts: # private_gpt/components/code_execution/base.py # private_gpt/components/code_execution/code_execution_component.py # private_gpt/components/code_execution/local.py # private_gpt/components/environment/manager.py # private_gpt/components/tools/builders/bash_tool_builder.py # private_gpt/components/tools/builders/text_editor_tool_builder.py # private_gpt/components/tools/processors/bash_processor.py * fix: mypy ... * fix: config * fix: sandbox config * feat: add forward cookies * fix: loop * feat: add present server * feat: add feature flag for tools * refactor: move principal to another better place * feat: add api key principal * docs: fix docs * docs: add present server * fix: principal * fix: mypy * fix: avoid to block the loop * fix: blocks in expansion * fix: remove maximum concurrent users ... * fix: multiplexer * fix: readers * fix: more fixes ... * fix: impl * feat: tool scheduler * feat: add adaptative * feat: add chat worker * fix: config * fix: max * feat: add chat/tools workers * fix: mypy * feat: add generic scheduler * fix: get result * feat: do serializable the tool executor * fix: tools * fix: config * fix: config * fix: args * fix: config * fix: serializer * Revert "fix: blocks in expansion" This reverts commita2110f94a8. * fix: unify all logic * feat: add ingestion scheduler * fix: settings * fix: config * feat: add arq worker to chat * fix: arq worker * fix: add nest * fix: mypy * fix: await * fix: script stress * fix: tokenizer * fix: chat scheduler * fix: mypy * fix: add async tokenizer * fix: improve condense * fix: tool scheduler * feat: add initial real async chat worker * fix: mypy * fix: do resumable local executor ... ... ... fix: revert usleess changes fix: remove parent chat job fix: refactor fix: loop ref: rename models fix: chat engine fix: mypy ... ... ... fix: fix deps * fix: tests * fix: tests * ... * fix: stream * fix: config * fix: scheduler * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * Handle PGPT_WORKER_MODE=celery in health check worker status * fix: cancel * fix: arch * fix: test ingestion * fix: deserialization of chat messages * fix: broken results * fix: mypy * fix: test * fix: config * fix: remove arq tool worker * fix: output cls * fix: preserve early resumable tool callbacks * fix: preserve async tool result order * refactor: address worker PR review comments * fix: mypy * test: colocate ARQ chat enqueue coverage * fix: remove redis from tests * fix: mypy * fix: tests * test: isolate chat mocks and cancellation timing * fix: tests * fix: tests * fix: test (cherry picked from commitf8ee460af2) * fix: worker config * test: remove flaky chat cancellation assertion (cherry picked from commit1115ff2349) # Conflicts: # tests/server/chat/test_chat_routes.py * fix: emit chat pings from stream listeners * fix: don't duplciate the output * fix: rss memory ... * fix: pass the args * fix: threads * fix: websearch --------- Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
170 lines
6.8 KiB
Python
170 lines
6.8 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
from abc import ABC, abstractmethod
|
|
from typing import TYPE_CHECKING
|
|
|
|
if TYPE_CHECKING:
|
|
from pathlib import Path
|
|
|
|
from private_gpt.components.sandbox.base import SandboxSession
|
|
from private_gpt.components.sandbox.content_bundle import ContentBundle
|
|
from private_gpt.components.sandbox.mount import VolumeSpec
|
|
|
|
|
|
def _volume_name(canonical_path: str, prefix: str = "bundle") -> str:
|
|
return f"{prefix}-{hashlib.sha1(canonical_path.encode()).hexdigest()[:8]}"
|
|
|
|
|
|
class ContentMounter(ABC):
|
|
"""Knows how to get a ContentBundle's content into a running sandbox.
|
|
|
|
Implementations are composed in a prioritized list; the manager picks the
|
|
first one whose can_handle() returns True for a given descriptor. Volume
|
|
mounters declare their bind-mount before container creation via
|
|
prepare_volume() and are no-ops in materialize(); copy-based mounters
|
|
leave prepare_volume() returning None and do the work in materialize(),
|
|
which is called lazily before the first exec() after the bundle is registered.
|
|
"""
|
|
|
|
@abstractmethod
|
|
def can_handle(self, descriptor: ContentBundle) -> bool:
|
|
"""Return True if this mounter can materialize this descriptor type."""
|
|
|
|
async def prepare_volume(
|
|
self, descriptor: ContentBundle, session_id: str
|
|
) -> VolumeSpec | None:
|
|
"""Return a VolumeSpec to bind-mount this content at container creation.
|
|
|
|
When non-None, the spec is wired into sandbox creation and materialize()
|
|
must be a no-op for this descriptor. Default: None (use materialize()).
|
|
"""
|
|
return None
|
|
|
|
@abstractmethod
|
|
async def materialize(
|
|
self, descriptor: ContentBundle, sandbox: SandboxSession
|
|
) -> None:
|
|
"""Write the content into the live sandbox at descriptor.canonical_path.
|
|
|
|
Called lazily just before the first exec() after the bundle is registered.
|
|
Always overwrites — no path_exists() check needed; idempotency is by design.
|
|
"""
|
|
|
|
|
|
class InlineContentMounter(ContentMounter):
|
|
"""Materializes ContentBundle instances whose files are already in memory."""
|
|
|
|
def can_handle(self, descriptor: ContentBundle) -> bool:
|
|
from private_gpt.components.sandbox.content_bundle import StoredBundle
|
|
|
|
return not isinstance(descriptor, StoredBundle)
|
|
|
|
async def materialize(
|
|
self, descriptor: ContentBundle, sandbox: SandboxSession
|
|
) -> None:
|
|
await sandbox.initialize_mount(descriptor.canonical_path, descriptor.files)
|
|
|
|
|
|
class FetchContentMounter(ContentMounter):
|
|
"""Materializes StoredBundle instances by calling their fetch() callable.
|
|
|
|
Works with any storage backend — fetch() is injected at bundle construction
|
|
time by the SkillLoader (or equivalent). Writes to whatever filesystem the
|
|
sandbox has, whether ephemeral or S3-backed.
|
|
"""
|
|
|
|
def can_handle(self, descriptor: ContentBundle) -> bool:
|
|
from private_gpt.components.sandbox.content_bundle import StoredBundle
|
|
|
|
return isinstance(descriptor, StoredBundle)
|
|
|
|
async def materialize(
|
|
self, descriptor: ContentBundle, sandbox: SandboxSession
|
|
) -> None:
|
|
from private_gpt.components.sandbox.content_bundle import StoredBundle
|
|
|
|
if isinstance(descriptor, StoredBundle):
|
|
files = await descriptor.fetch()
|
|
await sandbox.initialize_mount(descriptor.canonical_path, files)
|
|
|
|
|
|
class LocalStorageContentMounter(ContentMounter):
|
|
"""Volume-mounts StoredBundle instances from a local storage root on the host.
|
|
|
|
When skills are stored locally (storage_provider='local'), the bundle files
|
|
already exist at storage_root/storage_prefix and are bind-mounted directly.
|
|
When storage_provider='s3', the local directory may be empty; in that case
|
|
prepare_volume() fetches from the storage backend and caches the files
|
|
locally before returning the VolumeSpec — keeps the bind-mount the only
|
|
write path so callers never need to write to read-only container paths.
|
|
For bundles added lazily after container creation, materialize() re-fetches
|
|
and writes directly into the sandbox (bind-mounts cannot be added post-start).
|
|
"""
|
|
|
|
def __init__(self, storage_root: Path) -> None:
|
|
self._root = storage_root
|
|
|
|
def can_handle(self, descriptor: ContentBundle) -> bool:
|
|
from private_gpt.components.sandbox.content_bundle import StoredBundle
|
|
|
|
return isinstance(descriptor, StoredBundle)
|
|
|
|
async def prepare_volume(
|
|
self, descriptor: ContentBundle, session_id: str
|
|
) -> VolumeSpec | None:
|
|
from private_gpt.components.sandbox.content_bundle import StoredBundle
|
|
from private_gpt.components.sandbox.mount import VolumeSpec
|
|
|
|
if not isinstance(descriptor, StoredBundle):
|
|
return None
|
|
|
|
host_path = self._root / descriptor.storage_prefix
|
|
if not host_path.is_dir() or not any(host_path.iterdir()):
|
|
# Local directory is absent or empty — fetch from the storage backend
|
|
# (S3 etc.) and cache locally so the bind-mount has content.
|
|
files = await descriptor.fetch()
|
|
if not files:
|
|
return None
|
|
for f in files:
|
|
dest = host_path / f.path
|
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
dest.write_bytes(f.content)
|
|
|
|
return VolumeSpec(
|
|
name=_volume_name(descriptor.canonical_path, "stored"),
|
|
host_path=host_path,
|
|
mount_path=descriptor.canonical_path,
|
|
read_only=not descriptor.writable,
|
|
)
|
|
|
|
async def materialize(
|
|
self, descriptor: ContentBundle, sandbox: SandboxSession
|
|
) -> None:
|
|
from private_gpt.components.sandbox.content_bundle import StoredBundle
|
|
from private_gpt.components.sandbox.local import BashExecutorSandbox
|
|
|
|
if not isinstance(descriptor, StoredBundle):
|
|
return
|
|
|
|
if isinstance(sandbox, BashExecutorSandbox):
|
|
# Ensure skill files exist locally (fetch from backend if absent),
|
|
# then register the host-path mapping in the path translator so
|
|
# that subsequent exec() calls can resolve the canonical path.
|
|
host_path = self._root / descriptor.storage_prefix
|
|
if not host_path.is_dir() or not any(host_path.iterdir()):
|
|
files = await descriptor.fetch()
|
|
if not files:
|
|
return
|
|
for f in files:
|
|
dest = host_path / f.path
|
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
dest.write_bytes(f.content)
|
|
sandbox.add_local_mount(
|
|
descriptor.canonical_path, host_path, writable=descriptor.writable
|
|
)
|
|
else:
|
|
files = await descriptor.fetch()
|
|
if files:
|
|
await sandbox.initialize_mount(descriptor.canonical_path, files)
|