Files
privateGPT/private_gpt/components/environment/content_mounter.py
Javier Martinez 21d42fd97a feat: code execution v4 (#2295)
* feat: add bundle to remove

* fix: spaces

* feat: add xml render as skill spec

* feat: add skill volume root to cache

* fix: deduplicate values

* fix: change the mount path to skill_id

* fix: use different paths

* feat: add principal

(cherry picked from commit 5db64fe721d5706440ce9f342b1388ffe742bc16)

# Conflicts:
#	private_gpt/components/code_execution/base.py
#	private_gpt/components/code_execution/code_execution_component.py
#	private_gpt/components/code_execution/local.py
#	private_gpt/components/environment/manager.py
#	private_gpt/components/tools/builders/bash_tool_builder.py
#	private_gpt/components/tools/builders/text_editor_tool_builder.py
#	private_gpt/components/tools/processors/bash_processor.py

* fix: mypy

...

* fix: config

* fix: sandbox config

* feat: add forward cookies

* fix: loop

* feat: add present server

* feat: add feature flag for tools

* refactor: move principal to another better place

* feat: add api key principal

* docs: fix docs

* docs: add present server

* fix: principal

* fix: mypy

* fix: avoid to block the loop

* fix: blocks in expansion

* fix: remove maximum concurrent users

...

* fix: multiplexer

* fix: readers

* fix: more fixes

...

* fix: impl

* feat: tool scheduler

* feat: add adaptative

* feat: add chat worker

* fix: config

* fix: max

* feat: add chat/tools workers

* fix: mypy

* feat: add generic scheduler

* fix: get result

* feat: do serializable the tool executor

* fix: tools

* fix: config

* fix: config

* fix: args

* fix: config

* fix: serializer

* Revert "fix: blocks in expansion"

This reverts commit a2110f94a8.

* fix: unify all logic

* feat: add ingestion scheduler

* fix: settings

* fix: config

* feat: add arq worker to chat

* fix: arq worker

* fix: add nest

* fix: mypy

* fix: await

* fix: script stress

* fix: tokenizer

* fix: chat scheduler

* fix: mypy

* fix: add async tokenizer

* fix: improve condense

* fix: tool scheduler

* feat: add initial real async chat worker

* fix: mypy

* fix: do resumable local executor

...

...

...

fix: revert usleess changes

fix: remove parent chat job

fix: refactor

fix: loop

ref: rename models

fix: chat engine

fix: mypy

...

...

...

fix: fix deps

* fix: tests

* fix: tests

* ...

* fix: stream

* fix: config

* fix: scheduler

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* Handle PGPT_WORKER_MODE=celery in health check worker status

* fix: cancel

* fix: arch

* fix: test ingestion

* fix: deserialization of chat messages

* fix: broken results

* fix: mypy

* fix: test

* fix: config

* fix: remove arq tool worker

* fix: output cls

* fix: preserve early resumable tool callbacks

* fix: preserve async tool result order

* refactor: address worker PR review comments

* fix: mypy

* test: colocate ARQ chat enqueue coverage

* fix: remove redis from tests

* fix: mypy

* fix: tests

* test: isolate chat mocks and cancellation timing

* fix: tests

* fix: tests

* fix: test

(cherry picked from commit f8ee460af2)

* fix: worker config

* test: remove flaky chat cancellation assertion

(cherry picked from commit 1115ff2349)

# Conflicts:
#	tests/server/chat/test_chat_routes.py

* fix: emit chat pings from stream listeners

* fix: don't duplciate the output

* fix: rss memory

...

* fix: pass the args

* fix: threads

* fix: websearch

---------

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
2026-07-15 13:05:23 +02:00

170 lines
6.8 KiB
Python

from __future__ import annotations
import hashlib
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from pathlib import Path
from private_gpt.components.sandbox.base import SandboxSession
from private_gpt.components.sandbox.content_bundle import ContentBundle
from private_gpt.components.sandbox.mount import VolumeSpec
def _volume_name(canonical_path: str, prefix: str = "bundle") -> str:
return f"{prefix}-{hashlib.sha1(canonical_path.encode()).hexdigest()[:8]}"
class ContentMounter(ABC):
"""Knows how to get a ContentBundle's content into a running sandbox.
Implementations are composed in a prioritized list; the manager picks the
first one whose can_handle() returns True for a given descriptor. Volume
mounters declare their bind-mount before container creation via
prepare_volume() and are no-ops in materialize(); copy-based mounters
leave prepare_volume() returning None and do the work in materialize(),
which is called lazily before the first exec() after the bundle is registered.
"""
@abstractmethod
def can_handle(self, descriptor: ContentBundle) -> bool:
"""Return True if this mounter can materialize this descriptor type."""
async def prepare_volume(
self, descriptor: ContentBundle, session_id: str
) -> VolumeSpec | None:
"""Return a VolumeSpec to bind-mount this content at container creation.
When non-None, the spec is wired into sandbox creation and materialize()
must be a no-op for this descriptor. Default: None (use materialize()).
"""
return None
@abstractmethod
async def materialize(
self, descriptor: ContentBundle, sandbox: SandboxSession
) -> None:
"""Write the content into the live sandbox at descriptor.canonical_path.
Called lazily just before the first exec() after the bundle is registered.
Always overwrites — no path_exists() check needed; idempotency is by design.
"""
class InlineContentMounter(ContentMounter):
"""Materializes ContentBundle instances whose files are already in memory."""
def can_handle(self, descriptor: ContentBundle) -> bool:
from private_gpt.components.sandbox.content_bundle import StoredBundle
return not isinstance(descriptor, StoredBundle)
async def materialize(
self, descriptor: ContentBundle, sandbox: SandboxSession
) -> None:
await sandbox.initialize_mount(descriptor.canonical_path, descriptor.files)
class FetchContentMounter(ContentMounter):
"""Materializes StoredBundle instances by calling their fetch() callable.
Works with any storage backend — fetch() is injected at bundle construction
time by the SkillLoader (or equivalent). Writes to whatever filesystem the
sandbox has, whether ephemeral or S3-backed.
"""
def can_handle(self, descriptor: ContentBundle) -> bool:
from private_gpt.components.sandbox.content_bundle import StoredBundle
return isinstance(descriptor, StoredBundle)
async def materialize(
self, descriptor: ContentBundle, sandbox: SandboxSession
) -> None:
from private_gpt.components.sandbox.content_bundle import StoredBundle
if isinstance(descriptor, StoredBundle):
files = await descriptor.fetch()
await sandbox.initialize_mount(descriptor.canonical_path, files)
class LocalStorageContentMounter(ContentMounter):
"""Volume-mounts StoredBundle instances from a local storage root on the host.
When skills are stored locally (storage_provider='local'), the bundle files
already exist at storage_root/storage_prefix and are bind-mounted directly.
When storage_provider='s3', the local directory may be empty; in that case
prepare_volume() fetches from the storage backend and caches the files
locally before returning the VolumeSpec — keeps the bind-mount the only
write path so callers never need to write to read-only container paths.
For bundles added lazily after container creation, materialize() re-fetches
and writes directly into the sandbox (bind-mounts cannot be added post-start).
"""
def __init__(self, storage_root: Path) -> None:
self._root = storage_root
def can_handle(self, descriptor: ContentBundle) -> bool:
from private_gpt.components.sandbox.content_bundle import StoredBundle
return isinstance(descriptor, StoredBundle)
async def prepare_volume(
self, descriptor: ContentBundle, session_id: str
) -> VolumeSpec | None:
from private_gpt.components.sandbox.content_bundle import StoredBundle
from private_gpt.components.sandbox.mount import VolumeSpec
if not isinstance(descriptor, StoredBundle):
return None
host_path = self._root / descriptor.storage_prefix
if not host_path.is_dir() or not any(host_path.iterdir()):
# Local directory is absent or empty — fetch from the storage backend
# (S3 etc.) and cache locally so the bind-mount has content.
files = await descriptor.fetch()
if not files:
return None
for f in files:
dest = host_path / f.path
dest.parent.mkdir(parents=True, exist_ok=True)
dest.write_bytes(f.content)
return VolumeSpec(
name=_volume_name(descriptor.canonical_path, "stored"),
host_path=host_path,
mount_path=descriptor.canonical_path,
read_only=not descriptor.writable,
)
async def materialize(
self, descriptor: ContentBundle, sandbox: SandboxSession
) -> None:
from private_gpt.components.sandbox.content_bundle import StoredBundle
from private_gpt.components.sandbox.local import BashExecutorSandbox
if not isinstance(descriptor, StoredBundle):
return
if isinstance(sandbox, BashExecutorSandbox):
# Ensure skill files exist locally (fetch from backend if absent),
# then register the host-path mapping in the path translator so
# that subsequent exec() calls can resolve the canonical path.
host_path = self._root / descriptor.storage_prefix
if not host_path.is_dir() or not any(host_path.iterdir()):
files = await descriptor.fetch()
if not files:
return
for f in files:
dest = host_path / f.path
dest.parent.mkdir(parents=True, exist_ok=True)
dest.write_bytes(f.content)
sandbox.add_local_mount(
descriptor.canonical_path, host_path, writable=descriptor.writable
)
else:
files = await descriptor.fetch()
if files:
await sandbox.initialize_mount(descriptor.canonical_path, files)