Files
Javier Martinez 21d42fd97a feat: code execution v4 (#2295)
* feat: add bundle to remove

* fix: spaces

* feat: add xml render as skill spec

* feat: add skill volume root to cache

* fix: deduplicate values

* fix: change the mount path to skill_id

* fix: use different paths

* feat: add principal

(cherry picked from commit 5db64fe721d5706440ce9f342b1388ffe742bc16)

# Conflicts:
#	private_gpt/components/code_execution/base.py
#	private_gpt/components/code_execution/code_execution_component.py
#	private_gpt/components/code_execution/local.py
#	private_gpt/components/environment/manager.py
#	private_gpt/components/tools/builders/bash_tool_builder.py
#	private_gpt/components/tools/builders/text_editor_tool_builder.py
#	private_gpt/components/tools/processors/bash_processor.py

* fix: mypy

...

* fix: config

* fix: sandbox config

* feat: add forward cookies

* fix: loop

* feat: add present server

* feat: add feature flag for tools

* refactor: move principal to another better place

* feat: add api key principal

* docs: fix docs

* docs: add present server

* fix: principal

* fix: mypy

* fix: avoid to block the loop

* fix: blocks in expansion

* fix: remove maximum concurrent users

...

* fix: multiplexer

* fix: readers

* fix: more fixes

...

* fix: impl

* feat: tool scheduler

* feat: add adaptative

* feat: add chat worker

* fix: config

* fix: max

* feat: add chat/tools workers

* fix: mypy

* feat: add generic scheduler

* fix: get result

* feat: do serializable the tool executor

* fix: tools

* fix: config

* fix: config

* fix: args

* fix: config

* fix: serializer

* Revert "fix: blocks in expansion"

This reverts commit a2110f94a8.

* fix: unify all logic

* feat: add ingestion scheduler

* fix: settings

* fix: config

* feat: add arq worker to chat

* fix: arq worker

* fix: add nest

* fix: mypy

* fix: await

* fix: script stress

* fix: tokenizer

* fix: chat scheduler

* fix: mypy

* fix: add async tokenizer

* fix: improve condense

* fix: tool scheduler

* feat: add initial real async chat worker

* fix: mypy

* fix: do resumable local executor

...

...

...

fix: revert usleess changes

fix: remove parent chat job

fix: refactor

fix: loop

ref: rename models

fix: chat engine

fix: mypy

...

...

...

fix: fix deps

* fix: tests

* fix: tests

* ...

* fix: stream

* fix: config

* fix: scheduler

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* Handle PGPT_WORKER_MODE=celery in health check worker status

* fix: cancel

* fix: arch

* fix: test ingestion

* fix: deserialization of chat messages

* fix: broken results

* fix: mypy

* fix: test

* fix: config

* fix: remove arq tool worker

* fix: output cls

* fix: preserve early resumable tool callbacks

* fix: preserve async tool result order

* refactor: address worker PR review comments

* fix: mypy

* test: colocate ARQ chat enqueue coverage

* fix: remove redis from tests

* fix: mypy

* fix: tests

* test: isolate chat mocks and cancellation timing

* fix: tests

* fix: tests

* fix: test

(cherry picked from commit f8ee460af2)

* fix: worker config

* test: remove flaky chat cancellation assertion

(cherry picked from commit 1115ff2349)

# Conflicts:
#	tests/server/chat/test_chat_routes.py

* fix: emit chat pings from stream listeners

* fix: don't duplciate the output

* fix: rss memory

...

* fix: pass the args

* fix: threads

* fix: websearch

---------

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
2026-07-15 13:05:23 +02:00

123 lines
3.6 KiB
Python

from __future__ import annotations
import asyncio
import contextlib
import os
import signal
import sys
import time
from asyncio.subprocess import PIPE
from typing import TYPE_CHECKING
from private_gpt.components.code_execution.results import BashExecutionResult
if TYPE_CHECKING:
from pathlib import Path
_ISOLATION_AVAILABLE = sys.platform != "win32"
def _child_setup(cpu_s: int, mem_mb: int, fsize_mb: int, nproc: int) -> None:
"""Runs in child process before exec — sets up isolation via rlimit + setsid.
setrlimit calls are best-effort: some platforms (macOS) reject certain
limits (e.g. RLIMIT_AS always fails with a finite value).
"""
os.setsid()
import resource as r
def _set(which: int, limit: tuple[int, int]) -> None:
with contextlib.suppress(ValueError, OSError):
r.setrlimit(which, limit)
_set(r.RLIMIT_CPU, (cpu_s, cpu_s))
_set(r.RLIMIT_AS, (mem_mb << 20, mem_mb << 20))
_set(r.RLIMIT_FSIZE, (fsize_mb << 20, fsize_mb << 20))
_set(r.RLIMIT_NPROC, (nproc, nproc))
def _cap_bytes(data: bytes, limit: int) -> str:
if len(data) > limit:
truncated = data[:limit]
return (
truncated.decode("utf-8", errors="replace")
+ f"\n[output truncated at {limit} bytes]"
)
return data.decode("utf-8", errors="replace")
class LocalBashExecutor:
"""Async subprocess executor with resource isolation (Unix) and output capping.
On Unix: uses os.setsid + setrlimit (CPU, virtual memory, file size, nproc).
On Windows: runs without isolation (same behavior as before).
Timeout kills the entire process group via SIGKILL.
"""
def __init__(
self,
cpu_limit_seconds: int = 30,
memory_limit_mb: int = 512,
fsize_limit_mb: int = 50,
nproc_limit: int = 50,
output_cap_bytes: int = 10 * 1024 * 1024,
) -> None:
self._cpu_s = cpu_limit_seconds
self._mem_mb = memory_limit_mb
self._fsize_mb = fsize_limit_mb
self._nproc = nproc_limit
self._cap = output_cap_bytes
async def run(
self,
command: str,
cwd: Path,
timeout: int | None = None,
) -> BashExecutionResult:
import functools
preexec = (
functools.partial(
_child_setup, self._cpu_s, self._mem_mb, self._fsize_mb, self._nproc
)
if _ISOLATION_AVAILABLE
else None
)
t0 = time.monotonic()
proc = await asyncio.create_subprocess_shell(
command,
stdout=PIPE,
stderr=PIPE,
cwd=str(cwd),
preexec_fn=preexec,
)
try:
out, err = await asyncio.wait_for(
proc.communicate(), timeout=float(timeout) if timeout else 60.0
)
except TimeoutError:
try:
if _ISOLATION_AVAILABLE:
os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
else:
proc.kill()
except ProcessLookupError:
pass
await proc.wait()
return BashExecutionResult(
success=False,
stdout="",
stderr=f"Command timed out after {timeout}s.",
exit_code=124,
execution_time_ms=int((time.monotonic() - t0) * 1000),
)
return BashExecutionResult(
success=(proc.returncode == 0),
stdout=_cap_bytes(out, self._cap),
stderr=_cap_bytes(err, self._cap),
exit_code=proc.returncode or 0,
execution_time_ms=int((time.monotonic() - t0) * 1000),
)