diff --git a/Dockerfile b/Dockerfile index 22926587..9c98baf9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -122,7 +122,8 @@ ARG PGPT_DOWNLOAD_PLAYWRIGHT COPY pyproject.toml uv.lock ./ -ENV HF_HOME=local_data +ENV PGPT_HOME=/home/worker/app +ENV HF_HOME=/home/worker/app/local_data ENV PLAYWRIGHT_BROWSERS_PATH=/home/worker/app/.local-browsers ENV TIKTOKEN_CACHE_DIR=/home/worker/app/tiktoken_cache ENV TIKTOKEN_ENCODINGS_BASE=/home/worker/app/encodings @@ -206,11 +207,13 @@ COPY --chown=worker settings.yaml settings.yaml RUN --mount=type=cache,target=/root/.cache/uv uv pip install --python /home/worker/app/.venv/bin/python --no-deps . ENV PATH="/home/worker/app/.venv/bin:/usr/local/bin:${PATH}" -ENV HF_HOME=local_data +ENV PGPT_HOME=/home/worker/app +ENV HF_HOME=/home/worker/app/local_data ENV PYTHONPATH="$PYTHONPATH:/private_gpt/" ENV SETUPTOOLS_USE_DISTUTILS=stdlib ENV PLAYWRIGHT_BROWSERS_PATH=/home/worker/app/.local-browsers +ENV TIKTOKEN_CACHE_DIR=/home/worker/app/tiktoken_cache ENV TIKTOKEN_ENCODINGS_BASE=/home/worker/app/encodings ENV TIKTOKEN_RS_CACHE_DIR=/home/worker/app/encodings diff --git a/Makefile b/Makefile index 72594c08..d087eab6 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,10 @@ PROD_PYTHON ?= .venv/bin/python PROD_BINARY ?= .venv/bin/private-gpt PROD_ARGS ?= serve PROD_UV_CACHE_DIR ?= .uv-cache +TEST_PGPT_HOME ?= $(CURDIR) +TEST_LOCAL_DATA_DIR ?= $(TEST_PGPT_HOME)/local_data/tests +WIPE_PGPT_HOME := $(if $(PGPT_HOME),$(PGPT_HOME),$(HOME)/.local/share/private-gpt) +WIPE_LOCAL_DATA_DIR := $(WIPE_PGPT_HOME)/local_data .PHONY: test test-coverage black ruff format mypy check auto-discover-models update-openapi-spec run dev-windows dev prod-run api-docs docs ingest wipe celery flower @@ -14,12 +18,12 @@ PROD_UV_CACHE_DIR ?= .uv-cache ######################################################################################################################## test: - rm -rf local_data/tests/* - PYTHONPATH=. uv run pytest tests + rm -rf "$(TEST_LOCAL_DATA_DIR)"/* + PGPT_HOME=$(TEST_PGPT_HOME) PYTHONPATH=. uv run pytest tests test-coverage: - rm -rf local_data/tests/* - PYTHONPATH=. uv run pytest tests --cov private_gpt --cov-report term --cov-report=html --cov-report xml --junit-xml=tests-results.xml + rm -rf "$(TEST_LOCAL_DATA_DIR)"/* + PGPT_HOME=$(TEST_PGPT_HOME) PYTHONPATH=. uv run pytest tests --cov private_gpt --cov-report term --cov-report=html --cov-report xml --junit-xml=tests-results.xml black: uv run black . --check @@ -92,8 +96,8 @@ ingest: @uv run python scripts/ingest_folder.py $(call args) wipe: - @mkdir -p local_data - @find local_data -mindepth 1 ! -name '.gitignore' -exec rm -rf {} + + @mkdir -p "$(WIPE_LOCAL_DATA_DIR)" + @find "$(WIPE_LOCAL_DATA_DIR)" -mindepth 1 ! -name '.gitignore' -exec rm -rf {} + ######################################################################################################################## # Celery ######################################################################################################################## diff --git a/fern/docs/pages/api-guide/ingestion.mdx b/fern/docs/pages/api-guide/ingestion.mdx index 1fcd5d50..2065dac4 100644 --- a/fern/docs/pages/api-guide/ingestion.mdx +++ b/fern/docs/pages/api-guide/ingestion.mdx @@ -47,7 +47,7 @@ make wipe ``` - This deletes everything in `local_data/` including the vector store. It cannot be undone. + This deletes everything under `PGPT_HOME/local_data/` (default `~/.local/share/private-gpt/local_data/`) including the vector store. It cannot be undone. --- diff --git a/fern/docs/pages/installation/package.mdx b/fern/docs/pages/installation/package.mdx index 6e81c86b..03264bd7 100644 --- a/fern/docs/pages/installation/package.mdx +++ b/fern/docs/pages/installation/package.mdx @@ -98,6 +98,22 @@ After installing, set `OPENAI_API_BASE` to your LLM server and start: PrivateGPT starts on port `8080` by default. Change it with `private-gpt serve --port `. +## Data storage + +All application data (vector store, ingested documents, models, caches) is stored under a single home directory: + +| Platform | Default path | +|---|---| +| macOS / Linux | `~/.local/share/private-gpt/` | +| Windows | `%LOCALAPPDATA%\private-gpt\` | +| Docker | `/home/worker/app/` (fixed) | + +Override the location with `PGPT_HOME`: + +```bash +PGPT_HOME=/data/private-gpt private-gpt serve +``` + ## Key environment variables | Variable | Default | Description | @@ -106,6 +122,7 @@ PrivateGPT starts on port `8080` by default. Change it with `private-gpt serve - | `OPENAI_API_KEY` | _(empty)_ | API key, if your server requires one | | `OPENAI_EMBEDDING_API_BASE` | same as `OPENAI_API_BASE` | Override for the embeddings endpoint | | `PORT` | `8080` | Port the server listens on (also settable via `private-gpt serve --port`) | +| `PGPT_HOME` | `~/.local/share/private-gpt` | Root directory for all local data (vector store, models, caches) | | `PGPT_LLM_AUTO_DISCOVER_MODELS` | `true` | Discover LLM models from `/v1/models` on startup | ## What's next? diff --git a/fern/docs/pages/storage/storage.mdx b/fern/docs/pages/storage/storage.mdx index 97c01efc..5dffb2f2 100644 --- a/fern/docs/pages/storage/storage.mdx +++ b/fern/docs/pages/storage/storage.mdx @@ -26,7 +26,7 @@ skills: storage_provider: local ``` -Local storage paths default to `local_data/private_gpt/skills`. No additional configuration is needed. +Local storage paths default to `local_data/private_gpt/skills` under `PGPT_HOME` (e.g. `~/.local/share/private-gpt/local_data/private_gpt/skills`). No additional configuration is needed. --- diff --git a/fern/docs/pages/storage/vectordb.mdx b/fern/docs/pages/storage/vectordb.mdx index 707cb7fe..b07c2958 100644 --- a/fern/docs/pages/storage/vectordb.mdx +++ b/fern/docs/pages/storage/vectordb.mdx @@ -11,7 +11,7 @@ PrivateGPT uses [Qdrant](https://qdrant.tech/) as the default vector store. By d ```yaml qdrant: - path: local_data/private_gpt/qdrant + path: local_data/qdrant # relative to PGPT_HOME (~/.local/share/private-gpt by default) ``` --- diff --git a/private_gpt/__init__.py b/private_gpt/__init__.py index 805bbfb1..5bf5e848 100644 --- a/private_gpt/__init__.py +++ b/private_gpt/__init__.py @@ -6,6 +6,8 @@ import warnings from pydantic import PydanticDeprecatedSince20 +from private_gpt.constants import PGPT_HOME + # Set to 'DEBUG' to have extensive logging turned on, even for libraries ROOT_LOG_LEVEL = "INFO" @@ -15,7 +17,8 @@ logging.captureWarnings(True) # adding tiktoken cache path within repo to be able to run in offline environment. -os.environ["TIKTOKEN_CACHE_DIR"] = "tiktoken_cache" +if "TIKTOKEN_CACHE_DIR" not in os.environ: + os.environ["TIKTOKEN_CACHE_DIR"] = str(PGPT_HOME / "tiktoken_cache") # Disable warning tokenizer about torch os.environ["TRANSFORMERS_VERBOSITY"] = "error" diff --git a/private_gpt/celery/backend_config.py b/private_gpt/celery/backend_config.py index cd00ded0..b7e3ab45 100644 --- a/private_gpt/celery/backend_config.py +++ b/private_gpt/celery/backend_config.py @@ -3,6 +3,7 @@ from typing import Any from pydantic import BaseModel, Field +from private_gpt.paths import local_data_path from private_gpt.settings.settings import settings celery_settings = settings().celery @@ -17,7 +18,8 @@ BackendConfigProvider = Callable[[], BackendConfig] def _local_backend() -> BackendConfig: - return BackendConfig(url="db+sqlite:///celery_backend.db") + local_data_path.mkdir(parents=True, exist_ok=True) + return BackendConfig(url=f"db+sqlite:///{local_data_path / 'celery_backend.db'}") def _redis_backend() -> BackendConfig: diff --git a/private_gpt/celery/broker_config.py b/private_gpt/celery/broker_config.py index 150c76f1..2c30d58f 100644 --- a/private_gpt/celery/broker_config.py +++ b/private_gpt/celery/broker_config.py @@ -23,6 +23,7 @@ def _local_broker() -> BrokerConfig: transport_options={ "data_folder_in": local_data_path, "data_folder_out": local_data_path, + "control_folder": local_data_path / "control", }, ) diff --git a/private_gpt/components/llm/tokenizers/models/model_discovery.py b/private_gpt/components/llm/tokenizers/models/model_discovery.py index 60bbfb7e..f9a7ca0f 100644 --- a/private_gpt/components/llm/tokenizers/models/model_discovery.py +++ b/private_gpt/components/llm/tokenizers/models/model_discovery.py @@ -21,11 +21,12 @@ from private_gpt.components.llm.tokenizers.models.model_cache import ( validate_model_path, ) from private_gpt.components.llm.tokenizers.models.model_downloader import download_model +from private_gpt.constants import PGPT_HOME logger = logging.getLogger(__name__) # CLI constants -HF_HOME = Path(os.environ.get("HF_HOME", "./models/cache")) +HF_HOME = Path(os.environ.get("HF_HOME", str(PGPT_HOME / "models" / "cache"))) LOCK_FILE = HF_HOME / ".model-download.lock" LOCK_TIMEOUT = int(os.environ.get("DOWNLOAD_LOCK_TIMEOUT", "3600")) OFFLINE_MODE = os.environ.get("HF_HUB_OFFLINE", "0") == "1" diff --git a/private_gpt/components/sqlite/sqlite_client.py b/private_gpt/components/sqlite/sqlite_client.py index 4ac535dc..80f269ab 100644 --- a/private_gpt/components/sqlite/sqlite_client.py +++ b/private_gpt/components/sqlite/sqlite_client.py @@ -3,6 +3,7 @@ import threading from importlib.util import find_spec from pathlib import Path +from private_gpt.paths import resolve_data_path from private_gpt.settings.settings import Settings if find_spec("sqlalchemy") is None: @@ -18,7 +19,7 @@ logger.setLevel(logging.DEBUG) class SQLiteClient: def __init__(self, local_path: str, database: str) -> None: - self._local_path = Path(local_path).joinpath(f"{database}.db") + self._local_path = resolve_data_path(local_path) / f"{database}.db" Path(self._local_path).parent.mkdir(parents=True, exist_ok=True) self._sync_engine = create_engine(f"sqlite:///{self._local_path}") self._sync_engine = self._sync_engine.execution_options( diff --git a/private_gpt/components/vector_store/qdrant_client_builder.py b/private_gpt/components/vector_store/qdrant_client_builder.py index b370ed45..eab5d304 100644 --- a/private_gpt/components/vector_store/qdrant_client_builder.py +++ b/private_gpt/components/vector_store/qdrant_client_builder.py @@ -70,6 +70,10 @@ class QdrantClientBuilder: config = settings.qdrant.get_parameters(QdrantClient, exclude_none=True) if QdrantClientBuilder.is_local_path(config): + from private_gpt.paths import resolve_data_path + + config = dict(config) + config["path"] = str(resolve_data_path(config["path"])) # This is a workaround to allow to execute tests/local qdrant # when we want to support sync/async client. # To allow, remove .lock from the db_dir diff --git a/private_gpt/constants.py b/private_gpt/constants.py index f1e3b411..f3b838c2 100644 --- a/private_gpt/constants.py +++ b/private_gpt/constants.py @@ -6,3 +6,20 @@ PROJECT_ROOT_PATH: Path = ( .expanduser() .resolve() ) + + +def _default_pgpt_home() -> str: + if os.name == "nt": # Windows + local_app_data = os.environ.get("LOCALAPPDATA") + base = ( + Path(local_app_data) + if local_app_data + else Path.home() / "AppData" / "Local" + ) + return str(base / "private-gpt") + return str(Path.home() / ".local" / "share" / "private-gpt") + + +PGPT_HOME: Path = ( + Path(os.environ.get("PGPT_HOME", _default_pgpt_home())).expanduser().resolve() +) diff --git a/private_gpt/paths.py b/private_gpt/paths.py index 57ebff06..1e7021e7 100644 --- a/private_gpt/paths.py +++ b/private_gpt/paths.py @@ -1,21 +1,21 @@ from pathlib import Path -from private_gpt.constants import PROJECT_ROOT_PATH +from private_gpt.constants import PGPT_HOME, PROJECT_ROOT_PATH from private_gpt.settings.settings import settings -def _absolute_or_from_project_root(path: str) -> Path: - if path.startswith("/"): - return Path(path) - return PROJECT_ROOT_PATH / path +def resolve_data_path(path: str) -> Path: + """Resolve a data path, handling absolute paths and paths relative to PGPT_HOME.""" + p = Path(path).expanduser() + if p.is_absolute(): + return p.resolve() + return (PGPT_HOME / p).resolve() -models_path: Path = PROJECT_ROOT_PATH / "models" +models_path: Path = PGPT_HOME / "models" models_cache_path: Path = models_path / "cache" docs_path: Path = PROJECT_ROOT_PATH / "docs" -local_data_path: Path = _absolute_or_from_project_root( - settings().data.local_data_folder -) +local_data_path: Path = resolve_data_path(settings().data.local_data_folder) prompt_templates_path: Path = ( Path(__file__).resolve().parent / "components" / "prompts" / "templates"