github/privateGPT
1
0
Fork 1
mirror of https://github.com/imartinez/privateGPT.git synced 2025-07-13 07:04:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added new routes for deleting file from the ingested files

Browse Source
This commit is contained in:
Saurab-Shrestha 2024-02-01 17:22:39 +05:45
parent 93b3c4f574
commit 71999eb150
12 changed files with 243 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
private_gpt/constants.py
View File

@ -1,4 +1,5 @@
import os
from pathlib import Path
PROJECT_ROOT_PATH: Path = Path(__file__).parents[1]
UPLOAD_DIR = rf"C:\Users\Dbuser\QuickGPT\backend\privateGPT\private_gpt\uploads"
UPLOAD_DIR = os.path.join(os.getcwd(), "uploads")

69
private_gpt/home.py
View File

@ -1,4 +1,7 @@
"""This file should be imported only and only if you want to run the UI locally."""
from private_gpt.users.core import security
from private_gpt.users.api import deps
from private_gpt.users import crud, models, schemas
import time
from fastapi import File, Request, UploadFile
from fastapi.responses import StreamingResponse
@ -6,15 +9,16 @@ import itertools
import logging
from collections.abc import Iterable
from pathlib import Path
from typing import Any, List
from typing import Any, List, Literal
from fastapi import APIRouter, Depends, Request, FastAPI, Body
from fastapi import APIRouter, Depends, Request, FastAPI, Body, status, HTTPException, Security
from fastapi.responses import JSONResponse
from gradio.themes.utils.colors import slate # type: ignore
from injector import inject, singleton
from llama_index.llms import ChatMessage, ChatResponse, MessageRole
from pydantic import BaseModel
from private_gpt.server.ingest.model import IngestedDoc
from private_gpt.constants import PROJECT_ROOT_PATH
from private_gpt.di import global_injector
from private_gpt.server.chat.chat_service import ChatService, CompletionGen
@ -23,20 +27,27 @@ from private_gpt.server.ingest.ingest_service import IngestService
from private_gpt.settings.settings import settings
from private_gpt.ui.images import logo_svg
from private_gpt.ui.common import Source
from private_gpt.constants import UPLOAD_DIR
logger = logging.getLogger(__name__)
THIS_DIRECTORY_RELATIVE = Path(__file__).parent.relative_to(PROJECT_ROOT_PATH)
# Should be "private_gpt/ui/avatar-bot.ico"
AVATAR_BOT = THIS_DIRECTORY_RELATIVE / "avatar-bot.ico"
UI_TAB_TITLE = "My Private GPT"
SOURCES_SEPARATOR = "\n Sources: \n"
MODES = ["Query Docs", "Search in Docs", "LLM Chat"]
home_router = APIRouter(prefix="/v1")
DEFAULT_MODE = MODES[0]
chat_router = APIRouter(prefix="/v1", tags=["Chat"])
class ListFilesResponse(BaseModel):
uploaded_files: List[str]
class IngestResponse(BaseModel):
object: Literal["list"]
model: Literal["private-gpt"]
data: list[IngestedDoc]
@singleton
@ -99,7 +110,6 @@ class Home:
return history_messages[:20]
new_message = ChatMessage(content=message, role=MessageRole.USER)
# Appending user message to history
self._history.append([message, ""])
all_messages = [*build_history(), new_message]
match mode:
@ -149,27 +159,19 @@ class Home:
home_instance = global_injector.get(Home)
async def chat_simulation(message, mode):
response = home_instance._chat(message=message, mode=mode)
return StreamingResponse(
response,
media_type='text/event-stream'
)
DEFAULT_MODE = MODES[0]
def get_home_instance(request: Request) -> Home:
home_instance = request.state.injector.get(Home)
return home_instance
@home_router.post("/chat")
async def chat_endpoint(request: Request, message: str = Body(...), mode: str = Body(DEFAULT_MODE)):
# Create the Home instance
# home_instance = request.state.injector.get(Home)
@chat_router.post("/chat")
async def chat_endpoint(
home_instance: Home = Depends(get_home_instance),
message: str = Body(...), mode: str = Body(DEFAULT_MODE),
current_user: models.User = Security(
deps.get_current_user,
)
):
response = home_instance._chat(message=message, mode=mode)
return StreamingResponse(
response,
@ -177,16 +179,15 @@ async def chat_endpoint(request: Request, message: str = Body(...), mode: str =
)
class ListFilesResponse(BaseModel):
uploaded_files: List[str]
@home_router.get("/list_files")
async def list_files(home_instance: Home = Depends(get_home_instance)) -> dict:
@chat_router.get("/list_files")
async def list_files(
home_instance: Home = Depends(get_home_instance),
current_user: models.User = Security(
deps.get_current_user,
)) -> dict:
"""
List all uploaded files.
"""
uploaded_files = home_instance._list_ingested_files()
return {"uploaded_files": uploaded_files}

10
private_gpt/launcher.py
View File

@ -14,7 +14,7 @@ from private_gpt.server.ingest.ingest_router import ingest_router
from private_gpt.users.api.v1.api import api_router
from private_gpt.settings.settings import Settings
from private_gpt.home import home_router
from private_gpt.home import chat_router
logger = logging.getLogger(__name__)
@ -24,8 +24,7 @@ def create_app(root_injector: Injector) -> FastAPI:
async def bind_injector_to_request(request: Request) -> None:
request.state.injector = root_injector
# app = FastAPI(dependencies=[Depends(bind_injector_to_request)])
app = FastAPI()
app = FastAPI(dependencies=[Depends(bind_injector_to_request)])
app.include_router(completions_router)
app.include_router(chat_router)
app.include_router(chunks_router)
@ -34,7 +33,7 @@ def create_app(root_injector: Injector) -> FastAPI:
app.include_router(health_router)
app.include_router(api_router)
app.include_router(home_router)
app.include_router(chat_router)
settings = root_injector.get(Settings)
if settings.server.cors.enabled:
logger.debug("Setting up CORS middleware")
@ -52,8 +51,5 @@ def create_app(root_injector: Injector) -> FastAPI:
# admin_ui = root_injector.get(PrivateAdminGptUi)
# admin_ui.mount_in_admin_app(app, '/admin')
# from private_gpt.ui.ui import PrivateGptUi
# ui = root_injector.get(PrivateGptUi)
# ui.mount_in_app(app, settings.ui.path)
return app

77
private_gpt/server/ingest/ingest_router.py
View File

@ -1,17 +1,23 @@
import logging
from pathlib import Path
from typing import Literal
from fastapi import APIRouter, Depends, HTTPException, Request, UploadFile, File
from fastapi import APIRouter, Depends, HTTPException, Request, UploadFile, File, status, Security
from fastapi.responses import JSONResponse
from pydantic import BaseModel, Field
from private_gpt.home import Home
from private_gpt.users import crud, models, schemas
from private_gpt.users.api import deps
from private_gpt.server.ingest.ingest_service import IngestService
from private_gpt.server.ingest.model import IngestedDoc
from private_gpt.server.utils.auth import authenticated
from private_gpt.constants import UPLOAD_DIR
from pathlib import Path
ingest_router = APIRouter(prefix="/v1", dependencies=[Depends(authenticated)])
logger = logging.getLogger(__name__)
class IngestTextBody(BaseModel):
file_name: str = Field(examples=["Avatar: The Last Airbender"])
text: str = Field(
@ -39,7 +45,7 @@ def ingest(request: Request, file: UploadFile) -> IngestResponse:
return ingest_file(request, file)
@ingest_router.post("/ingest/file", tags=["Ingestion"])
@ingest_router.post("/ingest/file1", tags=["Ingestion"])
def ingest_file(request: Request, file: UploadFile = File(...)) -> IngestResponse:
"""Ingests and processes a file, storing its chunks to be used as context.
@ -56,7 +62,6 @@ def ingest_file(request: Request, file: UploadFile = File(...)) -> IngestRespons
can be used to filter the context used to create responses in
`/chat/completions`, `/completions`, and `/chunks` APIs.
"""
# try:
service = request.state.injector.get(IngestService)
if file.filename is None:
raise HTTPException(400, "No file name provided")
@ -116,12 +121,66 @@ def delete_ingested(request: Request, doc_id: str) -> None:
service.delete(doc_id)
@ingest_router.delete("/ingest", tags=["Ingestion"])
def delete_ingested(request: Request, doc_id: str) -> None:
"""Delete the specified ingested Document.
@ingest_router.delete("/ingest/file/{filename}", tags=["Ingestion"])
def delete_file(
request: Request,
filename: str,
current_user: models.User = Security(
deps.get_current_user,
)) -> dict:
"""Delete the specified filename.
The `doc_id` can be obtained from the `GET /ingest/list` endpoint.
The `filename` can be obtained from the `GET /ingest/list` endpoint.
The document will be effectively deleted from your storage context.
"""
service = request.state.injector.get(IngestService)
service.delete(doc_id)
try:
doc_ids = service.get_doc_ids_by_filename(filename)
if not doc_ids:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND,
detail=f"No documents found with filename '{filename}'")
for doc_id in doc_ids:
service.delete(doc_id)
return {"status": "SUCCESS", "message": f"{filename}' successfully deleted."}
except Exception as e:
logger.error(
f"Unexpected error deleting documents with filename '{filename}': {str(e)}")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Internal Server Error")
@ingest_router.post("/ingest/file", response_model=IngestResponse, tags=["Ingestion"])
def ingest_file(
request: Request,
file: UploadFile = File(...),
current_user: models.User = Security(
deps.get_current_user,
)) -> IngestResponse:
"""Ingests and processes a file, storing its chunks to be used as context."""
service = request.state.injector.get(IngestService)
try:
if file.filename is None:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST, detail="No file name provided")
upload_path = Path(f"{UPLOAD_DIR}/{file.filename}")
with open(upload_path, "wb") as f:
f.write(file.file.read())
with open(upload_path, "rb") as f:
ingested_documents = service.ingest_bin_data(file.filename, f)
return IngestResponse(object="list", model="private-gpt", data=ingested_documents)
except Exception as e:
logger.error(f"There was an error uploading the file(s): {str(e)}")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Internal Server Error: Unable to ingest file.",
)
finally:
file.file.close()

16
private_gpt/server/ingest/ingest_service.py
View File

@ -131,3 +131,19 @@ class IngestService:
)
self.ingest_component.delete(doc_id)
def get_doc_ids_by_filename(self, filename: str) -> list[str]:
doc_ids: set[str] = set()
try:
docstore = self.storage_context.docstore
for node in docstore.docs.values():
if node.metadata is not None and node.metadata.get("file_name") == filename:
doc_ids.add(node.ref_doc_id)
except ValueError:
logger.warning(
"Got an exception when getting doc_ids by filename", exc_info=True)
pass
logger.debug("Found count=%s doc_ids for filename '%s'",
len(doc_ids), filename)
return doc_ids

2
private_gpt/users/api/deps.py
View File

@ -46,7 +46,6 @@ async def get_current_user(
token: str = Depends(reusable_oauth2)
) -> models.User:
print("HELLO--------------------------------------------")
if security_scopes.scopes:
authenticate_value = f'Bearer scope="{security_scopes.scope_str}"'
else:
@ -89,7 +88,6 @@ async def get_current_user(
detail="Not enough permissions",
headers={"WWW-Authenticate": authenticate_value},
)
print("THE USER IS::::::::::::::::::::::", user)
return user

154
private_gpt/users/api/v1/routers/auth.py
View File

@ -2,11 +2,9 @@ from typing import Any, Optional
from datetime import timedelta, datetime
from sqlalchemy.orm import Session
from pydantic.networks import EmailStr
from fastapi.responses import JSONResponse
from fastapi.encoders import jsonable_encoder
from fastapi.security import OAuth2PasswordRequestForm
from fastapi import APIRouter, Body, Depends, HTTPException, Security, Path, status
from fastapi import APIRouter, Body, Depends, HTTPException, Security, status
from private_gpt.users.api import deps
from private_gpt.users.core import security
@ -76,7 +74,6 @@ def login_access_token(
raise HTTPException(
status_code=400, detail="Incorrect email or password"
)
access_token_expires = timedelta(
minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES
)
@ -89,9 +86,7 @@ def login_access_token(
company_id=user.company_id,
last_login=datetime.now()
)
user = crud.user.update(db, db_obj=user, obj_in=user_in)
if user.user_role:
role = user.user_role.role.name
if user.user_role.company_id:
@ -102,12 +97,11 @@ def login_access_token(
"id": str(user.id),
"email": str(user.email),
"username": str(user.fullname),
"role": role,
"company_id": company_id,
}
return {
response_dict = {
"access_token": security.create_access_token(
token_payload, expires_delta=access_token_expires
),
@ -116,127 +110,105 @@ def login_access_token(
),
"token_type": "bearer",
}
return JSONResponse(content=response_dict)
@router.post("/login/refresh-token", response_model=schemas.TokenSchema)
def refresh_access_token(
db: Session = Depends(deps.get_db),
form_data: OAuth2PasswordRequestForm = Depends(),
) -> Any:
"""
Refresh access token using a valid refresh token
"""
def refresh_access_token(db: Session = Depends(deps.get_db), form_data: OAuth2PasswordRequestForm = Depends()) -> Any:
refresh_token = form_data.refresh_token
token_payload = security.verify_refresh_token(refresh_token)
if not token_payload:
raise HTTPException(status_code=401, detail="Invalid refresh token")
access_token_expires = timedelta(minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES)
refresh_token_expires = timedelta(minutes=settings.REFRESH_TOKEN_EXPIRE_MINUTES)
access_token_expires = timedelta(
minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES)
refresh_token_expires = timedelta(
minutes=settings.REFRESH_TOKEN_EXPIRE_MINUTES)
return {
"access_token": security.create_access_token(
token_payload, expires_delta=access_token_expires
),
"refresh_token": security.create_refresh_token(
token_payload, expires_delta=refresh_token_expires
),
response_dict = {
"access_token": security.create_access_token(token_payload, expires_delta=access_token_expires),
"refresh_token": security.create_refresh_token(token_payload, expires_delta=refresh_token_expires),
"token_type": "bearer",
}
@router.post("/{company_id}/register", response_model=schemas.User)
def register_for_company(
*,
db: Session = Depends(deps.get_db),
email: str = Body(...),
fullname: str = Body(...),
company_id: int = Path(..., title="Company ID",
description="Only for company admin"),
current_user: models.User = Security(
deps.get_current_user,
scopes=[Role.SUPER_ADMIN["name"], Role.ADMIN['name']],
),
) -> Any:
"""
Register new user for a specific company.
"""
user = crud.user.get_by_email(db, email=email)
if user:
raise HTTPException(
status_code=409,
detail="The user with this username already exists in the system",
)
if current_user.user_role.role.name not in {Role.ADMIN["name"], Role.SUPER_ADMIN["name"]}:
raise HTTPException(
status_code=403,
detail="You do not have permission to register users for a company.",
)
company = crud.company.get_by_id(db, id=company_id)
print(f"Company is : {company.id}")
if not (current_user.user_role.role.name == Role.ADMIN["name"] and current_user.user_role.company_id == company.id):
raise HTTPException(
status_code=403,
detail="You are not the admin of the specified company.",
)
random_password = security.generate_random_password()
user = register_user(db, email, fullname, random_password, company)
user_role = create_user_role(db, user, Role.GUEST["name"], company)
token_payload = create_token_payload(user, user_role)
return JSONResponse(
status_code=status.HTTP_201_CREATED,
content={"message": "User registered successfully.\n\n Check respective user email for login credentials",
"user": jsonable_encoder(user)},
)
return JSONResponse(content=response_dict)
@router.post("/register", response_model=schemas.TokenSchema)
def register_without_company_assignment(
def register_user(
*,
db: Session = Depends(deps.get_db),
email: str = Body(...),
fullname: str = Body(...),
company_id: int = Body(None, title="Company ID", description="Company ID for the user (if applicable)"),
company_id: int = Body(None, title="Company ID",
description="Company ID for the user (if applicable)"),
role_name: str = Body(None, title="Role Name",
description="User role name (if applicable)"),
current_user: models.User = Security(
deps.get_current_user,
scopes=[Role.SUPER_ADMIN["name"], Role.ADMIN['name']],
scopes=[Role.ADMIN["name"], Role.SUPER_ADMIN["name"]],
),
) -> Any:
"""
Register new user with company assignment.
Register new user with optional company assignment and role selection.
"""
user = crud.user.get_by_email(db, email=email)
if user:
raise HTTPException(
status_code=409,
detail="The user with this username already exists in the system",
detail="The user with this email already exists in the system",
)
if current_user.user_role.role.name != Role.SUPER_ADMIN["name"]:
raise HTTPException(
status_code=403,
detail="You do not have permission to register users without a company.",
)
if company_id is not None:
# Registering user with a specific company
company = crud.company.get(db, company_id)
if not company:
raise HTTPException(
status_code=404,
detail="Company not found.",
)
if current_user.user_role.role.name not in {Role.SUPER_ADMIN["name"], Role.ADMIN["name"]}:
raise HTTPException(
status_code=403,
detail="You do not have permission to register users for a specific company.",
)
user_role_name = role_name or Role.GUEST["name"]
if user_role_name == Role.SUPER_USER["name"]:
raise HTTPException(
status_code=403,
detail="Cannot create a user with SUPER_USER role.",
)
user_role = create_user_role(db, user, user_role_name, company)
else:
# Registering user without a specific company
if current_user.user_role.role.name != Role.SUPER_ADMIN["name"]:
raise HTTPException(
status_code=403,
detail="You do not have permission to register users without a company.",
)
user_role_name = role_name or Role.ADMIN["name"]
if user_role_name == Role.SUPER_USER["name"]:
raise HTTPException(
status_code=403,
detail="Cannot create a user with SUPER_USER role.",
)
user_role = create_user_role(db, user, user_role_name, None)
if company_id is None:
raise HTTPException(
status_code=400,
detail="Company ID is required for registering a user without a specific company.",
)
random_password = security.generate_random_password()
company = crud.company.get(db, company_id)
user = register_user(db, email, fullname, random_password, company)
user_role = create_user_role(db, user, Role.ADMIN["name"], company)
token_payload = create_token_payload(user, user_role)
return {
response_dict = {
"access_token": security.create_access_token(token_payload, expires_delta=timedelta(minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES)),
"refresh_token": security.create_refresh_token(token_payload, expires_delta=timedelta(minutes=settings.REFRESH_TOKEN_EXPIRE_MINUTES)),
"token_type": "bearer",
}
return JSONResponse(content=response_dict, status_code=status.HTTP_201_CREATED)

16
private_gpt/users/api/v1/routers/companies.py
View File

@ -54,6 +54,22 @@ def create_company(
)
@router.get("", response_model=List[schemas.Company])
def list_companies(
db: Session = Depends(deps.get_db),
skip: int = 0,
limit: int = 100,
current_user: models.User = Security(
deps.get_current_user,
scopes=[Role.SUPER_ADMIN["name"]],
),
) -> List[schemas.Company]:
"""
Retrieve a list of companies with pagination support.
"""
companies = crud.company.get_multi(db, skip=skip, limit=limit)
return companies
@router.get("/{company_id}", response_model=schemas.Company)
def read_company(
company_id: int,

1
private_gpt/users/api/v1/routers/subscriptions.py
View File

@ -1,7 +1,6 @@
from typing import Any, List
from sqlalchemy.orm import Session
from pydantic.networks import EmailStr
from fastapi import APIRouter, Body, Depends, HTTPException, Security, status
from fastapi.encoders import jsonable_encoder
from fastapi.responses import JSONResponse

5
private_gpt/users/api/v1/routers/user_roles.py
View File

@ -70,8 +70,3 @@ def update_user_role(
)
company_router = APIRouter(prefix="/user-roles", tags=["user-roles"])

41
private_gpt/users/api/v1/routers/users.py
View File

@ -212,5 +212,42 @@ def home_page(
deps.get_active_subscription,
),
):
return JSONResponse(status_code=status.HTTP_200_OK, content={"message": "Welcome to QuickGPT"})
return JSONResponse(status_code=status.HTTP_200_OK, content={"message": "Welcome to QuickGPT"})
@router.patch("/{user_id}/change-password", response_model=schemas.User)
def admin_change_password(
*,
db: Session = Depends(deps.get_db),
user_id: int,
new_password: str = Body(..., embed=True),
current_user: models.User = Security(
deps.get_current_user,
scopes=[Role.ADMIN["name"], Role.SUPER_ADMIN["name"]],
),
) -> Any:
"""
Admin/Super Admin change user's password without confirming the previous password.
"""
user = crud.user.get(db, id=user_id)
if not user:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="The user with this id does not exist in the system",
)
new_password_hashed = get_password_hash(new_password)
user.hashed_password = new_password_hashed
db.commit()
user_data = schemas.UserBaseSchema(
id=user.id,
email=user.email,
fullname=user.fullname,
company_id=user.company_id,
)
return JSONResponse(
status_code=status.HTTP_200_OK,
content={"message": "User password changed successfully",
"user": jsonable_encoder(user_data)},
)

2
private_gpt/users/core/config.py
View File

@ -10,7 +10,7 @@ SQLALCHEMY_DATABASE_URI = "postgresql+psycopg2://{username}:{password}@{host}:{p
port='5432',
db_name='QuickGpt',
username='postgres',
password="admin",
password="quick",
)
class Settings(BaseSettings):