github/DB-GPT
1
0
Fork 0
mirror of https://github.com/csunny/DB-GPT.git synced 2025-08-13 14:06:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
csunny 2023-04-28 22:48:39 +08:00
parent 57fd9d48ad
commit fc4a9a953b
2 changed files with 16 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pilot/model/__init__.py
View File

@ -1,2 +0,0 @@
from model.loader import *

24
pilot/server/sqlgpt.py
View File

@ -1,15 +1,23 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
#-*- coding: utf-8 -*- #-*- coding: utf-8 -*-
from pilot.model.loader import ModerLoader import torch
from fastchat.serve.inference import generate_stream from fastchat.serve.inference import generate_stream, compress_module
from pilot.configs.model_config import *
BASE_MODE = "/home/magic/workspace/github/DB-GPT/models/vicuna-13b"
from transformers import AutoTokenizer, AutoModelForCausalLM
if __name__ == "__main__": if __name__ == "__main__":
model_path = llm_model_config[LLM_MODEL] device = "cuda" if torch.cuda.is_available() else "cpu"
tokenizer = AutoTokenizer.from_pretrained(BASE_MODE, use_fast=False)
model = AutoModelForCausalLM.from_pretrained(
BASE_MODE,
low_cpu_mem_usage=True,
torch_dtype=torch.float16,
device_map="auto",
)
ml = ModerLoader(model_path) print(device)
model, tokenizer = ml.loader(load_8bit=True) #compress_module(model, device)
print(model) print(model, tokenizer)
print(tokenizer)