github/DB-GPT
1
0
Fork 0
mirror of https://github.com/csunny/DB-GPT.git synced 2025-08-08 03:44:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

proxy llm, support stream format (#190)

Browse Source
This commit is contained in:
Aries-ckt 2023-06-12 22:11:45 +08:00 committed by GitHub
commit 17b190b691
1 changed files with 10 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
pilot/model/llm_out/proxy_llm.py
View File

@ -76,7 +76,13 @@ def proxyllm_generate_stream(model, tokenizer, params, device, context_len=2048)
for line in res.iter_lines():
if line:
decoded_line = line.decode("utf-8")
json_line = json.loads(decoded_line)
print(json_line)
text += json_line["choices"][0]["message"]["content"]
yield text
try:
json_line = json.loads(decoded_line)
print(json_line)
text += json_line["choices"][0]["message"]["content"]
yield text
except Exception as e:
text += decoded_line
yield json.loads(text)["choices"][0]["message"]["content"]