github/DB-GPT
1
0
Fork 0
mirror of https://github.com/csunny/DB-GPT.git synced 2026-01-29 21:49:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/feature-xuyuan-openai-proxy' into ty_test

Browse Source 
# Conflicts:
#	pilot/model/llm_out/proxy_llm.py
This commit is contained in:
yhjun1026
2023-06-13 15:17:06 +08:00
parent 4690ac26ea 6e3b48c7c4
commit 4e5ce4d98b
5 changed files with 52 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
pilot/model/llm_out/proxy_llm.py
View File

@@ -62,10 +62,11 @@ def proxyllm_generate_stream(model, tokenizer, params, device, context_len=2048)
history.append(last_user_input)
payloads = {
"model": "gpt-3.5-turbo", # just for test_py, remove this later
"model": "gpt-3.5-turbo", # just for test, remove this later
"messages": history,
"temperature": params.get("temperature"),
"max_tokens": params.get("max_new_tokens"),
"stream": True
}
res = requests.post(
@@ -75,14 +76,32 @@ def proxyllm_generate_stream(model, tokenizer, params, device, context_len=2048)
text = ""
for line in res.iter_lines():
if line:
decoded_line = line.decode("utf-8")
try:
json_line = json.loads(decoded_line)
print(json_line)
text += json_line["choices"][0]["message"]["content"]
yield text
except Exception as e:
text += decoded_line
yield json.loads(text)["choices"][0]["message"]["content"]
json_data = line.split(b': ', 1)[1]
decoded_line = json_data.decode("utf-8")
if decoded_line.lower() != '[DONE]'.lower():
obj = json.loads(json_data)
if obj['choices'][0]['delta'].get('content') is not None:
content = obj['choices'][0]['delta']['content']
text += content
yield text
# native result.
# payloads = {
# "model": "gpt-3.5-turbo", # just for test, remove this later
# "messages": history,
# "temperature": params.get("temperature"),
# "max_tokens": params.get("max_new_tokens"),
# }
#
# res = requests.post(
# CFG.proxy_server_url, headers=headers, json=payloads, stream=True
# )
#
# text = ""
# line = res.content
# if line:
# decoded_line = line.decode("utf-8")
# json_line = json.loads(decoded_line)
# print(json_line)
# text += json_line["choices"][0]["message"]["content"]
# yield text