github/DB-GPT
1
0
Fork 0
mirror of https://github.com/csunny/DB-GPT.git synced 2025-08-25 11:29:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

add openai proxy, support chatgpt model only if you provide open ai secret key.

Browse Source
This commit is contained in:
xuyuan23 2023-06-12 17:29:35 +08:00
parent 44bb5135cd
commit 62eb6c383e
1 changed files with 29 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
pilot/model/llm_out/proxy_llm.py
View File

@ -66,6 +66,7 @@ def proxyllm_generate_stream(model, tokenizer, params, device, context_len=2048)
"messages": history, "messages": history,
"temperature": params.get("temperature"), "temperature": params.get("temperature"),
"max_tokens": params.get("max_new_tokens"), "max_tokens": params.get("max_new_tokens"),
"stream": True
} }
res = requests.post( res = requests.post(
@ -75,8 +76,32 @@ def proxyllm_generate_stream(model, tokenizer, params, device, context_len=2048)
text = "" text = ""
for line in res.iter_lines(): for line in res.iter_lines():
if line: if line:
decoded_line = line.decode("utf-8") json_data = line.split(b': ', 1)[1]
json_line = json.loads(decoded_line) decoded_line = json_data.decode("utf-8")
print(json_line) if decoded_line.lower() != '[DONE]'.lower():
text += json_line["choices"][0]["message"]["content"] obj = json.loads(json_data)
if obj['choices'][0]['delta'].get('content') is not None:
content = obj['choices'][0]['delta']['content']
text += content
yield text yield text
# native result.
# payloads = {
# "model": "gpt-3.5-turbo", # just for test, remove this later
# "messages": history,
# "temperature": params.get("temperature"),
# "max_tokens": params.get("max_new_tokens"),
# }
#
# res = requests.post(
# CFG.proxy_server_url, headers=headers, json=payloads, stream=True
# )
#
# text = ""
# line = res.content
# if line:
# decoded_line = line.decode("utf-8")
# json_line = json.loads(decoded_line)
# print(json_line)
# text += json_line["choices"][0]["message"]["content"]
# yield text