github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-10 13:30:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

[Fix] resolve conflicts of merging main

Browse Source
This commit is contained in:
Yuanheng
2024-04-08 16:21:47 +08:00
parent 7ebdf48ac5 641b1ee71a
commit ed5ebd1735
451 changed files with 15350 additions and 10694 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
colossalai/legacy/inference/hybridengine/engine.py
View File

@@ -46,7 +46,7 @@ class CaiInferEngine:
model = LlamaForCausalLM.from_pretrained("your_path_to_model")
tokenizer = LlamaTokenizer.from_pretrained("/home/lczyh/share/models/llama-7b-hf")
# assume the model is infered with 2 pipeline stages
# assume the model is inferred with 2 pipeline stages
inferengine = CaiInferEngine(pp_size=2, model=model, model_policy=LlamaModelInferPolicy())
input = ["Introduce a landmark in China ","Introduce a landmark in China "]
@@ -70,7 +70,7 @@ class CaiInferEngine:
max_input_len: int = 32,
max_output_len: int = 32,
verbose: bool = False,
# TODO: implement early_stopping, and various gerneration options
# TODO: implement early_stopping, and various generation options
early_stopping: bool = False,
do_sample: bool = False,
num_beams: int = 1,