github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-02 01:28:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix wrong EOS token in ColossalChat

Browse Source
This commit is contained in:
Orion-Zheng
2023-11-14 09:58:00 +08:00
committed by Zian(Andy) Zheng
parent 70885d707d
commit 43ad0d9ef0
6 changed files with 6 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
applications/Chat/examples/community/peft/train_peft_prompts.py
View File

@@ -118,7 +118,7 @@ def main(args):
tokenizer.pad_token = tokenizer.eos_token
elif args.model == "llama":
tokenizer = LlamaTokenizer.from_pretrained(args.pretrain)
tokenizer.eos_token = "<\s>"
tokenizer.eos_token = "</s>"
tokenizer.pad_token = tokenizer.unk_token
else:
raise ValueError(f'Unsupported model "{args.model}"')

2
applications/Chat/examples/community/peft/train_peft_sft.py
View File

@@ -68,7 +68,7 @@ def train(args):
padding_side="right",
use_fast=False,
)
tokenizer.eos_token = "<\s>"
tokenizer.eos_token = "</s>"
tokenizer.pad_token = tokenizer.unk_token
else:
raise ValueError(f'Unsupported model "{args.model}"')

2
applications/Chat/examples/inference.py
View File

@@ -39,7 +39,7 @@ def eval(args):
tokenizer.pad_token = tokenizer.eos_token
elif args.model == "llama":
tokenizer = LlamaTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer")
tokenizer.eos_token = "<\s>"
tokenizer.eos_token = "</s>"
tokenizer.pad_token = tokenizer.unk_token
else:
raise ValueError(f'Unsupported model "{args.model}"')

2
applications/Chat/examples/train_prompts.py
View File

@@ -125,7 +125,7 @@ def main(args):
tokenizer = LlamaTokenizer.from_pretrained(
"hf-internal-testing/llama-tokenizer" if args.tokenizer is None else args.tokenizer
)
tokenizer.eos_token = "<\s>"
tokenizer.eos_token = "</s>"
tokenizer.pad_token = tokenizer.unk_token
else:
raise ValueError(f'Unsupported model "{args.model}"')

2
applications/Chat/examples/train_reward_model.py
View File

@@ -72,7 +72,7 @@ def train(args):
tokenizer = LlamaTokenizer.from_pretrained(
"hf-internal-testing/llama-tokenizer" if args.tokenizer is None else args.tokenizer
)
tokenizer.eos_token = "<\s>"
tokenizer.eos_token = "</s>"
tokenizer.pad_token = tokenizer.unk_token
else:
raise ValueError(f'Unsupported model "{args.model}"')

2
applications/Chat/examples/train_sft.py
View File

@@ -75,7 +75,7 @@ def train(args):
tokenizer = LlamaTokenizer.from_pretrained(
"hf-internal-testing/llama-tokenizer" if args.tokenizer is None else args.tokenizer
)
tokenizer.eos_token = "<\s>"
tokenizer.eos_token = "</s>"
tokenizer.pad_token = tokenizer.unk_token
elif args.model == "chatglm":
tokenizer = ChatGLMTokenizer.from_pretrained(