github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-20 09:01:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix style

Browse Source
This commit is contained in:
YeAnbang
2024-07-26 09:55:15 +00:00
parent 9688e19b32
commit 8a3ff4f315
7 changed files with 10 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
applications/ColossalChat/coati/dataset/tokenization_utils.py
View File

@@ -46,8 +46,7 @@ def supervised_tokenize_sft(
max_length: the maximum context length
"""
if ignore_index is None:
ignore_index = IGNORE_INDEX
ignore_index = IGNORE_INDEX
messages = data_point["messages"]
template = deepcopy(conversation_template)
@@ -146,8 +145,6 @@ def tokenize_prompt_dataset(
ignore_index: the ignore index when calculate loss during training
max_length: the maximum context length
"""
if ignore_index is None:
ignore_index = IGNORE_INDEX
messages = data_point["messages"]
template = deepcopy(conversation_template)
@@ -226,8 +223,6 @@ def tokenize_rlhf(
{"context": [{"from": "user", "content": "xxx"}, {"from": "assistant", "content": "xxx"}],
"chosen": {"from": "assistant", "content": "xxx"}, "rejected": {"from": "assistant", "content": "xxx"}}
"""
if ignore_index is None:
ignore_index = IGNORE_INDEX
context = data_point["context"]
template = deepcopy(conversation_template)