github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-20 09:01:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix style

Browse Source
This commit is contained in:
YeAnbang
2024-07-29 01:29:18 +00:00
parent de1bf08ed0
commit 6fd9e86864
2 changed files with 6 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
applications/ColossalChat/coati/dataset/tokenization_utils.py
View File

@@ -193,7 +193,7 @@ def apply_rlhf_data_format(template: Conversation, tokenizer: Any):
template.messages[: 2 * target_turn], prompt, template.end_of_assistant
)
# no truncation applied
tokenized, starts, ends = tokenize_and_concatenate(tokenizer, chunks, require_loss, max_length=int(1e10))
tokenized, starts, ends = tokenize_and_concatenate(tokenizer, chunks, require_loss, max_length=None)
loss_mask = [0] * len(tokenized)
label_decode = []