github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-02 09:38:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

add orpo

Browse Source
This commit is contained in:
YeAnbang
2024-06-27 07:20:28 +00:00
parent f3de5a025c
commit c8d1b4a968
12 changed files with 783 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
applications/ColossalChat/examples/data_preparation_scripts/prepare_sft_dataset.sh
View File

@@ -5,7 +5,7 @@ rm -rf $SAVE_DIR/jsonl
rm -rf $SAVE_DIR/arrow
python prepare_dataset.py --type sft \
--data_input_dirs /PATH/TO/PREFERENCE/DATASET \
--data_input_dirs /PATH/TO/SFT/DATASET \
--conversation_template_config /PATH/TO/CHAT/TEMPLATE/CONFIG.json \
--tokenizer_dir "" \
--data_cache_dir $SAVE_DIR/cache \