mirror of
https://github.com/hpcaitech/ColossalAI.git
synced 2025-05-20 14:20:20 +00:00
14 lines
604 B
Bash
Executable File
14 lines
604 B
Bash
Executable File
SAVE_DIR="/home/nvme-share/home/yeanbang/data/experiments/sft"
|
|
|
|
rm -rf $SAVE_DIR/cache
|
|
rm -rf $SAVE_DIR/jsonl
|
|
rm -rf $SAVE_DIR/arrow
|
|
|
|
python prepare_dataset.py --type sft \
|
|
--data_input_dirs /home/nvme-share/home/yeanbang/data/dataset/hh_rlhf/sft \
|
|
--conversation_template_config /home/nvme-share/home/yeanbang/ColossalAI/applications/ColossalChat/config/conversation_template/llama2.json \
|
|
--tokenizer_dir "/home/nvme-share/share/models/Sheared-LLaMA-1.3B" \
|
|
--data_cache_dir $SAVE_DIR/cache \
|
|
--data_jsonl_output_dir $SAVE_DIR/jsonl \
|
|
--data_arrow_output_dir $SAVE_DIR/arrow \
|