github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-01 01:06:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

[NFC] polish applications/Colossal-LLaMA-2/colossal_llama2/tokenizer/init_tokenizer.py code style (#5228)

Browse Source
This commit is contained in:
李文军
2024-01-25 13:14:48 +08:00
committed by GitHub
parent ddf879e2db
commit ec912b1ba9
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
applications/Colossal-LLaMA-2/colossal_llama2/tokenizer/init_tokenizer.py
View File

@@ -6,12 +6,12 @@ Initialize new tokenizer for continual pre-training
"""
import argparse
import os
import json
import os
from typing import List, Union
from transformers.models.llama.tokenization_llama import LlamaTokenizer
from sentencepiece import sentencepiece_model_pb2 as sp_pb2_model
from transformers.models.llama.tokenization_llama import LlamaTokenizer
from colossalai.logging import get_dist_logger