github/gpt4all
1
0
Fork 0
mirror of https://github.com/nomic-ai/gpt4all.git synced 2025-06-24 14:32:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Slight cleanup of superfluous comment and space after comma

Browse Source
This commit is contained in:
MalikMAlna 2023-04-06 19:56:49 -04:00
parent dc08c43867
commit b3be94a0ef
1 changed files with 1 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
data.py
View File

@ -57,7 +57,6 @@ def load_data(config, tokenizer):
dataset_path = config["dataset_path"] dataset_path = config["dataset_path"]
if os.path.exists(dataset_path): if os.path.exists(dataset_path):
# check if path is a directory
if os.path.isdir(dataset_path): if os.path.isdir(dataset_path):
files = glob.glob(os.path.join(dataset_path, "*_clean.jsonl")) files = glob.glob(os.path.join(dataset_path, "*_clean.jsonl"))
else: else:
@ -68,7 +67,7 @@ def load_data(config, tokenizer):
dataset = load_dataset("json", data_files=files, split="train") dataset = load_dataset("json", data_files=files, split="train")
else: else:
dataset = load_dataset(dataset_path,split='train') dataset = load_dataset(dataset_path, split='train')
dataset = dataset.train_test_split(test_size=.05, seed=config["seed"]) dataset = dataset.train_test_split(test_size=.05, seed=config["seed"])