mirror of
https://github.com/nomic-ai/gpt4all.git
synced 2025-08-10 12:21:42 +00:00
fix: stop gap to remove unused colulmns
This commit is contained in:
parent
405d8c1bbc
commit
7debf52fc2
4
data.py
4
data.py
@ -106,13 +106,13 @@ def load_data(config, tokenizer):
|
||||
train_dataset = train_dataset.map(
|
||||
lambda ele: tokenize_inputs(config, tokenizer, ele),
|
||||
batched=True,
|
||||
remove_columns=["source", "prompt"],
|
||||
remove_columns=["source", "prompt", "id", "response"],
|
||||
**kwargs
|
||||
)
|
||||
val_dataset = val_dataset.map(
|
||||
lambda ele: tokenize_inputs(config, tokenizer, ele),
|
||||
batched=True,
|
||||
remove_columns=["source", "prompt"],
|
||||
remove_columns=["source", "prompt", "id", "response"],
|
||||
**kwargs
|
||||
)
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user