github/gpt4all
1
0
Fork 0
mirror of https://github.com/nomic-ai/gpt4all.git synced 2025-08-13 13:46:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: stop gap to remove unused colulmns

Browse Source
This commit is contained in:
Zach Nussbaum 2023-04-19 21:16:22 +00:00
parent 405d8c1bbc
commit 7debf52fc2
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
data.py
View File

@ -106,13 +106,13 @@ def load_data(config, tokenizer):
train_dataset = train_dataset.map( train_dataset = train_dataset.map(
lambda ele: tokenize_inputs(config, tokenizer, ele), lambda ele: tokenize_inputs(config, tokenizer, ele),
batched=True, batched=True,
remove_columns=["source", "prompt"], remove_columns=["source", "prompt", "id", "response"],
**kwargs **kwargs
) )
val_dataset = val_dataset.map( val_dataset = val_dataset.map(
lambda ele: tokenize_inputs(config, tokenizer, ele), lambda ele: tokenize_inputs(config, tokenizer, ele),
batched=True, batched=True,
remove_columns=["source", "prompt"], remove_columns=["source", "prompt", "id", "response"],
**kwargs **kwargs
) )