From 7debf52fc294376613e81cf41db074882e3da843 Mon Sep 17 00:00:00 2001 From: Zach Nussbaum Date: Wed, 19 Apr 2023 21:16:22 +0000 Subject: [PATCH] fix: stop gap to remove unused colulmns --- data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data.py b/data.py index e77f76f0..7e103dc1 100644 --- a/data.py +++ b/data.py @@ -106,13 +106,13 @@ def load_data(config, tokenizer): train_dataset = train_dataset.map( lambda ele: tokenize_inputs(config, tokenizer, ele), batched=True, - remove_columns=["source", "prompt"], + remove_columns=["source", "prompt", "id", "response"], **kwargs ) val_dataset = val_dataset.map( lambda ele: tokenize_inputs(config, tokenizer, ele), batched=True, - remove_columns=["source", "prompt"], + remove_columns=["source", "prompt", "id", "response"], **kwargs )