feat: eval faster

.gitignore

@@ -1,3 +1,4 @@
+eval_data/*.pkl
 *.jsonl
 *tar.gz
 ckpts**

configs/eval/generate.yaml

@@ -1,15 +0,0 @@
-# model/tokenizer
-model_name: # update with llama 7b 
-tokenizer_name: # update with llama 7b
-lora: true
-lora_path: "nomic-ai/gpt4all-lora"
-
-max_new_tokens: 512
-temperature: 0.001
-prompt: | 
-  #this code prints a string reversed
-  my_string = "hello how are you"
-  print(len(my_string))
-
-
-  My code above does not work. Can you help me?

configs/eval/generate_baseline.yaml

@@ -1,11 +1,13 @@
 # model/tokenizer
-model_name: # update with llama model name
-tokenizer_name: # update with llama model name
+model_name: "zpn/llama-7b"
+tokenizer_name: "zpn/llama-7b"
 lora: true
 lora_path: "tloen/alpaca-lora-7b"
 
 
-
+stride: 512
+num_proc: 64
+batch_size: 1
 max_new_tokens: 512
 temperature: 0.001
 prompt: | 

configs/eval/generate_full.yaml

@@ -1,14 +0,0 @@
-# model/tokenizer
-model_name: # update
-tokenizer_name: # update
-lora_path: "no-lora"
-
-max_new_tokens: 512
-temperature: 0.001
-prompt: | 
-  #this code prints a string reversed
-  my_string = "hello how are you"
-  print(len(my_string))
-
-
-  My code above does not work. Can you help me?

configs/eval/generate_gpt4all.yaml

@@ -0,0 +1,9 @@
+# model/tokenizer
+model_name: "nomic-ai/gpt4all-gptj-multinode-deepspeed-epoch_0"
+tokenizer_name: "EleutherAI/gpt-j-6B"
+lora: false
+lora_path: null
+
+stride: 512
+num_proc: 64
+batch_size: 1

configs/eval/generate_gpt4all_lora.yaml

@@ -0,0 +1,9 @@
+# model/tokenizer
+model_name: "zpn/llama-7b"
+tokenizer_name: "zpn/llama-7b"
+lora: true
+lora_path: "nomic-ai/gpt4all-lora"
+
+stride: 512
+num_proc: 64
+batch_size: 1

configs/eval/generate_large_2.yaml

@@ -1,15 +0,0 @@
-# model/tokenizer
-model_name: # update
-tokenizer_name: # update
-lora: true
-lora_path: # update
-
-max_new_tokens: 512
-temperature: 0.001
-prompt: | 
-  #this code prints a string reversed
-  my_string = "hello how are you"
-  print(len(my_string))
-
-
-  My code above does not work. Can you help me?

configs/eval/generate_large_3.yaml

@@ -1,15 +0,0 @@
-# model/tokenizer
-model_name: # update
-tokenizer_name: # update
-lora: true
-lora_path: # update
-
-max_new_tokens: 512
-temperature: 0.001
-prompt: | 
-  #this code prints a string reversed
-  my_string = "hello how are you"
-  print(len(my_string))
-
-
-  My code above does not work. Can you help me?

eval_figures.py

@@ -5,8 +5,8 @@ from matplotlib import pyplot as plt
 
 plt.figure()
 for fpath in glob.glob('./eval_data/*.pkl'):
-    parts = fpath.split('__')
-    model_name = parts[1].replace('model-', '').replace('.pkl', '')
+    parts = fpath.split('_')
+    model_name = "-".join(fpath.replace(".pkl", "").split("_")[2:])
     lora_name = parts[2].replace('lora-', '').replace('.pkl', '')
     with open(fpath, 'rb') as f:
         data = pickle.load(f)
@@ -14,7 +14,7 @@ for fpath in glob.glob('./eval_data/*.pkl'):
         perplexities = np.nan_to_num(perplexities, 100)
         perplexities = np.clip(perplexities, 0, 100)
         if 'nomic' in fpath:
-            label = 'GPT4all-lora'
+            label = f'GPT4all-{model_name}'
         else:
             label = 'alpaca-lora'
         plt.hist(perplexities, label=label, alpha=.5)

requirements.txt

@@ -9,4 +9,5 @@ peft
 nodelist-inflator
 deepspeed
 sentencepiece
-jsonlines
+jsonlines
+matplotlib