fix: stop gap to remove unused colulmns

Correct link

.gitignore

@@ -164,4 +164,9 @@ cython_debug/
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+#.idea/
+
+
+# vs code
+.vscode
+*.bin

.gitmodules

@@ -1,6 +1,3 @@
-[submodule "transformers"]
-	path = transformers
-	url = https://github.com/huggingface/transformers.git
 [submodule "peft"]
 	path = peft
 	url = https://github.com/huggingface/peft.git

GPT-J_MAP.md

@@ -0,0 +1,17 @@
+# Inference on Training Data
+
+
+## Run Inference
+
+```bash
+torchrun --master_port=29085 --nproc-per-node 8 inference.py --config=configs/inference/gptj.yaml
+```
+
+
+## Visualizations
+
+```bash
+python build_map.py
+```
+ 
+will build a map in `Atlas`, one using the internal clustering algorithm provided by Nomic and one using the embeddings generated by the finetuned model.

LICENSE.txt

@@ -0,0 +1,19 @@
+Copyright (c) 2023 Nomic, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,13 +1,92 @@
 <h1 align="center">GPT4All</h1>
-<p align="center">Demo, data and code to train an assistant-style large language model with ~800k GPT-3.5-Turbo Generations based on LLaMa</p>
+<p align="center">Demo, data, and code to train open-source assistant-style large language model based on GPT-J and LLaMa</p>
+<p align="center">
+<a href="https://static.nomic.ai/gpt4all/2023_GPT4All-J_Technical_Report_2.pdf">:green_book: Technical Report 2: GPT4All-J </a>
+</p>
 
 <p align="center">
-<a href="https://s3.amazonaws.com/static.nomic.ai/gpt4all/2023_GPT4All_Technical_Report.pdf">:green_book: Technical Report</a>
+<a href="https://s3.amazonaws.com/static.nomic.ai/gpt4all/2023_GPT4All_Technical_Report.pdf">:green_book: Technical Report 1: GPT4All</a>
 </p>
+
 <p align="center">
-<a href="https://discord.gg/kvmy6dQB">Discord</a>
+<a href="https://github.com/nomic-ai/pyllamacpp">:snake: Official Python Bindings</a>
 </p>
 
+<p align="center">
+<a href="https://github.com/nomic-ai/gpt4all-ts">:computer: Official Typescript Bindings</a>
+</p>
+
+<p align="center">
+<a href="https://github.com/nomic-ai/gpt4all-ui">:speech_balloon: Official Web Chat Interface</a>
+</p>
+
+<p align="center">
+<a href="https://github.com/nomic-ai/gpt4all-chat">:speech_balloon: Official Chat Interface</a>
+</p>
+
+<p align="center">
+<a href="https://python.langchain.com/en/latest/modules/models/llms/integrations/gpt4all.html">🦜️🔗 Official Langchain Backend</a> 
+</p>
+
+
+<p align="center">
+<a href="https://discord.gg/mGZE39AS3e">Discord</a>
+</p>
+
+
+
+
+<p align="center">
+GPT4All is made possible by our compute partner <a href="https://www.paperspace.com/">Paperspace</a>.
+</p>
+
+
+
+## GPT4All-J: An Apache-2 Licensed GPT4All Model
+![gpt4all-j-demo](https://user-images.githubusercontent.com/13879686/231876409-e3de1934-93bb-4b4b-9013-b491a969ebbc.gif)
+
+Run on an M1 Mac (not sped up!)
+
+
+### GPT4All-J Chat UI Installers
+Installs a native chat-client with auto-update functionality that runs on your desktop with the GPT4All-J model baked into it.
+
+[Mac/OSX](https://gpt4all.io/installers/gpt4all-0.1.0-Darwin.dmg)
+
+[Windows](https://gpt4all.io/installers/gpt4all-0.1.0-win64.exe)
+
+[Ubuntu](https://gpt4all.io/installers/gpt4all-0.1.0-Linux.run)
+
+These files are not yet cert signed by Windows/Apple so you will see security warnings on initial installation. We did not want to delay release while waiting for their process to complete.
+
+Find the most up-to-date information on the [GPT4All Website](https://gpt4all.io/)
+
+### Raw Model
+[ggml Model Download Link](https://gpt4all.io/models/ggml-gpt4all-j.bin)
+
+Note this model is only compatible with the C++ bindings found [here](https://github.com/nomic-ai/gpt4all-chat). It will not work with any existing llama.cpp bindings as we had to do a large fork of llama.cpp. GPT4All will support the ecosystem around this new C++ backend going forward.
+
+Python bindings are imminent and will be integrated into this [repository](https://github.com/nomic-ai/pyllamacpp). Stay tuned on the [GPT4All discord](https://discord.gg/mGZE39AS3e) for updates.
+
+## Training GPT4All-J
+
+Please see [GPT4All-J Technical Report](https://static.nomic.ai/gpt4all/2023_GPT4All-J_Technical_Report_2.pdf) for details.
+
+### GPT4All-J Training Data
+
+- We are releasing the curated training data for anyone to replicate GPT4All-J here: [GPT4All-J Training Data](https://huggingface.co/datasets/nomic-ai/gpt4all-j-prompt-generations)
+   - [Atlas Map of Prompts](https://atlas.nomic.ai/map/gpt4all-j-prompts-curated)
+   - [Atlas Map of Responses](https://atlas.nomic.ai/map/gpt4all-j-response-curated)
+
+### GPT4All-J Training Instructions
+
+```bash
+accelerate launch --dynamo_backend=inductor --num_processes=8 --num_machines=1 --machine_rank=0 --deepspeed_multinode_launcher standard --mixed_precision=bf16  --use_deepspeed --deepspeed_config_file=configs/deepspeed/ds_config_gptj.json train.py --config configs/train/finetune_gptj.yaml
+```
+
+
+# Original GPT4All Model (based on GPL Licensed LLaMa)
+
 
 
 ![gpt4all-lora-demo](https://user-images.githubusercontent.com/13879686/228352356-de66ca7a-df70-474e-b929-2e3656165051.gif)
@@ -16,31 +95,115 @@ Run on M1 Mac (not sped up!)
 
 # Try it yourself
 
-Download the CPU quantized gpt4all model checkpoint: [gpt4all-lora-quantized.bin](https://the-eye.eu/public/AI/models/nomic-ai/gpt4all/gpt4all-lora-quantized.bin).
+Here's how to get started with the CPU quantized GPT4All model checkpoint:
 
+1. Download the `gpt4all-lora-quantized.bin` file from [Direct Link](https://the-eye.eu/public/AI/models/nomic-ai/gpt4all/gpt4all-lora-quantized.bin) or [[Torrent-Magnet]](https://tinyurl.com/gpt4all-lora-quantized).
+2. Clone this repository, navigate to `chat`, and place the downloaded file there.
+3. Run the appropriate command for your OS:
+   - M1 Mac/OSX: `cd chat;./gpt4all-lora-quantized-OSX-m1`
+   - Linux: `cd chat;./gpt4all-lora-quantized-linux-x86`
+   - Windows (PowerShell): `cd chat;./gpt4all-lora-quantized-win64.exe`
+   - Intel Mac/OSX: `cd chat;./gpt4all-lora-quantized-OSX-intel`
 
-Clone this repository down and place the quantized model in the `chat` directory and start chatting by running:
+For custom hardware compilation, see our [llama.cpp](https://github.com/zanussbaum/gpt4all.cpp) fork.
 
-- `cd chat;./gpt4all-lora-quantized-OSX-m1` on M1 Mac/OSX
-- `cd chat;./gpt4all-lora-quantized-linux-x86` on Linux
-- `cd chat;./gpt4all-lora-quantized-win64.exe` on Windows (PowerShell)
-- `cd chat;./gpt4all-lora-quantized-OSX-intel` on Intel Mac/OSX
+-----------
+Find all compatible models in the GPT4All Ecosystem section.
 
-To compile for custom hardware, see our fork of the [Alpaca C++](https://github.com/zanussbaum/gpt4all.cpp) repo.
+[Secret Unfiltered Checkpoint](https://the-eye.eu/public/AI/models/nomic-ai/gpt4all/gpt4all-lora-unfiltered-quantized.bin) - [[Torrent]](https://the-eye.eu/public/AI/models/nomic-ai/gpt4all/gpt4all-lora-unfiltered-quantized.bin.torrent)
 
+This model had all refusal to answer responses removed from training. Try it with:
+- M1 Mac/OSX: `cd chat;./gpt4all-lora-quantized-OSX-m1 -m gpt4all-lora-unfiltered-quantized.bin`
+- Linux: `cd chat;./gpt4all-lora-quantized-linux-x86 -m gpt4all-lora-unfiltered-quantized.bin`
+- Windows (PowerShell): `cd chat;./gpt4all-lora-quantized-win64.exe -m gpt4all-lora-unfiltered-quantized.bin`
+- Intel Mac/OSX: `cd chat;./gpt4all-lora-quantized-OSX-intel -m gpt4all-lora-unfiltered-quantized.bin`
+-----------
 Note: the full model on GPU (16GB of RAM required) performs much better in our qualitative evaluations.
 
+# Python Client
+## CPU Interface
+To run GPT4All in python, see the new [official Python bindings](https://github.com/nomic-ai/pyllamacpp).
+
+The old bindings are still available but now deprecated. They will not work in a notebook environment.
+To get running using the python client with the CPU interface, first install the [nomic client](https://github.com/nomic-ai/nomic) using `pip install nomic`
+Then, you can use the following script to interact with GPT4All:
+```
+from nomic.gpt4all import GPT4All
+m = GPT4All()
+m.open()
+m.prompt('write me a story about a lonely computer')
+```
+
+## GPU Interface
+There are two ways to get up and running with this model on GPU.
+The setup here is slightly more involved than the CPU model.
+1. clone the nomic client [repo](https://github.com/nomic-ai/nomic) and run `pip install .[GPT4All]` in the home dir.
+2. run `pip install nomic` and install the additional deps from the wheels built [here](https://github.com/nomic-ai/nomic/tree/main/bin)
+
+Once this is done, you can run the model on GPU with a script like the following:
+```
+from nomic.gpt4all import GPT4AllGPU
+m = GPT4AllGPU(LLAMA_PATH)
+config = {'num_beams': 2,
+          'min_new_tokens': 10,
+          'max_length': 100,
+          'repetition_penalty': 2.0}
+out = m.generate('write me a story about a lonely computer', config)
+print(out)
+```
+Where LLAMA_PATH is the path to a Huggingface Automodel compliant LLAMA model.
+Nomic is unable to distribute this file at this time.
+We are working on a GPT4All that does not have this limitation right now.
+
+You can pass any of the [huggingface generation config params](https://huggingface.co/docs/transformers/main_classes/text_generation#transformers.GenerationConfig) in the config.
+
+# GPT4All Compatibility Ecosystem
+Edge models in the GPT4All Ecosystem. Please PR as the [community grows](https://huggingface.co/models?sort=modified&search=4bit).
+Feel free to convert this to a more structured table.
+
+- [gpt4all](https://the-eye.eu/public/AI/models/nomic-ai/gpt4all/gpt4all-lora-quantized.bin) [[MD5 Signature](https://the-eye.eu/public/AI/models/nomic-ai/gpt4all/gpt4all-lora-quantized.bin.md5)]
+   - [gpt4all-ggml-converted](https://the-eye.eu/public/AI/models/nomic-ai/gpt4all/gpt4all-lora-quantized-ggml.bin) [[MD5 Signature](https://the-eye.eu/public/AI/models/nomic-ai/gpt4all/gpt4all-lora-quantized-ggml.bin.md5)]
+- [gpt4all-unfiltered](https://the-eye.eu/public/AI/models/nomic-ai/gpt4all/gpt4all-lora-unfiltered-quantized.bin) [[MD5 Signature](https://the-eye.eu/public/AI/models/nomic-ai/gpt4all/gpt4all-lora-unfiltered-quantized.bin.md5)]
+- [ggml-vicuna-7b-4bit](https://huggingface.co/eachadea/ggml-vicuna-7b-4bit)
+- [vicuna-13b-GPTQ-4bit-128g](https://huggingface.co/anon8231489123/vicuna-13b-GPTQ-4bit-128g)
+- [LLaMa-Storytelling-4Bit](https://huggingface.co/GamerUntouch/LLaMa-Storytelling-4Bit)
+- [Alpaca Native 4bit](https://huggingface.co/Sosaka/Alpaca-native-4bit-ggml/tree/main)
+
+
+# Roadmap
+## Short Term
+ - <span style="color:green">(Done)</span> Train a GPT4All model based on GPTJ to alleviate llama distribution issues.
+ - <span style="color:green">(Done)</span> Create improved CPU and GPU interfaces for this model.
+ - <span style="color:green">(Done)</span> [Integrate llama.cpp bindings](https://github.com/nomic-ai/pyllamacpp)
+ - <span style="color:green">(Done)</span> [Create a good conversational chat interface for the model.](https://github.com/nomic-ai/gpt4all-ui)
+ - <span style="color:green">(Done)</span> [Allow users to opt in and submit their chats for subsequent training runs](https://github.com/nomic-ai/gpt4all-ui)
+
+## Medium Term
+ - <span style="color:red">(NOT STARTED)</span> Integrate GPT4All with [Atlas](https://atlas.nomic.ai) to allow for document retrieval.
+   - BLOCKED by GPT4All based on GPTJ
+ - <span style="color:red">(Done)</span> Integrate GPT4All with Langchain.
+ - <span style="color:green">(IN PROGRESS)</span> Build easy custom training scripts to allow users to fine tune models.
+
+## Long Term
+ - <span style="color:red">(NOT STARTED)</span> Allow anyone to curate training data for subsequent GPT4All releases using Atlas.
+ - <span style="color:green">(IN PROGRESS)</span> Democratize AI. 
+
 # Reproducibility
 
-Trained LoRa Weights:
+Trained Model Weights:
 - gpt4all-lora (four full epochs of training):  https://huggingface.co/nomic-ai/gpt4all-lora
 - gpt4all-lora-epoch-2 (three full epochs of training) https://huggingface.co/nomic-ai/gpt4all-lora-epoch-2
+- gpt4all-j (one full epoch of training) (https://huggingface.co/nomic-ai/gpt4all-j)
+- gpt4all-j-lora (one full epoch of training) (https://huggingface.co/nomic-ai/gpt4all-j-lora)
 
 Raw Data:
-- [Training Data Without P3](https://s3.amazonaws.com/static.nomic.ai/gpt4all/2022_03_27/gpt4all_curated_data_without_p3_2022_03_27.tar.gz)
+- [Training Data Without P3](https://huggingface.co/datasets/nomic-ai/gpt4all_prompt_generations)
   - Explorer: https://atlas.nomic.ai/map/gpt4all_data_clean_without_p3
-- [Full Dataset with P3](https://s3.amazonaws.com/static.nomic.ai/gpt4all/2022_03_27/gpt4all_curated_data_full_2022_03_27.tar.gz)
+- [Full Dataset with P3](https://huggingface.co/datasets/nomic-ai/gpt4all_prompt_generations_with_p3)
   - Explorer: https://atlas.nomic.ai/map/gpt4all_data_clean
+- [GPT4All-J Dataset](https://huggingface.co/datasets/nomic-ai/gpt4all-j-prompt-generations)
+   -  Explorer Indexed on Prompts: https://atlas.nomic.ai/map/gpt4all-j-prompts-curated
+   -  Exporer Indexed on Responses: https://atlas.nomic.ai/map/gpt4all-j-response-curated
 
 We are not distributing a LLaMa 7B checkpoint.
 
@@ -50,18 +213,16 @@ You can reproduce our trained model by doing the following:
 
 Clone the repo
 
-`git clone --recurse-submodules https://github.com/nomic-ai/gpt4all.git`
-
-`git submodule configure && git submodule update`
+```
+git clone --recurse-submodules https://github.com/nomic-ai/gpt4all.git
+git submodule update --init
+```
 
 Setup the environment
 
 ```
 python -m pip install -r requirements.txt
 
-cd transformers
-pip install -e . 
-
 cd ../peft
 pip install -e .
 ```
@@ -78,6 +239,10 @@ accelerate launch --dynamo_backend=inductor --num_processes=8 --num_machines=1 -
 python generate.py --config configs/generate/generate.yaml --prompt "Write a script to reverse a string in Python"
 ```
 
+## Need Help?
+
+Join the <a href="https://discord.gg/E7q6fp9bmx"> Discord </a> and ask for help in `#gpt4all-help`
+
 # Sample Generations
 
 ### Provide instructions for the given exercise. Leg Raises
@@ -104,7 +269,8 @@ python generate.py --config configs/generate/generate.yaml --prompt "Write a scr
 
 ### Come up with an interesting idea for a new movie plot. Your plot should be described with a title and a summary.
 
->1. The title of your movie plot should be "The Last Stand". 2. The summary of your movie plot should be: A group of mercenaries are hired to protect a small town from a dangerous criminal who is on the run. However, the town's sheriff is not willing to let them take over and decides to take matters into his own hands.
+> 1. The title of your movie plot should be "The Last Stand". 
+> 2. The summary of your movie plot should be: A group of mercenaries are hired to protect a small town from a dangerous criminal who is on the run. However, the town's sheriff is not willing to let them take over and decides to take matters into his own hands.
 
 ### Reverse a string in python.
 > In Python, you can reverse a list or tuple by using the reversed() function on it. Here's an example of how to use this method with strings: 
@@ -114,42 +280,43 @@ python generate.py --config configs/generate/generate.yaml --prompt "Write a scr
 >   print(reversed_str)     # Output will be reversed version of the inputted string.
 
 ### List 10 dogs.
->1. Labrador Retriever
->2. Golden Retriever
->3. Beagle
->4. Poodle
->5. German Shepherd
->6. Bulldog
->7. Rottweiler
->8. Boxer
->9. Shih Tzu
->10. Pug
+> 1. Labrador Retriever
+> 2. Golden Retriever
+> 3. Beagle
+> 4. Poodle
+> 5. German Shepherd
+> 6. Bulldog
+> 7. Rottweiler
+> 8. Boxer
+> 9. Shih Tzu
+> 10. Pug
 
-### Write me a poem about the fall of Julius Ceasar into a ceasar salad in iambic pentameter. 
->The fall of Julius Ceasar into a ceasar salad
->In iambic pentameter,
->The mighty Roman emperor
->Fell into a salad of lettuce and croutons,
->His empire crumbling around him,
->As he was devoured by the hungry diners.
->The once mighty emperor
->Was now just a salad topping,
->His legacy reduced to a mere garnish,
->As he was consumed by the masses.
->The fall of Julius Ceasar into a ceasar salad,
->In iambic pentameter,
->Was a tragic end to a great man,
->Who had once ruled the world with an iron fist.
->But now, he was just a salad topping,
->His legacy reduced to a mere garnish,
->As he was consumed by the masses.
+### Write me a poem about the fall of Julius Caesar into a caesar salad in iambic pentameter. 
+
+> The fall of Julius Caesar into a caesar salad<br>
+> In iambic pentameter,<br>
+> The mighty Roman emperor<br>
+> Fell into a salad of lettuce and croutons,<br>
+> His empire crumbling around him,<br>
+> As he was devoured by the hungry diners.<br>
+> The once mighty emperor<br>
+> Was now just a salad topping,<br>
+> His legacy reduced to a mere garnish,<br>
+> As he was consumed by the masses.<br>
+> The fall of Julius Caesar into a caesar salad,<br>
+> In iambic pentameter,<br>
+> Was a tragic end to a great man,<br>
+> Who had once ruled the world with an iron fist.<br>
+> But now, he was just a salad topping,<br>
+> His legacy reduced to a mere garnish,<br>
+> As he was consumed by the masses.
     
 ### What is a three word topic describing the following keywords: baseball, football, soccer: 
->Sports, athletics, games
+> Sports, athletics, games
     
+## Citation
 
-
-If you utilize this reposistory, models or data in a downstream project, please consider citing it with:
+If you utilize this repository, models or data in a downstream project, please consider citing it with:
 ```
 @misc{gpt4all,
   author = {Yuvanesh Anand and Zach Nussbaum and Brandon Duderstadt and Benjamin Schmidt and Andriy Mulyar},
@@ -160,7 +327,3 @@ If you utilize this reposistory, models or data in a downstream project, please
   howpublished = {\url{https://github.com/nomic-ai/gpt4all}},
 }
 ```
-
-### Alternative Download Locations
-#### gpt4all-lora-quantized.bin Backup Torrent Link
-magnet:?xt=urn:btih:1F11A9691EE06C18F0040E359361DCA0479BCB5A&dn=gpt4all-lora-quantized.bin&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337%2Fannounce&tr=udp%3A%2F%2Fopentracker.i2p.rocks%3A6969%2Fannounce

TRAINING_LOG.md

@@ -23,7 +23,7 @@ We used the initial parameters:
 | Weight decay   | 0     |
 | Warmup Steps   | 100   |
 
-We randomly shuffle and set aside %5 of the data for validation.
+We randomly shuffle and set aside 5% of the data for validation.
 
 We had an initial bug in logging the training loss but we noticed a decrease in validation loss.
 
@@ -160,7 +160,7 @@ We realized that we had two bugs however:
 - We accidentally duplicated data and effectively trained for 2 epochs instead of 1
 - We added an eos token to every sequence, even those that we truncated (e.g. long code that exceeds the 1024).
 
-## Conditonal EOS and 1 Epoch
+## Conditional EOS and 1 Epoch
 
 Using the same parameters, we then trained a model using a "conditional" eos token where we only add an `eos` when the inputs are less than the maximum sequence length for one epoch.
 
@@ -235,3 +235,49 @@ Taking inspiration from [the Alpaca Repo](https://github.com/tatsu-lab/stanford_
 Comparing our model LoRa to the [Alpaca LoRa](https://huggingface.co/tloen/alpaca-lora-7b), our model has lower perplexity. Qualitatively, training on 3 epochs performed the best on perplexity as well as qualitative examples. 
 
 We tried training a full model using the parameters above, but found that during the second epoch the model diverged and samples generated post training were worse than the first epoch. 
+
+
+## GPT-J Training
+
+### Model Training Divergence
+
+We trained multiple [GPT-J models](https://huggingface.co/EleutherAI/gpt-j-6b) with varying success. We found that training the full model lead to diverged post epoch 1. ![](figs/overfit-gpt-j.png)
+
+
+We release the checkpoint after epoch 1.
+
+
+Using Atlas, we extracted the embeddings of each point in the dataset and calculated the loss per sequence. We then uploaded [this to Atlas](https://atlas.nomic.ai/map/gpt4all-j-post-epoch-1-embeddings) and noticed that the higher loss items seem to cluster. On further inspection, the highest density clusters seemded to be of prompt/response pairs that asked for creative-like generations such as `Generate a story about ...` ![](figs/clustering_overfit.png)
+
+
+
+### GPT4All-J Hyperparameters
+
+We varied learning rate, learning rate schedule, and weight decay following suggestions from the [original GPT-J codebase](https://github.com/kingoflolz/mesh-transformer-jax/blob/master/howto_finetune.md) but found no real performance difference (qualitatively or quantitatively) when varying these parameters.
+
+
+
+The final model was trained using the following hyperparameters with a linear warmup followed by constant learning rate:
+
+| Hyperparameter | Value |
+|----------------|-------|
+| Per Device BS  | 32    |
+| Global BS      | 256   |
+| Learning rate  | 2e-5  |
+| Epochs         | 2     |
+| Max length     | 1024  |
+| Weight decay   | 0     |
+| Warmup Steps   | 500   |
+
+
+The LoRA model was trained using using the following hyperparameters with a linear warmup followed by constant learning rate: 
+
+| Hyperparameter | Value |
+|----------------|-------|
+| Per Device BS  | 4     |
+| Global BS      | 32    |
+| Learning rate  | 2e-5  |
+| Epochs         | 2     |
+| Max length     | 1024  |
+| Weight decay   | 0     |
+| Warmup Steps   | 500   |

build_map.py

@@ -0,0 +1,54 @@
+import numpy as np
+from nomic import atlas
+import glob
+from tqdm import tqdm
+from datasets import load_dataset, concatenate_datasets
+from sklearn.decomposition import PCA
+
+files = glob.glob("inference/*.jsonl")
+print(files)
+df = concatenate_datasets([load_dataset("json", data_files=file, split="train") for file in tqdm(files)])
+
+print(len(df))
+print(df)
+
+df = df.map(lambda example: {"inputs": [prompt + "\n" + response for prompt, response in zip(example["prompt"], example["response"])]},
+            batched=True,
+            num_proc=64)
+
+df = df.map(lambda example: {"trained_on": [int(t) for t in example["is_train"]]},
+                batched=True,
+                num_proc=64)
+                
+df = df.remove_columns("is_train")
+
+text = df.remove_columns(["labels", "input_ids", "embeddings"])
+
+text_df = [text[i] for i in range(len(text))]
+
+atlas.map_text(text_df, indexed_field="inputs",
+               name="CHANGE ME!",
+               colorable_fields=["source", "loss", "trained_on"],
+               reset_project_if_exists=True,
+               )
+
+# index is local to train/test split, regenerate
+data = df.remove_columns(["labels", "input_ids", "index"])
+data = data.add_column("index", list(range(len(data))))
+# max embed dim is 2048 for now
+# note! this is slow in pyarrow/hf datasets
+embeddings = np.array(data["embeddings"])
+print("embeddings shape:", embeddings.shape)
+embeddings = PCA(n_components=2048).fit_transform(embeddings)
+
+data = data.remove_columns(["embeddings"])
+columns = data.to_pandas().to_dict("records") 
+
+atlas.map_embeddings(embeddings,
+                     data=columns,
+                     id_field="index",
+                     name="CHANGE ME!",
+                     colorable_fields=["source", "loss", "trained_on"],
+                     build_topic_model=True,
+                     topic_label_field="inputs",
+                     reset_project_if_exists=True,)

configs/deepspeed/ds_config_gptj.json

@@ -24,5 +24,25 @@
 	  "allgather_partitions": true,
 	  "allgather_bucket_size": 5e8,
 	  "contiguous_gradients": true
-	}
-  }
+	},
+	"optimizer": {
+		"type": "AdamW",
+		"params": {
+		  "lr": "auto",
+		  "betas": [
+			0.9,
+			0.999
+		  ],
+		  "eps": 1e-08
+		}
+	  },
+	  "scheduler": {
+		"type": "WarmupLR",
+		"params": {
+		  "warmup_min_lr": 0,
+		  "warmup_max_lr": "auto",
+		  "warmup_num_steps": "auto",
+		  "warmup_type": "linear"
+		}
+	  }
+}

configs/deepspeed/ds_config_gptj_lora.json

@@ -0,0 +1,48 @@
+{
+	"train_batch_size": "auto",
+	"gradient_accumulation_steps": "auto",
+	"train_micro_batch_size_per_gpu": "auto",
+	"fp16": {
+	  "enabled": "auto",
+	  "min_loss_scale": 1,
+	  "loss_scale_window": 1000,
+	  "hysteresis": 2,
+	  "initial_scale_power": 32
+	},
+	"bf16": {
+		"enabled": "auto"
+	},
+	"gradient_clipping": 1,
+	"zero_optimization": {
+	  "stage": 2,
+	  "offload_param": {
+		"device": "cpu"
+	  },
+	  "offload_optimizer": {
+		"device": "cpu"
+	  },
+	  "allgather_partitions": true,
+	  "allgather_bucket_size": 5e8,
+	  "contiguous_gradients": true
+	},
+	"optimizer": {
+	  "type": "AdamW",
+	  "params": {
+		"lr": "auto",
+		"betas": [
+		  0.9,
+		  0.999
+		],
+		"eps": 1e-08
+	  }
+	},
+	"scheduler": {
+	  "type": "WarmupLR",
+	  "params": {
+		"warmup_min_lr": 0,
+		"warmup_max_lr": "auto",
+		"warmup_num_steps": "auto",
+		"warmup_type": "linear"
+	  }
+	}
+  }

configs/eval/generate.yaml

@@ -1,15 +0,0 @@
-# model/tokenizer
-model_name: # update with llama 7b 
-tokenizer_name: # update with llama 7b
-lora: true
-lora_path: "nomic-ai/gpt4all-lora"
-
-max_new_tokens: 512
-temperature: 0.001
-prompt: | 
-  #this code prints a string reversed
-  my_string = "hello how are you"
-  print(len(my_string))
-
-
-  My code above does not work. Can you help me?

configs/eval/generate_baseline.yaml

@@ -1,17 +1,5 @@
 # model/tokenizer
-model_name: # update with llama model name
-tokenizer_name: # update with llama model name
+model_name: "zpn/llama-7b"
+tokenizer_name: "zpn/llama-7b"
 lora: true
-lora_path: "tloen/alpaca-lora-7b"
-
-
-
-max_new_tokens: 512
-temperature: 0.001
-prompt: | 
-  #this code prints a string reversed
-  my_string = "hello how are you"
-  print(len(my_string))
-
-
-  My code above does not work. Can you help me?
+lora_path: "tloen/alpaca-lora-7b"

configs/eval/generate_gpt4all_gptj.yaml

@@ -0,0 +1,4 @@
+# model/tokenizer
+model_name: "nomic-ai/gpt4all-warmup-lr-epoch_0"
+tokenizer_name: "EleutherAI/gpt-j-6b"
+lora: false

configs/eval/generate_gpt4all_gptj_lora.yaml

@@ -0,0 +1,5 @@
+# model/tokenizer
+model_name: "EleutherAI/gpt-j-6b"
+tokenizer_name: "EleutherAI/gpt-j-6B"
+lora: true
+lora_path: "nomic-ai/gpt4all-gptj-lora-epoch_1"

configs/eval/generate_gpt4all_llama_lora.yaml

@@ -0,0 +1,5 @@
+# model/tokenizer
+model_name: "zpn/llama-7b"
+tokenizer_name: "zpn/llama-7b"
+lora: true
+lora_path: "nomic-ai/gpt4all-lora"

configs/eval/generate_large_2.yaml

@@ -1,15 +0,0 @@
-# model/tokenizer
-model_name: # update
-tokenizer_name: # update
-lora: true
-lora_path: # update
-
-max_new_tokens: 512
-temperature: 0.001
-prompt: | 
-  #this code prints a string reversed
-  my_string = "hello how are you"
-  print(len(my_string))
-
-
-  My code above does not work. Can you help me?

configs/eval/generate_large_3.yaml

@@ -1,15 +0,0 @@
-# model/tokenizer
-model_name: # update
-tokenizer_name: # update
-lora: true
-lora_path: # update
-
-max_new_tokens: 512
-temperature: 0.001
-prompt: | 
-  #this code prints a string reversed
-  my_string = "hello how are you"
-  print(len(my_string))
-
-
-  My code above does not work. Can you help me?

configs/generate/generate.yaml

@@ -1,6 +1,6 @@
 # model/tokenizer
-model_name: # REPLACE HERE with the base llama model 
-tokenizer_name: # REPLACE HERE with the llama tokenizer
+model_name: "zpn/llama-7b"
+tokenizer_name: "zpn/llama-7b"
 lora: true
 lora_path: "nomic-ai/gpt4all-lora"
 

configs/generate/generate_gptj.yaml

@@ -1,7 +1,8 @@
 # model/tokenizer
-model_name: # update
-tokenizer_name: # update
-lora_path: "no-lora"
+model_name: "nomic-ai/gpt4all-warmup-lr-epoch_1"
+tokenizer_name: "EleutherAI/gpt-j-6b"
+lora: false
+
 
 max_new_tokens: 512
 temperature: 0.001

configs/generate/generate_gptj_lora.yaml

@@ -0,0 +1,15 @@
+# model/tokenizer
+model_name: "EleutherAI/gpt-j-6b"
+tokenizer_name: "EleutherAI/gpt-j-6b"
+lora: true
+lora_path: "nomic-ai/gpt4all-gptj-lora-epoch_0"
+
+max_new_tokens: 512
+temperature: 0
+prompt: | 
+  #this code prints a string reversed
+  my_string = "hello how are you"
+  print(len(my_string))
+
+
+  My code above does not work. Can you help me?

configs/inference/gptj.yaml

@@ -1,11 +1,11 @@
 # model/tokenizer
-model_name: "nomic-ai/gpt4all-gptj-multinode-deepspeed-finetuned-epoch_0"
+model_name: "nomic-ai/gpt4all-warmup-lr-epoch_1"
 tokenizer_name: "EleutherAI/gpt-j-6B"
 
 # dataset
 streaming: false
 num_proc: 64
-dataset_path: "data_multiplus" 
+dataset_path: "nomic-ai/turbo-500k-multi" 
 max_length: 1024
 batch_size: 32 
 

configs/train/finetune.yaml

@@ -2,12 +2,13 @@
 model_name: # add model here
 tokenizer_name: # add model here
 gradient_checkpointing: true
-save_name: "nomic-ai/gpt4all-full-multi-turn"
+save_name: # CHANGE 
 
 # dataset
 streaming: false
 num_proc: 64
 dataset_path: # update
+revision: null
 max_length: 1024
 batch_size: 32
 
@@ -16,7 +17,7 @@ lr: 5.0e-5
 eval_every: 800
 eval_steps: 100
 save_every: 800
-output_dir: "ckpts/gpt4all-full-multi"
+output_dir: # CHANGE
 checkpoint: null
 lora: false
 warmup_steps: 100

configs/train/finetune_gptj.yaml

@@ -2,14 +2,15 @@
 model_name: "EleutherAI/gpt-j-6B"
 tokenizer_name: "EleutherAI/gpt-j-6B"
 gradient_checkpointing: true
-save_name: "nomic-ai/gpt4all-mosaic"
+save_name: # CHANGE
 
 # dataset
 streaming: false
 num_proc: 64
-dataset_path: "nomic-ai/turbo-500k-multi"
+dataset_path: # CHANGE
+revision: null
 max_length: 1024
-batch_size: 8
+batch_size: 32
 
 # train dynamics
 lr: 2.0e-5
@@ -18,8 +19,8 @@ weight_decay: 0.0
 eval_every: 500
 eval_steps: 105
 save_every: 500
-log_grads_every: 500
-output_dir: "ckpts/gpt4all-gptj-multinode"
+log_grads_every: 100
+output_dir: # CHANGE
 checkpoint: null
 lora: false
 warmup_steps: 500
@@ -27,7 +28,7 @@ num_epochs: 2
 
 # logging
 wandb: true
-wandb_entity: vicuna
-wandb_project_name: vicuna
+wandb_entity: # CHANGE
+wandb_project_name: # CHANGE
 seed: 42
 

configs/train/finetune_gptj_lora.yaml

@@ -2,14 +2,15 @@
 model_name: "EleutherAI/gpt-j-6b"
 tokenizer_name: "EleutherAI/gpt-j-6b"
 gradient_checkpointing: false
-save_name: "nomic-ai/gpt4all-mosaic"
+save_name: # CHANGE
 
 # dataset
 streaming: false
 num_proc: 64
-dataset_path: "nomic-ai/turbo-500k-multi"
+dataset_path: # CHANGE
+revision: null
 max_length: 1024
-batch_size: 4
+batch_size: 1 
 
 # train dynamics
 lr: 2.0e-5
@@ -19,7 +20,7 @@ eval_every: 500
 eval_steps: 105
 save_every: 500
 log_grads_every: 500
-output_dir: "ckpts/gpt4all-gptj-multinode"
+output_dir: # CHANGE
 checkpoint: null
 lora: true
 warmup_steps: 500
@@ -27,7 +28,7 @@ num_epochs: 2
 
 # logging
 wandb: true
-wandb_entity: zanussbaum
-wandb_project_name: mosaic
+wandb_entity: # CHANGE
+wandb_project_name: # CHANGE
 seed: 42
 

configs/train/finetune_lora.yaml

@@ -2,12 +2,13 @@
 model_name: # update
 tokenizer_name: # update
 gradient_checkpointing: false
-save_name: "nomic-ai/gpt4all-lora-multi-turn"
+save_name: # CHANGE
 
 # dataset
 streaming: false
 num_proc: 64
-dataset_path: "nomic-ai/turbo-500k-multi"
+dataset_path: # CHANGE
+revision: null
 max_length: 1024
 batch_size: 4
 
@@ -18,7 +19,7 @@ weight_decay: 0.0
 eval_every: 2000
 eval_steps: 100
 save_every: 2000
-output_dir: "ckpts/gpt4all-lora-multi"
+output_dir: # CHANGE
 checkpoint: null
 lora: true
 warmup_steps: 100

data.py

@@ -15,8 +15,8 @@ def tokenize_inputs(config, tokenizer, examples):
     out = {"labels": [], "input_ids": []}
     for prompt, response in zip(examples["prompt"], examples["response"]):
         if different_eos:
-            if response.count("</s>") > 0:
-                response = response.replace("</s>", tokenizer.eos_token)
+            if response.count("</s> \n") > 0:
+                response = response.replace("</s> \n", f"{tokenizer.eos_token} \n") 
 
         prompt_len = len(tokenizer(prompt + "\n", return_tensors="pt")["input_ids"][0])
 
@@ -61,8 +61,24 @@ def tokenize_inputs(config, tokenizer, examples):
 def load_data(config, tokenizer):
     dataset_path = config["dataset_path"]
 
-    if os.path.exists(dataset_path):
-        # check if path is a directory
+    if isinstance(dataset_path, list):
+        all_datasets = []
+        for path in dataset_path:
+            dataset = load_dataset(path, split="train")
+
+            current_columns = dataset.column_names
+            columns_to_keep = ["prompt", "response"]
+            to_remove = set(current_columns) - set(columns_to_keep)
+
+            dataset = dataset.remove_columns(to_remove)
+            if "source" not in current_columns:
+                dataset = dataset.add_column("source", [path.split("/")[-1]] * len(dataset))
+            all_datasets.append(dataset)
+
+        dataset = concatenate_datasets(all_datasets)
+
+    # load local json dataset
+    elif os.path.exists(dataset_path):
         if os.path.isdir(dataset_path):
             files = glob.glob(os.path.join(dataset_path, "*_clean.jsonl"))
         else:
@@ -71,9 +87,11 @@ def load_data(config, tokenizer):
         print(f"Reading files {files}")
 
         dataset = load_dataset("json", data_files=files, split="train")
-
+    
+    # read from huggingface
     else:
-        dataset = load_dataset(dataset_path, split="train")
+        revision = config["revision"] 
+        dataset = load_dataset(dataset_path, split="train", revision=revision)
 
     dataset = dataset.train_test_split(test_size=.05, seed=config["seed"])
 
@@ -88,13 +106,13 @@ def load_data(config, tokenizer):
     train_dataset = train_dataset.map(
         lambda ele: tokenize_inputs(config, tokenizer, ele),
         batched=True,
-        remove_columns=["source", "prompt"],
+        remove_columns=["source", "prompt", "id", "response"],
         **kwargs
     )
     val_dataset = val_dataset.map(
-        lambda ele: tokenize_inputs(config, tokenizer, ele), 
+        lambda ele: tokenize_inputs(config, tokenizer, ele),
         batched=True,
-        remove_columns=["source", "prompt"],
+        remove_columns=["source", "prompt", "id", "response"],
         **kwargs
     )
 

eval_figures.py

@@ -6,18 +6,20 @@ from matplotlib import pyplot as plt
 plt.figure()
 for fpath in glob.glob('./eval_data/*.pkl'):
     parts = fpath.split('__')
-    model_name = parts[1].replace('model-', '').replace('.pkl', '')
-    lora_name = parts[2].replace('lora-', '').replace('.pkl', '')
+    model_name = "-".join(fpath.replace(".pkl", "").split("_")[2:])
     with open(fpath, 'rb') as f:
         data = pickle.load(f)
         perplexities = data['perplexities']
         perplexities = np.nan_to_num(perplexities, 100)
         perplexities = np.clip(perplexities, 0, 100)
-        if 'nomic' in fpath:
-            label = 'GPT4all-lora'
+        if 'alpaca' not in fpath:
+            identifier = model_name = "-".join(fpath.replace(".pkl", "").split("eval__model-")[1:]) 
+            label = 'GPT4all-'
+            label += identifier
+            
         else:
             label = 'alpaca-lora'
-        plt.hist(perplexities, label=label, alpha=.5)
+        plt.hist(perplexities, label=label, alpha=.5, bins=50)
 
 plt.xlabel('Perplexity')
 plt.ylabel('Frequency')

eval_self_instruct.py

@@ -49,28 +49,6 @@ def eval_example(model, tokenizer, example, config):
     input = tokenizer(prompt, return_tensors="pt")
     input = {k: v.to(model.device) for k, v in input.items()}
 
-    continuations = []
-    tokenized_continuations = []
-    trajectories = []
-    for i in range(1):
-        with torch.no_grad():
-            outputs = model.generate(input_ids=input['input_ids'],
-                                     max_new_tokens=config["max_new_tokens"],
-                                     min_new_tokens=5,
-                                     temperature=config["temperature"],
-                                     repetition_penalty=1.0,
-                                     do_sample=True)
-            decoded = tokenizer.decode(outputs[0], skip_special_tokens=True).strip()
-
-            y = model(input_ids=outputs)
-        trajectory = y.hidden_states[0].detach().cpu().numpy()[0]
-        trajectory = trajectory / np.linalg.norm(trajectory, axis=1, keepdims=True)
-        trajectory = np.cumsum(trajectory, axis=0) / np.arange(1, trajectory.shape[0]+1).reshape(-1, 1)
-
-        trajectories.append(trajectory)
-        continuations.append(decoded)
-        tokenized_continuations.append(tokenizer.tokenize(decoded))
-
     #compute the ground truth perplexity
     gt_input = tokenizer(gt, return_tensors="pt")
     gt_input = {k: v.to(model.device) for k, v in gt_input.items()}
@@ -101,30 +79,23 @@ def eval_example(model, tokenizer, example, config):
 
     print(prompt)
     print(80*'-')
-    for continuation in continuations:
-        print(continuation)
-        print(80*'-')
+   
 
-    return ppl, trajectories, continuations, tokenized_continuations
+    return ppl
 
 def do_eval(config):
     eval_data = read_jsonl_file('eval_data/user_oriented_instructions.jsonl')
     model, tokenizer = setup_model(config)
-    all_trajectories = []
     all_perplexities = []
-    all_continuations = []
-    all_tokenized_continuations = []
     for example in tqdm(eval_data):
-        gt_perplexity, trajectories, continuations, tokenized_continuations = eval_example(model, tokenizer, example, config)
-        all_trajectories.append(trajectories)
+        gt_perplexity = eval_example(model, tokenizer, example, config)
         all_perplexities.append(gt_perplexity)
-        all_continuations.append(continuations)
 
-    with open('eval_data/eval__model-{}__lora-{}.pkl'.format(config['model_name'].replace('/', '_'), config['lora_path'].replace('/', '_')), 'wb') as f:
-        r = {'trajectories': all_trajectories,
-             'perplexities': all_perplexities,
-             'continuations': all_continuations,
-             'tokenized_continuations': all_tokenized_continuations}
+        
+    name = f"eval_data/eval__model-{config['model_name'].replace('/', '_')}{'__lora-' + config['lora_path'].replace('/', '_') if config['lora'] else ''}.pkl"
+
+    with open(name, 'wb') as f:
+        r = {'perplexities': all_perplexities}
         pickle.dump(r, f)
 
 

head_node_setup.sh

@@ -1,19 +0,0 @@
-#!/bin/sh
-WORKER_IP=$1
-N_GPUS=$2
-
-sudo apt install -y nfs-kernel-server
-sudo mkdir -p ./data_multiplus
-sudo chmod 777 ./data_multiplus
-printf "${PWD}/data_multiplus ${WORKER_IP}(rw,sync,no_subtree_check)" | sudo tee -a /etc/exports
-sudo systemctl restart nfs-kernel-server
-
-sudo apt-get install -y pdsh
-export DSHPATH=$PATH
-export PDSH_RCMD_TYPE=ssh
-
-ssh-keygen -t rsa -N ''
-cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
-
-sudo mkdir -p /job
-printf "localhost slots=$N_GPUS\n$WORKER_IP slots=$N_GPUS" | sudo tee /job/hostfile

inference.py

@@ -115,7 +115,6 @@ def inference(config):
         train_outputs["embeddings"] = np.concatenate(train_outputs["embeddings"])
 
         df_train = Dataset.from_dict(train_outputs)
-        df_train = df_train.sort("index")
         curr_idx = df_train["index"]
 
         # compute mask in pyarrow since it's super fast
@@ -136,11 +135,11 @@ def inference(config):
         for batch in tqdm(val_dataloader, disable=local_rank != 0):
             batch["input_ids"] = batch["input_ids"].to(f"cuda:{local_rank}")
             batch["labels"] = batch["labels"].to(f"cuda:{local_rank}")
-            outputs = model(input_ids=batch["input_ids"], labels=batch["labels"])
+            outputs = model(input_ids=batch["input_ids"], labels=batch["labels"], output_hidden_states=True)
             loss = calc_cross_entropy_no_reduction(outputs.logits, batch["labels"])
             val_outputs["loss"].extend(loss)
 
-            logits = outputs.logits
+            embeddings = outputs.hidden_states[-1]
             batch_size = batch["input_ids"].shape[0]
             sequence_lengths = []
             # since we use mutiturn with multiple <|endoftext|>, we need to find the place where 
@@ -149,17 +148,17 @@ def inference(config):
                 indices = torch.where(item == tokenizer.pad_token_id)[0]
                 found = False
                 for index in indices:
+                    # case where sequence is less than max length
                     if torch.all(item[index:] == tokenizer.pad_token_id):
                         sequence_lengths.append(index)
                         found = True
                         break
-
-                # no match found
+                # case where sequence is >= max length
                 if not found:
                     sequence_lengths.append(len(item) - 1)
 
             sequence_lengths = torch.tensor(sequence_lengths)
-            pooled_logits = logits[torch.arange(batch_size, device=logits.device), sequence_lengths]
+            pooled_logits = embeddings[torch.arange(batch_size, device=embeddings.device), sequence_lengths]
 
             val_outputs["embeddings"].append(pooled_logits)
             val_outputs["index"].extend(batch["index"].to(model.device))
@@ -172,7 +171,6 @@ def inference(config):
         val_outputs["embeddings"] = np.concatenate(val_outputs["embeddings"])
 
         df_val = Dataset.from_dict(val_outputs)
-        df_val = df_val.sort("index")
         curr_idx = df_val["index"]
 
         # compute mask in pyarrow since it's super fast
@@ -182,7 +180,6 @@ def inference(config):
         filtered_table = table.filter(mask)
         # convert from pyarrow to Dataset
         filtered_val = Dataset.from_dict(filtered_table.to_pydict())
-
         filtered_val = filtered_val.add_column("embeddings", df_val["embeddings"])
         filtered_val = filtered_val.add_column("loss", df_val["loss"])
         filtered_val = filtered_val.add_column("is_train", [False] * len(filtered_val))

launcher.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+
+# Display header
+echo "=========================================================="
+echo " ██████  ██████  ████████ ██   ██  █████  ██      ██      "
+echo "██       ██   ██    ██    ██   ██ ██   ██ ██      ██      "
+echo "██   ███ ██████     ██    ███████ ███████ ██      ██      "
+echo "██    ██ ██         ██         ██ ██   ██ ██      ██      "
+echo " ██████  ██         ██         ██ ██   ██ ███████ ███████ "
+echo " └─> https://github.com/nomic-ai/gpt4all"
+
+# Function to detect macOS architecture and set the binary filename
+detect_mac_arch() {
+  local mac_arch
+  mac_arch=$(uname -m)
+  case "$mac_arch" in
+    arm64)
+      os_type="M1 Mac/OSX"
+      binary_filename="gpt4all-lora-quantized-OSX-m1"
+      ;;
+    x86_64)
+      os_type="Intel Mac/OSX"
+      binary_filename="gpt4all-lora-quantized-OSX-intel"
+      ;;
+    *)
+      echo "Unknown macOS architecture"
+      exit 1
+      ;;
+  esac
+}
+
+# Detect operating system and set the binary filename
+case "$(uname -s)" in
+  Darwin*)
+    detect_mac_arch
+    ;;
+  Linux*)
+    if grep -q Microsoft /proc/version; then
+      os_type="Windows (WSL)"
+      binary_filename="gpt4all-lora-quantized-win64.exe"
+    else
+      os_type="Linux"
+      binary_filename="gpt4all-lora-quantized-linux-x86"
+    fi
+    ;;
+  CYGWIN*|MINGW32*|MSYS*|MINGW*)
+    os_type="Windows (Cygwin/MSYS/MINGW)"
+    binary_filename="gpt4all-lora-quantized-win64.exe"
+    ;;
+  *)
+    echo "Unknown operating system"
+    exit 1
+    ;;
+esac
+echo "================================"
+echo "== You are using $os_type."
+
+
+# Change to the chat directory
+cd chat
+
+# List .bin files and prompt user to select one
+bin_files=(*.bin)
+echo "== Available .bin files:"
+for i in "${!bin_files[@]}"; do
+  echo "   [$((i+1))] ${bin_files[i]}"
+done
+
+# Function to get user input and validate it
+get_valid_user_input() {
+  local input_valid=false
+
+  while ! $input_valid; do
+    echo "==> Please enter a number:"
+    read -r user_selection
+    if [[ $user_selection =~ ^[0-9]+$ ]] && (( user_selection >= 1 && user_selection <= ${#bin_files[@]} )); then
+      input_valid=true
+    else
+      echo "Invalid input. Please enter a number between 1 and ${#bin_files[@]}."
+    fi
+  done
+}
+
+get_valid_user_input
+selected_bin_file="${bin_files[$((user_selection-1))]}"
+
+# Run the selected .bin file with the appropriate command
+./"$binary_filename" -m "$selected_bin_file"

requirements.txt

@@ -2,7 +2,7 @@ accelerate
 datasets
 torchmetrics
 evaluate
-transformers
+transformers>=4.28.0
 wandb
 pip
 peft
@@ -10,4 +10,6 @@ nodelist-inflator
 deepspeed
 sentencepiece
 jsonlines
-nomic
+nomic
+scikit-learn
+matplotlib

train.py

@@ -100,7 +100,7 @@ def train(accelerator, config):
             name="cosine",
             optimizer=optimizer,
             num_warmup_steps=config["warmup_steps"] * accelerator.num_processes,
-            num_training_steps=total_num_steps * accelerator.num_processes,
+            num_training_steps=total_num_steps,
         )
     else:
         scheduler = DummyScheduler(
@@ -192,7 +192,7 @@ def train(accelerator, config):
             accelerator.print(f"Failed to push to hub")
 
         unwrapped_model.save_pretrained(
-            f"{config['output_dir']}/-epoch_{epoch}",
+            f"{config['output_dir']}/epoch_{epoch}",
             is_main_process=accelerator.is_main_process,
             save_function=accelerator.save,
             state_dict=accelerator.get_state_dict(model),

worker_node_setup.sh

@@ -1,6 +0,0 @@
-#!/bin/sh
-HEAD_IP=$1
-
-sudo apt install -y nfs-common
-sudo mkdir -p ./data_multiplus
-sudo mount ${HEAD_IP}:${PWD}/data_multiplus ./data_multiplus