From c7f4ca25e4af7263468d8e75cdbae913610f48ac Mon Sep 17 00:00:00 2001
From: Zach Nussbaum
Date: Tue, 4 Apr 2023 20:46:57 +0000
Subject: [PATCH 01/63] chore: gitignore ckpts
---
.gitignore | 1 +
1 file changed, 1 insertion(+)
diff --git a/.gitignore b/.gitignore
index 8addd972..bbd0f77f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
+ckpts*
*.jsonl
*tar.gz
ckpts**
From f45eb001a16641f7e7ae52d662d049753c282da4 Mon Sep 17 00:00:00 2001
From: Zach Nussbaum
Date: Tue, 4 Apr 2023 20:47:21 +0000
Subject: [PATCH 02/63] fix: clean where prompt is randomly 1 char
---
clean.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/clean.py b/clean.py
index 4712820b..6d1cec81 100644
--- a/clean.py
+++ b/clean.py
@@ -64,6 +64,7 @@ for file in glob.glob(os.path.join(prompt_generation_dir, "*.jsonl")):
df = df.dropna(subset=['prompt', 'response'])
df = df[df['prompt'] != '']
df = df[df['response'] != '']
+ df = df[df["prompt"].str.len() > 1]
curr_len = len(df)
print(f"Removed {prev_len - curr_len} rows")
From c68311810a0e53b9130bc25131d9e480db07bec6 Mon Sep 17 00:00:00 2001
From: Zach Nussbaum
Date: Tue, 4 Apr 2023 20:53:23 +0000
Subject: [PATCH 03/63] fix: clean up data, pad at end
---
configs/train/finetune_gptj.yaml | 33 ++++++++++++++++++++
data.py | 52 +++++++++++++++-----------------
2 files changed, 57 insertions(+), 28 deletions(-)
create mode 100644 configs/train/finetune_gptj.yaml
diff --git a/configs/train/finetune_gptj.yaml b/configs/train/finetune_gptj.yaml
new file mode 100644
index 00000000..31c25050
--- /dev/null
+++ b/configs/train/finetune_gptj.yaml
@@ -0,0 +1,33 @@
+# model/tokenizer
+model_name: "EleutherAI/gpt-j-6B"
+tokenizer_name: "EleutherAI/gpt-j-6B"
+gradient_checkpointing: true
+save_name: "nomic-ai/gpt4all-gptj-multiturn-lr-aggressive"
+
+# dataset
+streaming: false
+num_proc: 64
+dataset_path: "data_multiplus"
+max_length: 1024
+batch_size: 8
+
+# train dynamics
+lr: 2.0e-5
+min_lr: 0
+weight_decay: 0.0
+eval_every: 200
+eval_steps: 105
+save_every: 400
+log_grads_every: 200
+output_dir: "ckpts/gpt4all-gptj-full-multiturn-lr-aggreive"
+checkpoint: null
+lora: false
+warmup_steps: 500
+num_epochs: 4
+
+# logging
+wandb: true
+wandb_entity: vicuna
+wandb_project_name: vicuna
+seed: 42
+
diff --git a/data.py b/data.py
index 0e356f7d..0de26cfa 100644
--- a/data.py
+++ b/data.py
@@ -11,42 +11,38 @@ def tokenize_inputs(config, tokenizer, examples):
max_length = config["max_length"]
input_ids = torch.full((len(examples["prompt"]), max_length), tokenizer.pad_token_id)
# ignore bos
- newline_tokens = tokenizer("\n", return_tensors="pt")["input_ids"][0, 1:]
+ newline_tokens = tokenizer("\n", return_tensors="pt")["input_ids"][0]
+ if newline_tokens[0] == tokenizer.bos_token_id:
+ newline_tokens = newline_tokens[1:]
- out = {"labels": [], "attention_mask": []}
- for i, (prompt, response) in enumerate(zip(examples["prompt"], examples["response"])):
- input_tokens = tokenizer(prompt, truncation=True, max_length=max_length // 2, return_tensors="pt")["input_ids"].squeeze()
- input_len = len(input_tokens)
+ # hacky backward compatible
+ different_eos = tokenizer.eos_token != ""
+ out = {"labels": [], "input_ids": []}
+ for prompt, response in zip(examples["prompt"], examples["response"]):
+ if different_eos:
+ if response.count("") > 0:
+ response = response.replace("", tokenizer.eos_token)
- # plus one since we remove bos from response
- # but we subtract one since we want to add eos token
- remaining_tokens = max_length - input_len - len(newline_tokens) + 1
- # remove bos
- target_tokens = tokenizer(response, truncation=True, max_length=remaining_tokens, return_tensors="pt")["input_ids"].squeeze()[1:]
+ prompt_len = len(tokenizer(prompt, truncation=True, return_tensors="pt")["input_ids"][0])
- input_ids[i, :input_len] = input_tokens
- # add newline between prompt and response
- newline_plus_inputs = input_len + len(newline_tokens)
- input_ids[i, input_len: newline_plus_inputs] = newline_tokens
+ # hack if our prompt is super long
+ # we need to include some labels
+ if prompt_len >= max_length - 1:
+ prompt = prompt[:len(prompt) // 2]
- # add target tokens, remove bos
- input_ids[i, newline_plus_inputs: newline_plus_inputs + len(target_tokens)] = target_tokens
- # add eos token, enforce stopping if we don't truncate
- # we don't want long code to stop generating if truncated during training
- if newline_plus_inputs + len(target_tokens) < max_length:
- input_ids[i, newline_plus_inputs + len(target_tokens)] = tokenizer.eos_token_id
+ input_tokens = tokenizer(prompt + "\n" + response + tokenizer.eos_token,
+ truncation=True, max_length=max_length, return_tensors="pt")["input_ids"].squeeze()
- labels = input_ids[i].clone()
- labels[: newline_plus_inputs] = -100
- labels[labels == tokenizer.pad_token_id] = -100
- # to debug this, can set all values == -100 to the pad token, then assert that tokenizer.decode(labels, skip_special_tokens=True).strip() == response
- attention_mask = input_ids[i].ne(tokenizer.pad_token_id).int()
+ labels = input_tokens.clone()
+ labels[:prompt_len + len(newline_tokens)] = -100
+ if len(labels) < max_length:
+ # pad to max_length with -100
+ labels = torch.cat([labels, torch.full((max_length - len(labels),), -100)])
+ input_tokens = tokenizer.pad({"input_ids": input_tokens}, padding="max_length", max_length=max_length)["input_ids"]
out["labels"].append(labels)
- out["attention_mask"].append(attention_mask)
-
- out["input_ids"] = input_ids
+ out["input_ids"].append(input_tokens)
out = {k: torch.stack(v) if isinstance(v, list) else v for k, v in out.items()}
From 32357c920f2b972a34c6258f6d2d82c5100a369b Mon Sep 17 00:00:00 2001
From: Zach Nussbaum
Date: Tue, 4 Apr 2023 20:57:42 +0000
Subject: [PATCH 04/63] feat: adamw, fix training, log gradients
---
train.py | 84 ++++++++++++++++++++++++++++++++++++--------------------
1 file changed, 54 insertions(+), 30 deletions(-)
diff --git a/train.py b/train.py
index 4344ee24..6c2c0515 100644
--- a/train.py
+++ b/train.py
@@ -1,5 +1,5 @@
import os
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers import AutoModelForCausalLM, AutoTokenizer, AdamW, get_scheduler
from transformers.trainer_pt_utils import get_parameter_names
import torch
import torch.nn as nn
@@ -11,6 +11,7 @@ from peft import get_peft_model, LoraConfig, TaskType
from data import load_data
from torchmetrics import MeanMetric
from tqdm import tqdm
+import wandb
def format_metrics(metrics, split, prefix=""):
@@ -20,17 +21,12 @@ def format_metrics(metrics, split, prefix=""):
return log
-def evaluate(config, model, val_dataloader):
+def evaluate(model, val_dataloader):
model.eval()
val_loss = MeanMetric().to(model.device)
with torch.no_grad():
- for i, batch in enumerate(
- tqdm(val_dataloader),
- ):
- if i == config["eval_steps"]:
- break
-
+ for batch in tqdm(val_dataloader):
loss = model(**batch).loss
loss_values = accelerator.gather_for_metrics({"loss": loss.detach()})
@@ -49,8 +45,7 @@ def train(accelerator, config):
tokenizer = AutoTokenizer.from_pretrained(config['tokenizer_name'])
# llama has no pad token, set it to new token
if tokenizer.pad_token is None:
- # these tokens are already in the vocab, just not mapped correctly
- added_tokens = tokenizer.add_special_tokens({"bos_token": "", "eos_token": "", "pad_token": ""})
+ tokenizer.pad_token = tokenizer.eos_token
with accelerator.main_process_first():
@@ -61,10 +56,6 @@ def train(accelerator, config):
model = AutoModelForCausalLM.from_pretrained(config["model_name"],
use_cache=False if checkpoint else True,
trust_remote_code=True)
-
- if added_tokens > 0:
- model.resize_token_embeddings(len(tokenizer))
-
if checkpoint:
model.gradient_checkpointing_enable()
@@ -77,19 +68,55 @@ def train(accelerator, config):
model.print_trainable_parameters()
optimizer_cls = (
- torch.optim.AdamW
+ AdamW
if accelerator.state.deepspeed_plugin is None
or "optimizer" not in accelerator.state.deepspeed_plugin.deepspeed_config
else DummyOptim
)
+ no_decay = ["bias", "LayerNorm.weight"]
+ optimizer_grouped_parameters = [
+ {
+ "params": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],
+ "weight_decay": config["weight_decay"],
+ },
+ {
+ "params": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)],
+ "weight_decay": 0.0,
+ },
+ ]
+
# karpathy doesn't decay embeddding, maybe we should exclude
# https://github.com/karpathy/minGPT/commit/bbbdac74fa9b2e55574d70056163ffbae42310c1#diff-2075fa9c224b395be5bda85544dd36572b59c76c54562819eadadbf268602834R157s
- optimizer = optimizer_cls(model.parameters(), lr=config["lr"])
+ optimizer = optimizer_cls(optimizer_grouped_parameters, lr=config["lr"])
- # scheduler defined in Deepspeed config
- scheduler = DummyScheduler(
- optimizer, warmup_num_steps=config["warmup_steps"],
+ if accelerator.state.deepspeed_plugin is not None:
+ gradient_accumulation_steps = accelerator.state.deepspeed_plugin.deepspeed_config[
+ "gradient_accumulation_steps"
+ ]
+
+ # decay to min_lr instead of 0
+ lr_ratio = config["min_lr"] / config["lr"]
+ accelerator.print(f"Len of train_dataloader: {len(train_dataloader)}")
+ total_num_steps = (len(train_dataloader) / gradient_accumulation_steps) * config["num_epochs"]
+ # instead of decaying to zero, decay to ratio of min_lr / lr
+ total_num_steps += int(total_num_steps * lr_ratio) + config["warmup_steps"]
+ accelerator.print(f"Total training steps: {total_num_steps}")
+
+ # Creates Dummy Scheduler if `scheduler` was spcified in the config file else creates `args.lr_scheduler_type` Scheduler
+ if (
+ accelerator.state.deepspeed_plugin is None
+ or "scheduler" not in accelerator.state.deepspeed_plugin.deepspeed_config
+ ):
+ scheduler = get_scheduler(
+ name="cosine",
+ optimizer=optimizer,
+ num_warmup_steps=config["warmup_steps"] * accelerator.num_processes,
+ num_training_steps=total_num_steps * accelerator.num_processes,
+ )
+ else:
+ scheduler = DummyScheduler(
+ optimizer, total_num_steps=config["warmup_steps"], warmup_num_steps=config["warmup_steps"]
)
model, optimizer, train_dataloader, val_dataloader, scheduler = accelerator.prepare(
@@ -108,14 +135,13 @@ def train(accelerator, config):
accelerator.skip_first_batches(train_dataloader, resume_step)
accelerator.print(f"Resuming from step {resume_step}")
- train_loss = MeanMetric().to(model.device)
- if accelerator.state.deepspeed_plugin is not None:
- gradient_accumulation_steps = accelerator.state.deepspeed_plugin.deepspeed_config[
- "gradient_accumulation_steps"
- ]
+ # log gradients
+ if accelerator.is_local_main_process and config["wandb"]:
+ wandb.watch(model, log_freq=config["log_grads_every"])
for epoch in range(config["num_epochs"]):
+ train_loss = MeanMetric().to(model.device)
for step, batch in enumerate(tqdm(train_dataloader)):
model.train()
outputs = model(**batch)
@@ -139,9 +165,10 @@ def train(accelerator, config):
train_loss.update(loss_values["loss"])
if step > 0 and step % config["save_every"] == 0:
- accelerator.save_state(f"{config['output_dir']}/step_{step}")
+ curr_step = step + epoch * len(train_dataloader)
+ accelerator.save_state(f"{config['output_dir']}/step_{curr_step}")
- if step > 0 and step % config["eval_every"] == 0:
+ if step > 0 and (step % config["eval_every"] == 0 or step == len(train_dataloader) - 1):
val_loss = evaluate(config, model, val_dataloader)
log_train = {
@@ -166,7 +193,7 @@ def train(accelerator, config):
accelerator.wait_for_everyone()
unwrapped_model = accelerator.unwrap_model(model)
if accelerator.is_main_process:
- unwrapped_model.push_to_hub(config["save_name"] + "_first_epoch", private=True)
+ unwrapped_model.push_to_hub(config["save_name"] + f"-epoch_{epoch}", private=True)
accelerator.wait_for_everyone()
@@ -178,9 +205,6 @@ def train(accelerator, config):
state_dict=accelerator.get_state_dict(model),
)
- if accelerator.is_main_process:
- unwrapped_model.push_to_hub(config["save_name"], private=True)
-
accelerator.end_training()
From 63ff39653de905bc551b35ef096c5dc13a65efc7 Mon Sep 17 00:00:00 2001
From: Zach Nussbaum
Date: Tue, 4 Apr 2023 20:58:08 +0000
Subject: [PATCH 05/63] feat: gpt-j config
---
configs/deepspeed/ds_config_gptj.json | 28 +++++++++++++++++++++++++++
1 file changed, 28 insertions(+)
create mode 100644 configs/deepspeed/ds_config_gptj.json
diff --git a/configs/deepspeed/ds_config_gptj.json b/configs/deepspeed/ds_config_gptj.json
new file mode 100644
index 00000000..dc856e1b
--- /dev/null
+++ b/configs/deepspeed/ds_config_gptj.json
@@ -0,0 +1,28 @@
+{
+ "train_batch_size": "auto",
+ "gradient_accumulation_steps": 4,
+ "train_micro_batch_size_per_gpu": "auto",
+ "fp16": {
+ "enabled": "auto",
+ "min_loss_scale": 1,
+ "loss_scale_window": 1000,
+ "hysteresis": 2,
+ "initial_scale_power": 32
+ },
+ "bf16": {
+ "enabled": "auto"
+ },
+ "gradient_clipping": 1.0,
+ "zero_optimization": {
+ "stage": 2,
+ "offload_param": {
+ "device": "none"
+ },
+ "offload_optimizer": {
+ "device": "none"
+ },
+ "allgather_partitions": true,
+ "allgather_bucket_size": 5e8,
+ "contiguous_gradients": true
+ }
+ }
\ No newline at end of file
From 8dd99cc00af653035845677916005d5213f8331e Mon Sep 17 00:00:00 2001
From: Zach
Date: Tue, 4 Apr 2023 22:01:55 +0000
Subject: [PATCH 06/63] fix: prompt len for larger
---
data.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/data.py b/data.py
index 0de26cfa..358dd007 100644
--- a/data.py
+++ b/data.py
@@ -9,7 +9,6 @@ from transformers import DefaultDataCollator
def tokenize_inputs(config, tokenizer, examples):
max_length = config["max_length"]
- input_ids = torch.full((len(examples["prompt"]), max_length), tokenizer.pad_token_id)
# ignore bos
newline_tokens = tokenizer("\n", return_tensors="pt")["input_ids"][0]
if newline_tokens[0] == tokenizer.bos_token_id:
@@ -29,6 +28,7 @@ def tokenize_inputs(config, tokenizer, examples):
# we need to include some labels
if prompt_len >= max_length - 1:
prompt = prompt[:len(prompt) // 2]
+ prompt_len = len(tokenizer(prompt, truncation=True, return_tensors="pt")["input_ids"][0])
input_tokens = tokenizer(prompt + "\n" + response + tokenizer.eos_token,
truncation=True, max_length=max_length, return_tensors="pt")["input_ids"].squeeze()
From ad33b83a48d8aefb09799f2f15d5ff5be0f6fd97 Mon Sep 17 00:00:00 2001
From: Zach Nussbaum
Date: Tue, 4 Apr 2023 23:25:37 +0000
Subject: [PATCH 07/63] fix: eval func
---
train.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/train.py b/train.py
index 6c2c0515..1f0b4852 100644
--- a/train.py
+++ b/train.py
@@ -169,7 +169,7 @@ def train(accelerator, config):
accelerator.save_state(f"{config['output_dir']}/step_{curr_step}")
if step > 0 and (step % config["eval_every"] == 0 or step == len(train_dataloader) - 1):
- val_loss = evaluate(config, model, val_dataloader)
+ val_loss = evaluate(model, val_dataloader)
log_train = {
"train_loss": train_loss.compute()
From 9567f445b88b8b7d662ed0339f9ea72fddeb4c2a Mon Sep 17 00:00:00 2001
From: Zach Nussbaum
Date: Wed, 5 Apr 2023 02:36:22 +0000
Subject: [PATCH 08/63] fix: only on first process, not once on every node
---
train.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/train.py b/train.py
index 1f0b4852..0f35bc5b 100644
--- a/train.py
+++ b/train.py
@@ -137,7 +137,7 @@ def train(accelerator, config):
# log gradients
- if accelerator.is_local_main_process and config["wandb"]:
+ if accelerator.is_main_process and config["wandb"]:
wandb.watch(model, log_freq=config["log_grads_every"])
for epoch in range(config["num_epochs"]):
From 0c8666f635a33a8c97ea98a3f64c9d6ab3ca90b2 Mon Sep 17 00:00:00 2001
From: Zach Nussbaum
Date: Wed, 5 Apr 2023 02:52:21 +0000
Subject: [PATCH 09/63] fix: ignore env
---
.gitignore | 1 +
1 file changed, 1 insertion(+)
diff --git a/.gitignore b/.gitignore
index bbd0f77f..780ecb37 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
ckpts*
+.deepspeed_env
*.jsonl
*tar.gz
ckpts**
From 98ceac34e175876d704cbf49d8c58d1fc67e05b4 Mon Sep 17 00:00:00 2001
From: Zach Nussbaum
Date: Wed, 5 Apr 2023 02:52:44 +0000
Subject: [PATCH 10/63] fix: gptj multinode
---
configs/deepspeed/ds_config_gptj.json | 4 ++--
configs/train/finetune_gptj.yaml | 14 +++++++-------
2 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/configs/deepspeed/ds_config_gptj.json b/configs/deepspeed/ds_config_gptj.json
index dc856e1b..9e7a410a 100644
--- a/configs/deepspeed/ds_config_gptj.json
+++ b/configs/deepspeed/ds_config_gptj.json
@@ -1,6 +1,6 @@
{
"train_batch_size": "auto",
- "gradient_accumulation_steps": 4,
+ "gradient_accumulation_steps": "auto",
"train_micro_batch_size_per_gpu": "auto",
"fp16": {
"enabled": "auto",
@@ -19,7 +19,7 @@
"device": "none"
},
"offload_optimizer": {
- "device": "none"
+ "device": "cpu"
},
"allgather_partitions": true,
"allgather_bucket_size": 5e8,
diff --git a/configs/train/finetune_gptj.yaml b/configs/train/finetune_gptj.yaml
index 31c25050..1b42d780 100644
--- a/configs/train/finetune_gptj.yaml
+++ b/configs/train/finetune_gptj.yaml
@@ -2,24 +2,24 @@
model_name: "EleutherAI/gpt-j-6B"
tokenizer_name: "EleutherAI/gpt-j-6B"
gradient_checkpointing: true
-save_name: "nomic-ai/gpt4all-gptj-multiturn-lr-aggressive"
+save_name: "nomic-ai/gpt4all-gptj-multinode"
# dataset
streaming: false
num_proc: 64
dataset_path: "data_multiplus"
max_length: 1024
-batch_size: 8
+batch_size: 32
# train dynamics
-lr: 2.0e-5
+lr: 4.0e-5
min_lr: 0
weight_decay: 0.0
-eval_every: 200
+eval_every: 100
eval_steps: 105
-save_every: 400
-log_grads_every: 200
-output_dir: "ckpts/gpt4all-gptj-full-multiturn-lr-aggreive"
+save_every: 100
+log_grads_every: 100
+output_dir: "ckpts/gpt4all-gptj-multinode"
checkpoint: null
lora: false
warmup_steps: 500
From 885b7f1a3ad817d4c497588d4e1fb4f18a0c91e3 Mon Sep 17 00:00:00 2001
From: Zach Nussbaum
Date: Wed, 5 Apr 2023 02:53:04 +0000
Subject: [PATCH 11/63] feat: multinode setup
---
create_hostname.sh | 8 ++++++++
head_node_setup.sh | 19 +++++++++++++++++++
worker_node_setup.sh | 6 ++++++
3 files changed, 33 insertions(+)
create mode 100644 create_hostname.sh
create mode 100644 head_node_setup.sh
create mode 100644 worker_node_setup.sh
diff --git a/create_hostname.sh b/create_hostname.sh
new file mode 100644
index 00000000..8a9187f2
--- /dev/null
+++ b/create_hostname.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+export WORKER_IP=$1
+N_GPUS=8
+# create dir if doesn't exist
+sudo mkdir -p /job
+printf "localhost slots=$N_GPUS\n$WORKER_IP slots=$N_GPUS" | sudo tee /job/hostfile
+echo /job/hostfile
\ No newline at end of file
diff --git a/head_node_setup.sh b/head_node_setup.sh
new file mode 100644
index 00000000..7a813d1e
--- /dev/null
+++ b/head_node_setup.sh
@@ -0,0 +1,19 @@
+#!/bin/sh
+WORKER_IP=$1
+N_GPUS=$2
+
+sudo apt install -y nfs-kernel-server
+sudo mkdir -p ./data_multiplus
+sudo chmod 777 ./data_multiplus
+printf "${PWD}/data_multiplus ${WORKER_IP}(rw,sync,no_subtree_check)" | sudo tee -a /etc/exports
+sudo systemctl restart nfs-kernel-server
+
+sudo apt-get install -y pdsh
+export DSHPATH=$PATH
+export PDSH_RCMD_TYPE=ssh
+
+ssh-keygen -t rsa -N ''
+cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
+
+sudo mkdir -p /job
+printf "localhost slots=$N_GPUS\n$WORKER_IP slots=$N_GPUS" | sudo tee /job/hostfile
\ No newline at end of file
diff --git a/worker_node_setup.sh b/worker_node_setup.sh
new file mode 100644
index 00000000..2c50e29d
--- /dev/null
+++ b/worker_node_setup.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+HEAD_IP=$1
+
+sudo apt install -y nfs-common
+sudo mkdir -p ./data_multiplus
+sudo mount ${HEAD_IP}:${PWD}/data_multiplus ./data_multiplus
\ No newline at end of file
From 838b19bea53fd323dc107771315f9e4dc7c7da47 Mon Sep 17 00:00:00 2001
From: Zach
Date: Wed, 5 Apr 2023 20:42:22 +0000
Subject: [PATCH 12/63] fix: try except push
---
train.py | 16 ++++++++++++++--
1 file changed, 14 insertions(+), 2 deletions(-)
diff --git a/train.py b/train.py
index 0f35bc5b..2cf92aa8 100644
--- a/train.py
+++ b/train.py
@@ -13,6 +13,7 @@ from torchmetrics import MeanMetric
from tqdm import tqdm
import wandb
+torch.backends.cuda.matmul.allow_tf32 = True
def format_metrics(metrics, split, prefix=""):
log = f"[{split}]" + prefix
@@ -192,9 +193,20 @@ def train(accelerator, config):
accelerator.print(f"Pushing to HF hub")
accelerator.wait_for_everyone()
unwrapped_model = accelerator.unwrap_model(model)
- if accelerator.is_main_process:
- unwrapped_model.push_to_hub(config["save_name"] + f"-epoch_{epoch}", private=True)
+ try:
+ if accelerator.is_main_process:
+ unwrapped_model.push_to_hub(config["save_name"] + f"-epoch_{epoch}", private=True)
+ except Exception as e:
+ accelerator.print(e)
+ accelerator.print(f"Failed to push to hub")
+
+ unwrapped_model.save_pretrained(
+ f"{config['output_dir']}/-epoch_{epoch}",
+ is_main_process=accelerator.is_main_process,
+ save_function=accelerator.save,
+ state_dict=accelerator.get_state_dict(model),
+ )
accelerator.wait_for_everyone()
unwrapped_model = accelerator.unwrap_model(model)
From c2fc16477988dc34a6d8cf33e8cbc66f12656b80 Mon Sep 17 00:00:00 2001
From: Zach
Date: Wed, 5 Apr 2023 20:42:35 +0000
Subject: [PATCH 13/63] fix: configs
---
configs/deepspeed/ds_config_gptj.json | 2 +-
configs/train/finetune_gptj.yaml | 10 +++++-----
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/configs/deepspeed/ds_config_gptj.json b/configs/deepspeed/ds_config_gptj.json
index 9e7a410a..3e933966 100644
--- a/configs/deepspeed/ds_config_gptj.json
+++ b/configs/deepspeed/ds_config_gptj.json
@@ -19,7 +19,7 @@
"device": "none"
},
"offload_optimizer": {
- "device": "cpu"
+ "device": "none"
},
"allgather_partitions": true,
"allgather_bucket_size": 5e8,
diff --git a/configs/train/finetune_gptj.yaml b/configs/train/finetune_gptj.yaml
index 1b42d780..f37283b3 100644
--- a/configs/train/finetune_gptj.yaml
+++ b/configs/train/finetune_gptj.yaml
@@ -2,7 +2,7 @@
model_name: "EleutherAI/gpt-j-6B"
tokenizer_name: "EleutherAI/gpt-j-6B"
gradient_checkpointing: true
-save_name: "nomic-ai/gpt4all-gptj-multinode"
+save_name: "nomic-ai/gpt4all-gptj-multinode-deepspeed"
# dataset
streaming: false
@@ -12,13 +12,13 @@ max_length: 1024
batch_size: 32
# train dynamics
-lr: 4.0e-5
+lr: 2.0e-5
min_lr: 0
weight_decay: 0.0
-eval_every: 100
+eval_every: 500
eval_steps: 105
-save_every: 100
-log_grads_every: 100
+save_every: 500
+log_grads_every: 500
output_dir: "ckpts/gpt4all-gptj-multinode"
checkpoint: null
lora: false
From e4e88dff33577ac3b446633ccd8d358b7dac31f6 Mon Sep 17 00:00:00 2001
From: Zach
Date: Thu, 6 Apr 2023 03:03:34 +0000
Subject: [PATCH 14/63] fix: data processing
---
data.py | 23 +++++++++++++++++------
1 file changed, 17 insertions(+), 6 deletions(-)
diff --git a/data.py b/data.py
index 358dd007..ff519abb 100644
--- a/data.py
+++ b/data.py
@@ -22,24 +22,35 @@ def tokenize_inputs(config, tokenizer, examples):
if response.count("") > 0:
response = response.replace("", tokenizer.eos_token)
- prompt_len = len(tokenizer(prompt, truncation=True, return_tensors="pt")["input_ids"][0])
+ prompt_len = len(tokenizer(prompt, return_tensors="pt")["input_ids"][0])
# hack if our prompt is super long
- # we need to include some labels
- if prompt_len >= max_length - 1:
- prompt = prompt[:len(prompt) // 2]
- prompt_len = len(tokenizer(prompt, truncation=True, return_tensors="pt")["input_ids"][0])
+ # we need to include some labels so we arbitrarily trunacate at max_length // 2
+ # if the length is too long
+ if prompt_len >= max_length // 2:
+ # if prompt is too long, truncate
+ # but make sure to truncate to at max 1024 tokens
+ new_len = min(max_length // 2, len(prompt) // 2)
+ prompt = prompt[:new_len]
+ # get new prompt length
+ prompt_len = tokenizer(prompt, return_tensors="pt", max_length=max_length // 2, truncation=True).input_ids.ne(tokenizer.pad_token_id).sum().item()
+
+ assert prompt_len <= max_length // 2, f"prompt length {prompt_len} exceeds max length {max_length}"
input_tokens = tokenizer(prompt + "\n" + response + tokenizer.eos_token,
truncation=True, max_length=max_length, return_tensors="pt")["input_ids"].squeeze()
-
labels = input_tokens.clone()
labels[:prompt_len + len(newline_tokens)] = -100
if len(labels) < max_length:
# pad to max_length with -100
labels = torch.cat([labels, torch.full((max_length - len(labels),), -100)])
+ if (labels == -100).sum() == len(labels) - 1:
+ print(prompt)
+ print(response)
+ raise
+
input_tokens = tokenizer.pad({"input_ids": input_tokens}, padding="max_length", max_length=max_length)["input_ids"]
out["labels"].append(labels)
out["input_ids"].append(input_tokens)
From 8f2eb1a583ab2edfe151891bd46e1315b3e91f34 Mon Sep 17 00:00:00 2001
From: Zach
Date: Thu, 6 Apr 2023 12:11:10 +0000
Subject: [PATCH 15/63] fix: grad accum loss calc
---
train.py | 22 +++++++++++++---------
1 file changed, 13 insertions(+), 9 deletions(-)
diff --git a/train.py b/train.py
index 2cf92aa8..463dd155 100644
--- a/train.py
+++ b/train.py
@@ -1,8 +1,9 @@
import os
-from transformers import AutoModelForCausalLM, AutoTokenizer, AdamW, get_scheduler
+from transformers import AutoModelForCausalLM, AutoTokenizer, get_scheduler
from transformers.trainer_pt_utils import get_parameter_names
import torch
import torch.nn as nn
+from torch.optim import AdamW
from argparse import ArgumentParser
from read import read_config
from accelerate import Accelerator
@@ -24,7 +25,7 @@ def format_metrics(metrics, split, prefix=""):
def evaluate(model, val_dataloader):
model.eval()
- val_loss = MeanMetric().to(model.device)
+ val_loss = MeanMetric(nan_strategy="error").to(model.device)
with torch.no_grad():
for batch in tqdm(val_dataloader):
@@ -43,7 +44,7 @@ def train(accelerator, config):
accelerator.print(config)
accelerator.print(f"Using {accelerator.num_processes} GPUs")
- tokenizer = AutoTokenizer.from_pretrained(config['tokenizer_name'])
+ tokenizer = AutoTokenizer.from_pretrained(config['tokenizer_name'], model_max_length=config['max_length'])
# llama has no pad token, set it to new token
if tokenizer.pad_token is None:
tokenizer.pad_token = tokenizer.eos_token
@@ -51,7 +52,7 @@ def train(accelerator, config):
with accelerator.main_process_first():
train_dataloader, val_dataloader = load_data(config, tokenizer)
-
+
checkpoint = config["gradient_checkpointing"]
model = AutoModelForCausalLM.from_pretrained(config["model_name"],
@@ -139,17 +140,22 @@ def train(accelerator, config):
# log gradients
if accelerator.is_main_process and config["wandb"]:
- wandb.watch(model, log_freq=config["log_grads_every"])
+ wandb.watch(model, log_freq=config["log_grads_every"], log="all")
for epoch in range(config["num_epochs"]):
- train_loss = MeanMetric().to(model.device)
+ train_loss = MeanMetric(nan_strategy="error").to(model.device)
for step, batch in enumerate(tqdm(train_dataloader)):
model.train()
outputs = model(**batch)
loss = outputs.loss
- loss = loss / gradient_accumulation_steps
+ # gather loss before backprop in case of gradient accumulation
+ loss_values = accelerator.gather_for_metrics({"loss": loss.detach().float()})
+ train_loss.update(loss_values["loss"])
+
+ loss = loss / gradient_accumulation_steps
accelerator.backward(loss)
+ # get gradient norm of all params
# log LR in case something weird happens
if step > 0 and step % (config["eval_every"] // 10) == 0:
@@ -162,8 +168,6 @@ def train(accelerator, config):
scheduler.step()
optimizer.zero_grad()
- loss_values = accelerator.gather_for_metrics({"loss": loss.detach()})
- train_loss.update(loss_values["loss"])
if step > 0 and step % config["save_every"] == 0:
curr_step = step + epoch * len(train_dataloader)
From 3f9fea47d7b0bca7007dfaca8a8bd890f6148d76 Mon Sep 17 00:00:00 2001
From: Zach
Date: Fri, 7 Apr 2023 01:40:39 +0000
Subject: [PATCH 16/63] feat: inference for embedding plots
---
configs/inference/gptj.yaml | 14 +++
inference.py | 185 ++++++++++++++++++++++++++++++++++++
2 files changed, 199 insertions(+)
create mode 100644 configs/inference/gptj.yaml
create mode 100644 inference.py
diff --git a/configs/inference/gptj.yaml b/configs/inference/gptj.yaml
new file mode 100644
index 00000000..8c20efac
--- /dev/null
+++ b/configs/inference/gptj.yaml
@@ -0,0 +1,14 @@
+# model/tokenizer
+model_name: "nomic-ai/gpt4all-gptj-multinode-deepspeed-finetuned-epoch_0"
+tokenizer_name: "EleutherAI/gpt-j-6B"
+
+# dataset
+streaming: false
+num_proc: 64
+dataset_path: "data_multiplus"
+max_length: 1024
+batch_size: 32
+
+# logging
+seed: 42
+
diff --git a/inference.py b/inference.py
new file mode 100644
index 00000000..e8585d3e
--- /dev/null
+++ b/inference.py
@@ -0,0 +1,185 @@
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import torch
+import torch.nn as nn
+from argparse import ArgumentParser
+from read import read_config
+from accelerate.utils import set_seed
+from data import load_data_for_inference
+from tqdm import tqdm
+from datasets import concatenate_datasets, Dataset
+import torch.distributed as dist
+from transformers.trainer_pt_utils import ShardSampler, distributed_concat, nested_numpify
+from transformers import DefaultDataCollator
+from torch.utils.data import DataLoader
+
+
+def calc_cross_entropy_no_reduction(lm_logits, labels):
+ # calculate cross entropy across batch dim
+ shift_logits = lm_logits[..., :-1, :].contiguous()
+ shift_labels = labels[..., 1:].contiguous()
+ # Flatten the tokens
+ loss_fct = nn.CrossEntropyLoss(reduction='none')
+ loss = loss_fct(shift_logits.permute(0, 2, 1), shift_labels).mean(dim=1)
+
+ return loss
+
+
+def rank0_print(msg):
+ if dist.get_rank() == 0:
+ print(msg)
+
+
+def inference(config):
+ set_seed(config['seed'])
+
+ rank0_print(f"World size: {dist.get_world_size()}")
+
+ tokenizer = AutoTokenizer.from_pretrained(config['tokenizer_name'], model_max_length=config['max_length'])
+ # llama has no pad token, set it to new token
+ if tokenizer.pad_token is None:
+ tokenizer.pad_token = tokenizer.eos_token
+
+
+ train_dataset, val_dataset = load_data_for_inference(config, tokenizer)
+
+ num_processes = dist.get_world_size()
+ local_rank = dist.get_rank()
+
+ train_sampler = ShardSampler(train_dataset, config["batch_size"], num_processes=num_processes, process_index=local_rank)
+ train_dataloader = DataLoader(
+ train_dataset,
+ collate_fn=DefaultDataCollator(),
+ batch_size=config["batch_size"],
+ sampler=train_sampler
+ )
+
+ val_sampler = ShardSampler(val_dataset, config["batch_size"], num_processes=num_processes, process_index=local_rank)
+ val_dataloader = DataLoader(
+ val_dataset,
+ collate_fn=DefaultDataCollator(),
+ batch_size=config["batch_size"],
+ sampler=val_sampler
+ )
+
+
+ model = AutoModelForCausalLM.from_pretrained(config["model_name"],
+ trust_remote_code=True,
+ torch_dtype=torch.bfloat16,
+ )
+
+ model.to(f"cuda:{local_rank}")
+
+ with torch.no_grad():
+ train_outputs = {"loss": [], "embeddings": [], "index": []}
+ for batch in tqdm(train_dataloader, disable=local_rank != 0):
+ batch["input_ids"] = batch["input_ids"].to(f"cuda:{local_rank}")
+ batch["labels"] = batch["labels"].to(f"cuda:{local_rank}")
+ outputs = model(input_ids=batch["input_ids"], labels=batch["labels"], output_hidden_states=True)
+ loss = calc_cross_entropy_no_reduction(outputs.logits, batch["labels"])
+ train_outputs["loss"].extend(loss)
+
+ embeddings = outputs.hidden_states[-1]
+ batch_size = batch["input_ids"].shape[0]
+ sequence_lengths = []
+ # since we use mutiturn with multiple <|endoftext|>, we need to find the place where
+ # <|endoftext|> is repeated
+ for item in batch["input_ids"]:
+ indices = torch.where(item == tokenizer.pad_token_id)[0]
+ found = False
+ for index in indices:
+ # case where sequence is less than max length
+ if torch.all(item[index:] == tokenizer.pad_token_id):
+ sequence_lengths.append(index)
+ found = True
+ break
+ # case where sequence is >= max length
+ if not found:
+ sequence_lengths.append(len(item) - 1)
+
+ sequence_lengths = torch.tensor(sequence_lengths)
+ pooled_logits = embeddings[torch.arange(batch_size, device=embeddings.device), sequence_lengths]
+
+ train_outputs["embeddings"].extend(pooled_logits)
+ train_outputs["index"].extend(batch["index"].to(model.device))
+
+ torch.cuda.empty_cache()
+
+ dist.barrier()
+ gathered_train = nested_numpify(distributed_concat(train_outputs))
+
+ gathered_train["index"] = [t.item() for t in gathered_train["index"]]
+ gathered_train["loss"] = [t.item() for t in gathered_train["loss"]]
+
+ df_train = Dataset.from_dict(gathered_train)
+ df_train = df_train.sort("index")
+
+ train_dataset = train_dataset.add_column("embeddings", df_train["embeddings"])
+ train_dataset = train_dataset.add_column("loss", df_train["loss"])
+ train_dataset = train_dataset.add_column("is_train", [True] * len(train_dataset))
+
+ val_outputs = {"loss": [], "embeddings": [], "index": []}
+ for batch in tqdm(val_dataloader, disable=local_rank != 0):
+ batch["input_ids"] = batch["input_ids"].to(f"cuda:{local_rank}")
+ batch["labels"] = batch["labels"].to(f"cuda:{local_rank}")
+ outputs = model(input_ids=batch["input_ids"], labels=batch["labels"])
+ loss = calc_cross_entropy_no_reduction(outputs.logits, batch["labels"])
+ val_outputs["loss"].extend(loss)
+
+ logits = outputs.logits
+ batch_size = batch["input_ids"].shape[0]
+ sequence_lengths = []
+ # since we use mutiturn with multiple <|endoftext|>, we need to find the place where
+ # <|endoftext|> is repeated
+ for item in batch["input_ids"]:
+ indices = torch.where(item == tokenizer.pad_token_id)[0]
+ found = False
+ for index in indices:
+ if torch.all(item[index:] == tokenizer.pad_token_id):
+ sequence_lengths.append(index)
+ found = True
+ break
+
+ # no match found
+ if not found:
+ sequence_lengths.append(len(item) - 1)
+
+ sequence_lengths = torch.tensor(sequence_lengths)
+ pooled_logits = logits[torch.arange(batch_size, device=logits.device), sequence_lengths]
+
+ val_outputs["embeddings"].extend(pooled_logits)
+ val_outputs["index"].extend(batch["index"].to(model.device))
+
+ torch.cuda.empty_cache()
+
+ dist.barrier()
+ gathered_val = nested_numpify(distributed_concat(val_outputs))
+
+ gathered_val["index"] = [t.item() for t in gathered_val["index"]]
+ gathered_val["loss"] = [t.item() for t in gathered_val["loss"]]
+
+ df_val = Dataset.from_dict(gathered_val)
+ df_val = df_val.sort("index")
+
+ val_dataset = val_dataset.add_column("embeddings", df_val["embeddings"])
+ val_dataset = val_dataset.add_column("loss", df_val["loss"])
+ val_dataset = val_dataset.add_column("is_train", [False] * len(val_dataset))
+
+ df = concatenate_datasets([train_dataset, val_dataset])
+ df.to_json("epoch_1_checkpoint.jsonl", lines=True, orient="records", num_proc=64)
+
+
+def main():
+ dist.init_process_group("nccl")
+ parser = ArgumentParser()
+ parser.add_argument("--config", type=str, default="config.yaml")
+
+ args = parser.parse_args()
+ config = read_config(args.config)
+
+ inference(config)
+
+
+if __name__ == "__main__":
+ # parse arguments by reading in a config
+ main()
+
From 57eb78675679c224b15fb679639ab03eb654fad1 Mon Sep 17 00:00:00 2001
From: Zach
Date: Fri, 7 Apr 2023 01:45:07 +0000
Subject: [PATCH 17/63] fix: data for inference
---
data.py | 48 +++++++++++++++++++++++++++++++++++++++++++++++-
requirements.txt | 3 ++-
2 files changed, 49 insertions(+), 2 deletions(-)
diff --git a/data.py b/data.py
index ff519abb..6375584d 100644
--- a/data.py
+++ b/data.py
@@ -75,7 +75,7 @@ def load_data(config, tokenizer):
dataset = load_dataset("json", data_files=files, split="train")
else:
- dataset = load_dataset(dataset_path)
+ dataset = load_dataset(dataset_path, split="train")
dataset = dataset.train_test_split(test_size=.05, seed=config["seed"])
@@ -118,3 +118,49 @@ def load_data(config, tokenizer):
)
return train_dataloader, val_dataloader
+
+
+def load_data_for_inference(config, tokenizer):
+ dataset_path = config["dataset_path"]
+
+ if os.path.exists(dataset_path):
+ # check if path is a directory
+ if os.path.isdir(dataset_path):
+ files = glob.glob(os.path.join(dataset_path, "*_clean.jsonl"))
+ else:
+ files = [dataset_path]
+
+ print(f"Reading files {files}")
+
+ dataset = load_dataset("json", data_files=files, split="train")
+
+ else:
+ dataset = load_dataset(dataset_path, split="train")
+
+ dataset = dataset.train_test_split(test_size=.05, seed=config["seed"])
+
+ train_dataset, val_dataset = dataset["train"], dataset["test"]
+
+ train_dataset = train_dataset.add_column("index", list(range(len(train_dataset))))
+ val_dataset = val_dataset.add_column("index", list(range(len(val_dataset))))
+
+ if config["streaming"] is False:
+ kwargs = {"num_proc": config["num_proc"]}
+ else:
+ kwargs = {}
+
+ # tokenize inputs and return labels and attention mask
+ train_dataset = train_dataset.map(
+ lambda ele: tokenize_inputs(config, tokenizer, ele),
+ batched=True,
+ **kwargs
+ )
+ val_dataset = val_dataset.map(
+ lambda ele: tokenize_inputs(config, tokenizer, ele),
+ batched=True,
+ **kwargs
+ )
+ train_dataset = train_dataset.with_format("torch")
+ val_dataset = val_dataset.with_format("torch")
+
+ return train_dataset, val_dataset
diff --git a/requirements.txt b/requirements.txt
index 8a91fd74..a30fc8a5 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -9,4 +9,5 @@ peft
nodelist-inflator
deepspeed
sentencepiece
-jsonlines
\ No newline at end of file
+jsonlines
+nomic
\ No newline at end of file
From d93f6c2cedfdc5d4563bdc13f0e9f79f8f86dbab Mon Sep 17 00:00:00 2001
From: Zach
Date: Fri, 7 Apr 2023 04:33:34 +0000
Subject: [PATCH 18/63] fix: concat
---
inference.py | 19 +++++++++++--------
1 file changed, 11 insertions(+), 8 deletions(-)
diff --git a/inference.py b/inference.py
index e8585d3e..3facd91c 100644
--- a/inference.py
+++ b/inference.py
@@ -11,6 +11,7 @@ import torch.distributed as dist
from transformers.trainer_pt_utils import ShardSampler, distributed_concat, nested_numpify
from transformers import DefaultDataCollator
from torch.utils.data import DataLoader
+import numpy as np
def calc_cross_entropy_no_reduction(lm_logits, labels):
@@ -99,16 +100,16 @@ def inference(config):
sequence_lengths = torch.tensor(sequence_lengths)
pooled_logits = embeddings[torch.arange(batch_size, device=embeddings.device), sequence_lengths]
- train_outputs["embeddings"].extend(pooled_logits)
+ train_outputs["embeddings"].append(pooled_logits)
train_outputs["index"].extend(batch["index"].to(model.device))
torch.cuda.empty_cache()
dist.barrier()
gathered_train = nested_numpify(distributed_concat(train_outputs))
-
- gathered_train["index"] = [t.item() for t in gathered_train["index"]]
- gathered_train["loss"] = [t.item() for t in gathered_train["loss"]]
+ gathered_train["index"] = np.concatenate(gathered_train["index"])
+ gathered_train["loss"] = np.concatenate(gathered_train["loss"])
+ gathered_train["embeddings"] = np.concatenate(gathered_train["embeddings"])
df_train = Dataset.from_dict(gathered_train)
df_train = df_train.sort("index")
@@ -146,7 +147,7 @@ def inference(config):
sequence_lengths = torch.tensor(sequence_lengths)
pooled_logits = logits[torch.arange(batch_size, device=logits.device), sequence_lengths]
- val_outputs["embeddings"].extend(pooled_logits)
+ val_outputs["embeddings"].append(pooled_logits)
val_outputs["index"].extend(batch["index"].to(model.device))
torch.cuda.empty_cache()
@@ -154,8 +155,9 @@ def inference(config):
dist.barrier()
gathered_val = nested_numpify(distributed_concat(val_outputs))
- gathered_val["index"] = [t.item() for t in gathered_val["index"]]
- gathered_val["loss"] = [t.item() for t in gathered_val["loss"]]
+ gathered_val["index"] = np.concatenate(gathered_val["index"])
+ gathered_val["loss"] = np.concatenate(gathered_val["loss"])
+ gathered_val["embeddings"] = np.concatenate(gathered_val["embeddings"])
df_val = Dataset.from_dict(gathered_val)
df_val = df_val.sort("index")
@@ -165,7 +167,8 @@ def inference(config):
val_dataset = val_dataset.add_column("is_train", [False] * len(val_dataset))
df = concatenate_datasets([train_dataset, val_dataset])
- df.to_json("epoch_1_checkpoint.jsonl", lines=True, orient="records", num_proc=64)
+ if local_rank == 0:
+ df.to_json("epoch_1_checkpoint.jsonl", lines=True, orient="records", num_proc=64)
def main():
From 573272ad69d38ce281f76d80e648357437a7d727 Mon Sep 17 00:00:00 2001
From: Zach
Date: Fri, 7 Apr 2023 12:09:31 +0000
Subject: [PATCH 19/63] fix: drop uneven batch size
---
data.py | 4 ++++
inference.py | 11 ++++++-----
2 files changed, 10 insertions(+), 5 deletions(-)
diff --git a/data.py b/data.py
index 6375584d..8a0dd83f 100644
--- a/data.py
+++ b/data.py
@@ -142,7 +142,11 @@ def load_data_for_inference(config, tokenizer):
train_dataset, val_dataset = dataset["train"], dataset["test"]
train_dataset = train_dataset.add_column("index", list(range(len(train_dataset))))
+ # select first N batches that are divisible by batch_size
+ # gather is a bit annoying (or the way I'm using it) to get uneven batches as it duplicates data
+ train_dataset = train_dataset.select(range((len(train_dataset) // config["batch_size"]) * config["batch_size"]))
val_dataset = val_dataset.add_column("index", list(range(len(val_dataset))))
+ val_dataset = val_dataset.select(range((len(val_dataset) // config["batch_size"]) * config["batch_size"]))
if config["streaming"] is False:
kwargs = {"num_proc": config["num_proc"]}
diff --git a/inference.py b/inference.py
index 3facd91c..1fb620ab 100644
--- a/inference.py
+++ b/inference.py
@@ -46,20 +46,22 @@ def inference(config):
num_processes = dist.get_world_size()
local_rank = dist.get_rank()
- train_sampler = ShardSampler(train_dataset, config["batch_size"], num_processes=num_processes, process_index=local_rank)
+ train_sampler = ShardSampler(train_dataset, config["batch_size"], drop_last=True, num_processes=num_processes, process_index=local_rank)
train_dataloader = DataLoader(
train_dataset,
collate_fn=DefaultDataCollator(),
batch_size=config["batch_size"],
- sampler=train_sampler
+ sampler=train_sampler,
+ drop_last=True
)
- val_sampler = ShardSampler(val_dataset, config["batch_size"], num_processes=num_processes, process_index=local_rank)
+ val_sampler = ShardSampler(val_dataset, config["batch_size"], drop_last=True, num_processes=num_processes, process_index=local_rank)
val_dataloader = DataLoader(
val_dataset,
collate_fn=DefaultDataCollator(),
batch_size=config["batch_size"],
- sampler=val_sampler
+ sampler=val_sampler,
+ drop_last=True
)
@@ -113,7 +115,6 @@ def inference(config):
df_train = Dataset.from_dict(gathered_train)
df_train = df_train.sort("index")
-
train_dataset = train_dataset.add_column("embeddings", df_train["embeddings"])
train_dataset = train_dataset.add_column("loss", df_train["loss"])
train_dataset = train_dataset.add_column("is_train", [True] * len(train_dataset))
From f974ca651ce1d2342acb632a6d5bcdd9efec07b2 Mon Sep 17 00:00:00 2001
From: Zach Nussbaum
Date: Fri, 7 Apr 2023 16:23:34 +0000
Subject: [PATCH 20/63] fix: inference save shards
---
inference.py | 60 ++++++++++++++++++++++++++++------------------------
1 file changed, 32 insertions(+), 28 deletions(-)
diff --git a/inference.py b/inference.py
index 1fb620ab..0fa73025 100644
--- a/inference.py
+++ b/inference.py
@@ -6,11 +6,11 @@ from read import read_config
from accelerate.utils import set_seed
from data import load_data_for_inference
from tqdm import tqdm
-from datasets import concatenate_datasets, Dataset
+from datasets import Dataset
import torch.distributed as dist
-from transformers.trainer_pt_utils import ShardSampler, distributed_concat, nested_numpify
+from transformers.trainer_pt_utils import nested_numpify
from transformers import DefaultDataCollator
-from torch.utils.data import DataLoader
+from torch.utils.data import DataLoader, DistributedSampler
import numpy as np
@@ -46,7 +46,7 @@ def inference(config):
num_processes = dist.get_world_size()
local_rank = dist.get_rank()
- train_sampler = ShardSampler(train_dataset, config["batch_size"], drop_last=True, num_processes=num_processes, process_index=local_rank)
+ train_sampler = DistributedSampler(train_dataset, shuffle=False, drop_last=True, num_replicas=num_processes, rank=local_rank)
train_dataloader = DataLoader(
train_dataset,
collate_fn=DefaultDataCollator(),
@@ -55,7 +55,7 @@ def inference(config):
drop_last=True
)
- val_sampler = ShardSampler(val_dataset, config["batch_size"], drop_last=True, num_processes=num_processes, process_index=local_rank)
+ val_sampler = DistributedSampler(val_dataset, shuffle=False, drop_last=True, num_replicas=num_processes, rank=local_rank)
val_dataloader = DataLoader(
val_dataset,
collate_fn=DefaultDataCollator(),
@@ -69,7 +69,6 @@ def inference(config):
trust_remote_code=True,
torch_dtype=torch.bfloat16,
)
-
model.to(f"cuda:{local_rank}")
with torch.no_grad():
@@ -107,17 +106,23 @@ def inference(config):
torch.cuda.empty_cache()
- dist.barrier()
- gathered_train = nested_numpify(distributed_concat(train_outputs))
- gathered_train["index"] = np.concatenate(gathered_train["index"])
- gathered_train["loss"] = np.concatenate(gathered_train["loss"])
- gathered_train["embeddings"] = np.concatenate(gathered_train["embeddings"])
+ train_outputs = nested_numpify(train_outputs)
+ # stack since they're 0-dim arrays
+ train_outputs["index"] = np.stack(train_outputs["index"])
+ train_outputs["loss"] = np.stack(train_outputs["loss"])
+ train_outputs["embeddings"] = np.concatenate(train_outputs["embeddings"])
- df_train = Dataset.from_dict(gathered_train)
+ df_train = Dataset.from_dict(train_outputs)
df_train = df_train.sort("index")
- train_dataset = train_dataset.add_column("embeddings", df_train["embeddings"])
- train_dataset = train_dataset.add_column("loss", df_train["loss"])
- train_dataset = train_dataset.add_column("is_train", [True] * len(train_dataset))
+ curr_idx = df_train["index"]
+
+ filtered_train = train_dataset.filter(lambda example: example["index"] in curr_idx)
+
+ filtered_train = filtered_train.add_column("embeddings", df_train["embeddings"])
+ filtered_train = filtered_train.add_column("loss", df_train["loss"])
+ filtered_train = filtered_train.add_column("is_train", [True] * len(filtered_train))
+
+ filtered_train.to_json(f"inference/epoch_2_embeddings_train_shard_{local_rank}.jsonl", lines=True, orient="records", num_proc=64)
val_outputs = {"loss": [], "embeddings": [], "index": []}
for batch in tqdm(val_dataloader, disable=local_rank != 0):
@@ -153,25 +158,24 @@ def inference(config):
torch.cuda.empty_cache()
- dist.barrier()
- gathered_val = nested_numpify(distributed_concat(val_outputs))
+ val_outputs = nested_numpify(val_outputs)
+ val_outputs["index"] = np.stack(val_outputs["index"])
+ val_outputs["loss"] = np.stack(val_outputs["loss"])
+ val_outputs["embeddings"] = np.concatenate(val_outputs["embeddings"])
- gathered_val["index"] = np.concatenate(gathered_val["index"])
- gathered_val["loss"] = np.concatenate(gathered_val["loss"])
- gathered_val["embeddings"] = np.concatenate(gathered_val["embeddings"])
-
- df_val = Dataset.from_dict(gathered_val)
+ df_val = Dataset.from_dict(val_outputs)
df_val = df_val.sort("index")
+ curr_idx = df_val["index"]
- val_dataset = val_dataset.add_column("embeddings", df_val["embeddings"])
- val_dataset = val_dataset.add_column("loss", df_val["loss"])
- val_dataset = val_dataset.add_column("is_train", [False] * len(val_dataset))
+ filtered_val = val_dataset.filter(lambda example: example["index"] in curr_idx)
- df = concatenate_datasets([train_dataset, val_dataset])
- if local_rank == 0:
- df.to_json("epoch_1_checkpoint.jsonl", lines=True, orient="records", num_proc=64)
+ filtered_val = filtered_val.add_column("embeddings", df_val["embeddings"])
+ filtered_val = filtered_val.add_column("loss", df_val["loss"])
+ filtered_val = filtered_val.add_column("is_train", [False] * len(filtered_val))
+ filtered_val.to_json(f"inference/epoch_2_embeddings_val_shard_{local_rank}.jsonl", lines=True, orient="records", num_proc=64)
+
def main():
dist.init_process_group("nccl")
parser = ArgumentParser()
From 1b5e660476f5249a57d35bbb3dbc7dac64117604 Mon Sep 17 00:00:00 2001
From: Zach Nussbaum
Date: Fri, 7 Apr 2023 19:04:19 +0000
Subject: [PATCH 21/63] fix: pyarrow filter
---
inference.py | 18 ++++++++++++++++--
1 file changed, 16 insertions(+), 2 deletions(-)
diff --git a/inference.py b/inference.py
index 0fa73025..25e3b508 100644
--- a/inference.py
+++ b/inference.py
@@ -12,6 +12,8 @@ from transformers.trainer_pt_utils import nested_numpify
from transformers import DefaultDataCollator
from torch.utils.data import DataLoader, DistributedSampler
import numpy as np
+import pyarrow as pa
+from pyarrow import compute as pc
def calc_cross_entropy_no_reduction(lm_logits, labels):
@@ -116,7 +118,13 @@ def inference(config):
df_train = df_train.sort("index")
curr_idx = df_train["index"]
- filtered_train = train_dataset.filter(lambda example: example["index"] in curr_idx)
+ # compute mask in pyarrow since it's super fast
+ # ty @bmschmidt for showing me this!
+ table = train_dataset.data
+ mask = pc.is_in(table['index'], value_set=pa.array(curr_idx, pa.int32()))
+ filtered_table = table.filter(mask)
+ # convert from pyarrow to Dataset
+ filtered_train = Dataset.from_dict(filtered_table.to_pydict())
filtered_train = filtered_train.add_column("embeddings", df_train["embeddings"])
filtered_train = filtered_train.add_column("loss", df_train["loss"])
@@ -167,7 +175,13 @@ def inference(config):
df_val = df_val.sort("index")
curr_idx = df_val["index"]
- filtered_val = val_dataset.filter(lambda example: example["index"] in curr_idx)
+ # compute mask in pyarrow since it's super fast
+ # ty @bmschmidt for showing me this!
+ table = val_dataset.data
+ mask = pc.is_in(table['index'], value_set=pa.array(curr_idx, pa.int32()))
+ filtered_table = table.filter(mask)
+ # convert from pyarrow to Dataset
+ filtered_val = Dataset.from_dict(filtered_table.to_pydict())
filtered_val = filtered_val.add_column("embeddings", df_val["embeddings"])
filtered_val = filtered_val.add_column("loss", df_val["loss"])
From aa8dd7a6365934db6627e3cf93230fd0427913e3 Mon Sep 17 00:00:00 2001
From: zanussbaum
Date: Fri, 7 Apr 2023 16:54:29 -0400
Subject: [PATCH 22/63] feat: update for mosaic
---
configs/train/finetune_gptj.yaml | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/configs/train/finetune_gptj.yaml b/configs/train/finetune_gptj.yaml
index f37283b3..aa6bf4a8 100644
--- a/configs/train/finetune_gptj.yaml
+++ b/configs/train/finetune_gptj.yaml
@@ -2,14 +2,14 @@
model_name: "EleutherAI/gpt-j-6B"
tokenizer_name: "EleutherAI/gpt-j-6B"
gradient_checkpointing: true
-save_name: "nomic-ai/gpt4all-gptj-multinode-deepspeed"
+save_name: "nomic-ai/gpt4all-mosaic"
# dataset
streaming: false
num_proc: 64
-dataset_path: "data_multiplus"
+dataset_path: "nomic-ai/turbo-500k-multi"
max_length: 1024
-batch_size: 32
+batch_size: 16
# train dynamics
lr: 2.0e-5
@@ -23,7 +23,7 @@ output_dir: "ckpts/gpt4all-gptj-multinode"
checkpoint: null
lora: false
warmup_steps: 500
-num_epochs: 4
+num_epochs: 2
# logging
wandb: true
From dcc52043524c9fb7d7ede3a9c42ff9780a47c7fe Mon Sep 17 00:00:00 2001
From: zanussbaum
Date: Fri, 7 Apr 2023 17:41:45 -0400
Subject: [PATCH 23/63] fix: batch size
---
configs/train/finetune_gptj.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/configs/train/finetune_gptj.yaml b/configs/train/finetune_gptj.yaml
index aa6bf4a8..ce6feef7 100644
--- a/configs/train/finetune_gptj.yaml
+++ b/configs/train/finetune_gptj.yaml
@@ -9,7 +9,7 @@ streaming: false
num_proc: 64
dataset_path: "nomic-ai/turbo-500k-multi"
max_length: 1024
-batch_size: 16
+batch_size: 8
# train dynamics
lr: 2.0e-5
From fbb788e4517ebaab78fc29eda97336da011b50e4 Mon Sep 17 00:00:00 2001
From: zanussbaum
Date: Fri, 7 Apr 2023 17:53:07 -0400
Subject: [PATCH 24/63] feat: lora gptj
---
configs/train/finetune_gptj_lora.yaml | 33 +++++++++++++++++++++++++++
1 file changed, 33 insertions(+)
create mode 100644 configs/train/finetune_gptj_lora.yaml
diff --git a/configs/train/finetune_gptj_lora.yaml b/configs/train/finetune_gptj_lora.yaml
new file mode 100644
index 00000000..3bc20cd4
--- /dev/null
+++ b/configs/train/finetune_gptj_lora.yaml
@@ -0,0 +1,33 @@
+# model/tokenizer
+model_name: "EleutherAI/gpt-j-6B"
+tokenizer_name: "EleutherAI/gpt-j-6B"
+gradient_checkpointing: false
+save_name: "nomic-ai/gpt4all-mosaic"
+
+# dataset
+streaming: false
+num_proc: 64
+dataset_path: "nomic-ai/turbo-500k-multi"
+max_length: 1024
+batch_size: 4
+
+# train dynamics
+lr: 2.0e-5
+min_lr: 0
+weight_decay: 0.0
+eval_every: 500
+eval_steps: 105
+save_every: 500
+log_grads_every: 500
+output_dir: "ckpts/gpt4all-gptj-multinode"
+checkpoint: null
+lora: true
+warmup_steps: 500
+num_epochs: 2
+
+# logging
+wandb: true
+wandb_entity: zanussbaum
+wandb_project_name: mosaic
+seed: 42
+
From 2034f0c4792ddc6742fcb17e0461733884f93cda Mon Sep 17 00:00:00 2001
From: Zach
Date: Sat, 8 Apr 2023 17:05:40 +0000
Subject: [PATCH 25/63] fix: embeddings instead of logits!!!
---
inference.py | 13 +++++--------
1 file changed, 5 insertions(+), 8 deletions(-)
diff --git a/inference.py b/inference.py
index 25e3b508..8a4efb51 100644
--- a/inference.py
+++ b/inference.py
@@ -115,7 +115,6 @@ def inference(config):
train_outputs["embeddings"] = np.concatenate(train_outputs["embeddings"])
df_train = Dataset.from_dict(train_outputs)
- df_train = df_train.sort("index")
curr_idx = df_train["index"]
# compute mask in pyarrow since it's super fast
@@ -136,11 +135,11 @@ def inference(config):
for batch in tqdm(val_dataloader, disable=local_rank != 0):
batch["input_ids"] = batch["input_ids"].to(f"cuda:{local_rank}")
batch["labels"] = batch["labels"].to(f"cuda:{local_rank}")
- outputs = model(input_ids=batch["input_ids"], labels=batch["labels"])
+ outputs = model(input_ids=batch["input_ids"], labels=batch["labels"], output_hidden_states=True)
loss = calc_cross_entropy_no_reduction(outputs.logits, batch["labels"])
val_outputs["loss"].extend(loss)
- logits = outputs.logits
+ embeddings = outputs.hidden_states[-1]
batch_size = batch["input_ids"].shape[0]
sequence_lengths = []
# since we use mutiturn with multiple <|endoftext|>, we need to find the place where
@@ -149,17 +148,17 @@ def inference(config):
indices = torch.where(item == tokenizer.pad_token_id)[0]
found = False
for index in indices:
+ # case where sequence is less than max length
if torch.all(item[index:] == tokenizer.pad_token_id):
sequence_lengths.append(index)
found = True
break
-
- # no match found
+ # case where sequence is >= max length
if not found:
sequence_lengths.append(len(item) - 1)
sequence_lengths = torch.tensor(sequence_lengths)
- pooled_logits = logits[torch.arange(batch_size, device=logits.device), sequence_lengths]
+ pooled_logits = embeddings[torch.arange(batch_size, device=embeddings.device), sequence_lengths]
val_outputs["embeddings"].append(pooled_logits)
val_outputs["index"].extend(batch["index"].to(model.device))
@@ -172,7 +171,6 @@ def inference(config):
val_outputs["embeddings"] = np.concatenate(val_outputs["embeddings"])
df_val = Dataset.from_dict(val_outputs)
- df_val = df_val.sort("index")
curr_idx = df_val["index"]
# compute mask in pyarrow since it's super fast
@@ -182,7 +180,6 @@ def inference(config):
filtered_table = table.filter(mask)
# convert from pyarrow to Dataset
filtered_val = Dataset.from_dict(filtered_table.to_pydict())
-
filtered_val = filtered_val.add_column("embeddings", df_val["embeddings"])
filtered_val = filtered_val.add_column("loss", df_val["loss"])
filtered_val = filtered_val.add_column("is_train", [False] * len(filtered_val))
From f479a378187e232a2f95574c76fea8ea0d043eaa Mon Sep 17 00:00:00 2001
From: Zach Nussbaum
Date: Sat, 8 Apr 2023 19:30:53 +0000
Subject: [PATCH 26/63] feat: build map script
---
build_map.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++++
requirements.txt | 3 ++-
2 files changed, 54 insertions(+), 1 deletion(-)
create mode 100644 build_map.py
diff --git a/build_map.py b/build_map.py
new file mode 100644
index 00000000..400dbb9b
--- /dev/null
+++ b/build_map.py
@@ -0,0 +1,52 @@
+import numpy as np
+from nomic import atlas
+import glob
+from tqdm import tqdm
+from datasets import load_dataset, concatenate_datasets
+from sklearn.decomposition import PCA
+
+files = glob.glob("inference/*.jsonl")
+print(files)
+df = concatenate_datasets([load_dataset("json", data_files=file, split="train") for file in tqdm(files)])
+
+print(len(df))
+print(df)
+
+df = df.map(lambda example: {"inputs": [prompt + "\n" + response for prompt, response in zip(example["prompt"], example["response"])]},
+ batched=True,
+ num_proc=64)
+
+df = df.map(lambda example: {"trained_on": [int(t) for t in example["is_train"]]},
+ batched=True,
+ num_proc=64)
+
+df = df.remove_columns("is_train")
+
+text = df.remove_columns(["labels", "input_ids", "embeddings"])
+
+text_df = [text[i] for i in range(len(text))]
+
+atlas.map_text(text_df, indexed_field="inputs",
+ name="Post Epoch 1 Inputs",
+ colorable_fields=["source", "loss", "trained_on"],
+ reset_project_if_exists=True,
+ )
+
+# index is local to train/test split, regenerate
+data = df.remove_columns(["labels", "input_ids", "index"])
+data = data.add_column("index", list(range(len(data))))
+# max embed dim is 2048 for now
+# note! this is slow in pyarrow/hf datasets
+embeddings = np.array(data["embeddings"])
+print("embeddings shape:", embeddings.shape)
+embeddings = PCA(n_components=2048).fit_transform(embeddings)
+
+data = data.remove_columns(["embeddings"])
+columns = data.to_pandas().to_dict("records")
+
+atlas.map_embeddings(embeddings,
+ data=columns,
+ id_field="index",
+ name="Post Epoch 1 Embeddings",
+ colorable_fields=["source", "loss", "trained_on"],
+ reset_project_if_exists=True,)
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index a30fc8a5..656dd476 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,4 +10,5 @@ nodelist-inflator
deepspeed
sentencepiece
jsonlines
-nomic
\ No newline at end of file
+nomic
+scikit-learn
\ No newline at end of file
From df29c6252110ba7c4ed789e8292d093954b2bddc Mon Sep 17 00:00:00 2001
From: Zach Nussbaum
Date: Sat, 8 Apr 2023 20:33:02 +0000
Subject: [PATCH 27/63] fix: config + ignore pkl
---
.gitignore | 1 +
configs/train/finetune_gptj_lora.yaml | 4 ++--
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/.gitignore b/.gitignore
index 780ecb37..bfce2c0c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
+*.pkl
ckpts*
.deepspeed_env
*.jsonl
diff --git a/configs/train/finetune_gptj_lora.yaml b/configs/train/finetune_gptj_lora.yaml
index 3bc20cd4..1586b7e7 100644
--- a/configs/train/finetune_gptj_lora.yaml
+++ b/configs/train/finetune_gptj_lora.yaml
@@ -1,6 +1,6 @@
# model/tokenizer
-model_name: "EleutherAI/gpt-j-6B"
-tokenizer_name: "EleutherAI/gpt-j-6B"
+model_name: "EleutherAI/gpt-j-6b"
+tokenizer_name: "EleutherAI/gpt-j-6b"
gradient_checkpointing: false
save_name: "nomic-ai/gpt4all-mosaic"
From c0a9065032eeff8e5933306fe1a4e869e22022eb Mon Sep 17 00:00:00 2001
From: Zach Nussbaum
Date: Sat, 8 Apr 2023 20:33:51 +0000
Subject: [PATCH 28/63] fix: tokenization error
---
data.py | 12 +++++-------
1 file changed, 5 insertions(+), 7 deletions(-)
diff --git a/data.py b/data.py
index 8a0dd83f..7d61154d 100644
--- a/data.py
+++ b/data.py
@@ -9,10 +9,6 @@ from transformers import DefaultDataCollator
def tokenize_inputs(config, tokenizer, examples):
max_length = config["max_length"]
- # ignore bos
- newline_tokens = tokenizer("\n", return_tensors="pt")["input_ids"][0]
- if newline_tokens[0] == tokenizer.bos_token_id:
- newline_tokens = newline_tokens[1:]
# hacky backward compatible
different_eos = tokenizer.eos_token != ""
@@ -22,7 +18,7 @@ def tokenize_inputs(config, tokenizer, examples):
if response.count("") > 0:
response = response.replace("", tokenizer.eos_token)
- prompt_len = len(tokenizer(prompt, return_tensors="pt")["input_ids"][0])
+ prompt_len = len(tokenizer(prompt + "\n", return_tensors="pt")["input_ids"][0])
# hack if our prompt is super long
# we need to include some labels so we arbitrarily trunacate at max_length // 2
@@ -33,7 +29,7 @@ def tokenize_inputs(config, tokenizer, examples):
new_len = min(max_length // 2, len(prompt) // 2)
prompt = prompt[:new_len]
# get new prompt length
- prompt_len = tokenizer(prompt, return_tensors="pt", max_length=max_length // 2, truncation=True).input_ids.ne(tokenizer.pad_token_id).sum().item()
+ prompt_len = tokenizer(prompt + "\n", return_tensors="pt", max_length=max_length // 2, truncation=True).input_ids.ne(tokenizer.pad_token_id).sum().item()
assert prompt_len <= max_length // 2, f"prompt length {prompt_len} exceeds max length {max_length}"
@@ -41,11 +37,13 @@ def tokenize_inputs(config, tokenizer, examples):
truncation=True, max_length=max_length, return_tensors="pt")["input_ids"].squeeze()
labels = input_tokens.clone()
- labels[:prompt_len + len(newline_tokens)] = -100
+ labels[:prompt_len] = -100
if len(labels) < max_length:
# pad to max_length with -100
labels = torch.cat([labels, torch.full((max_length - len(labels),), -100)])
+ assert (labels == -100).sum() < len(labels), f"Labels are all -100, something wrong. prompt length {prompt_len} exceeds max length {max_length}"
+
if (labels == -100).sum() == len(labels) - 1:
print(prompt)
print(response)
From 5de765410a7faf13d9ffc194ab9d6e1d3256161a Mon Sep 17 00:00:00 2001
From: Zach Nussbaum
Date: Sat, 8 Apr 2023 20:37:51 +0000
Subject: [PATCH 29/63] fix: add wd + min lr to config
---
configs/train/finetune_lora.yaml | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/configs/train/finetune_lora.yaml b/configs/train/finetune_lora.yaml
index acdc0e95..cf916d3b 100644
--- a/configs/train/finetune_lora.yaml
+++ b/configs/train/finetune_lora.yaml
@@ -7,12 +7,14 @@ save_name: "nomic-ai/gpt4all-lora-multi-turn"
# dataset
streaming: false
num_proc: 64
-dataset_path: "data_multiturn"
+dataset_path: "nomic-ai/turbo-500k-multi"
max_length: 1024
batch_size: 4
# train dynamics
lr: 5.0e-5
+min_lr: 0
+weight_decay: 0.0
eval_every: 2000
eval_steps: 100
save_every: 2000
From c1ca081cda5602cd062e914bf0c34671e10e2964 Mon Sep 17 00:00:00 2001
From: Zach Nussbaum
Date: Sat, 8 Apr 2023 20:38:10 +0000
Subject: [PATCH 30/63] fix: eos/pad token + wd
---
train.py | 20 +++-----------------
1 file changed, 3 insertions(+), 17 deletions(-)
diff --git a/train.py b/train.py
index 463dd155..72e53f4d 100644
--- a/train.py
+++ b/train.py
@@ -1,8 +1,6 @@
import os
-from transformers import AutoModelForCausalLM, AutoTokenizer, get_scheduler
-from transformers.trainer_pt_utils import get_parameter_names
+from transformers import AutoModelForCausalLM, AutoTokenizer, get_scheduler, LlamaForCausalLM
import torch
-import torch.nn as nn
from torch.optim import AdamW
from argparse import ArgumentParser
from read import read_config
@@ -45,7 +43,7 @@ def train(accelerator, config):
accelerator.print(f"Using {accelerator.num_processes} GPUs")
tokenizer = AutoTokenizer.from_pretrained(config['tokenizer_name'], model_max_length=config['max_length'])
- # llama has no pad token, set it to new token
+ # if no pad token, set it to eos
if tokenizer.pad_token is None:
tokenizer.pad_token = tokenizer.eos_token
@@ -76,21 +74,9 @@ def train(accelerator, config):
else DummyOptim
)
- no_decay = ["bias", "LayerNorm.weight"]
- optimizer_grouped_parameters = [
- {
- "params": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],
- "weight_decay": config["weight_decay"],
- },
- {
- "params": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)],
- "weight_decay": 0.0,
- },
- ]
-
# karpathy doesn't decay embeddding, maybe we should exclude
# https://github.com/karpathy/minGPT/commit/bbbdac74fa9b2e55574d70056163ffbae42310c1#diff-2075fa9c224b395be5bda85544dd36572b59c76c54562819eadadbf268602834R157s
- optimizer = optimizer_cls(optimizer_grouped_parameters, lr=config["lr"])
+ optimizer = optimizer_cls(model.parameters(), lr=config["lr"], weight_decay=config["weight_decay"])
if accelerator.state.deepspeed_plugin is not None:
gradient_accumulation_steps = accelerator.state.deepspeed_plugin.deepspeed_config[
From 6de58dd1fc492cd81132496d7b5045b17f50f554 Mon Sep 17 00:00:00 2001
From: Zach Nussbaum
Date: Sat, 8 Apr 2023 20:56:13 +0000
Subject: [PATCH 31/63] fix: saving name
---
train.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/train.py b/train.py
index 72e53f4d..189df41c 100644
--- a/train.py
+++ b/train.py
@@ -192,7 +192,7 @@ def train(accelerator, config):
accelerator.print(f"Failed to push to hub")
unwrapped_model.save_pretrained(
- f"{config['output_dir']}/-epoch_{epoch}",
+ f"{config['output_dir']}/epoch_{epoch}",
is_main_process=accelerator.is_main_process,
save_function=accelerator.save,
state_dict=accelerator.get_state_dict(model),
From f5ca8bedcf0608edd08b4b933d1bb98781453808 Mon Sep 17 00:00:00 2001
From: Zach Nussbaum
Date: Sat, 8 Apr 2023 21:17:12 +0000
Subject: [PATCH 32/63] fix: config for lora gptj
---
configs/deepspeed/ds_config_gptj_lora.json | 48 ++++++++++++++++++++++
1 file changed, 48 insertions(+)
create mode 100644 configs/deepspeed/ds_config_gptj_lora.json
diff --git a/configs/deepspeed/ds_config_gptj_lora.json b/configs/deepspeed/ds_config_gptj_lora.json
new file mode 100644
index 00000000..0a578ba2
--- /dev/null
+++ b/configs/deepspeed/ds_config_gptj_lora.json
@@ -0,0 +1,48 @@
+{
+ "train_batch_size": "auto",
+ "gradient_accumulation_steps": "auto",
+ "train_micro_batch_size_per_gpu": "auto",
+ "fp16": {
+ "enabled": "auto",
+ "min_loss_scale": 1,
+ "loss_scale_window": 1000,
+ "hysteresis": 2,
+ "initial_scale_power": 32
+ },
+ "bf16": {
+ "enabled": "auto"
+ },
+ "gradient_clipping": 1,
+ "zero_optimization": {
+ "stage": 2,
+ "offload_param": {
+ "device": "cpu"
+ },
+ "offload_optimizer": {
+ "device": "cpu"
+ },
+ "allgather_partitions": true,
+ "allgather_bucket_size": 5e8,
+ "contiguous_gradients": true
+ },
+ "optimizer": {
+ "type": "AdamW",
+ "params": {
+ "lr": "auto",
+ "betas": [
+ 0.9,
+ 0.999
+ ],
+ "eps": 1e-08
+ }
+ },
+ "scheduler": {
+ "type": "WarmupLR",
+ "params": {
+ "warmup_min_lr": 0,
+ "warmup_max_lr": "auto",
+ "warmup_num_steps": "auto",
+ "warmup_type": "linear"
+ }
+ }
+ }
\ No newline at end of file
From 2827693a57051ae9e2528b2b528360047b7f8325 Mon Sep 17 00:00:00 2001
From: Zach Nussbaum
Date: Sat, 8 Apr 2023 21:36:20 +0000
Subject: [PATCH 33/63] fix: smaller bs for 40gb
---
configs/train/finetune_gptj_lora.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/configs/train/finetune_gptj_lora.yaml b/configs/train/finetune_gptj_lora.yaml
index 1586b7e7..423d8420 100644
--- a/configs/train/finetune_gptj_lora.yaml
+++ b/configs/train/finetune_gptj_lora.yaml
@@ -9,7 +9,7 @@ streaming: false
num_proc: 64
dataset_path: "nomic-ai/turbo-500k-multi"
max_length: 1024
-batch_size: 4
+batch_size: 2
# train dynamics
lr: 2.0e-5
From c11ea7dc31d9431ace1ad29bfa14cc799c0a9d77 Mon Sep 17 00:00:00 2001
From: Zach Nussbaum
Date: Sat, 8 Apr 2023 21:47:07 +0000
Subject: [PATCH 34/63] fix: bs try one more time?
---
configs/train/finetune_gptj_lora.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/configs/train/finetune_gptj_lora.yaml b/configs/train/finetune_gptj_lora.yaml
index 423d8420..325c30f7 100644
--- a/configs/train/finetune_gptj_lora.yaml
+++ b/configs/train/finetune_gptj_lora.yaml
@@ -9,7 +9,7 @@ streaming: false
num_proc: 64
dataset_path: "nomic-ai/turbo-500k-multi"
max_length: 1024
-batch_size: 2
+batch_size: 1
# train dynamics
lr: 2.0e-5
From 5caed073eedd8efe5395f022376d5f340376fa4e Mon Sep 17 00:00:00 2001
From: Zach
Date: Sun, 9 Apr 2023 15:12:49 +0000
Subject: [PATCH 35/63] fix: topic model for embeddings
---
build_map.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/build_map.py b/build_map.py
index 400dbb9b..53741077 100644
--- a/build_map.py
+++ b/build_map.py
@@ -49,4 +49,6 @@ atlas.map_embeddings(embeddings,
id_field="index",
name="Post Epoch 1 Embeddings",
colorable_fields=["source", "loss", "trained_on"],
+ build_topic_model=True,
+ topic_label_field="inputs",
reset_project_if_exists=True,)
\ No newline at end of file
From 8733576664094de524e54b783ac3c127176471d5 Mon Sep 17 00:00:00 2001
From: Zach
Date: Mon, 10 Apr 2023 02:14:20 +0000
Subject: [PATCH 36/63] feat: evals on new gptj models
---
GPTJ.md | 17 +++++++
configs/eval/generate.yaml | 14 +-----
configs/eval/generate_baseline.yaml | 14 +-----
configs/eval/generate_full.yaml | 14 ------
configs/eval/generate_gpt4all_gptj.yaml | 4 ++
configs/eval/generate_gpt4all_gptj_lora.yaml | 5 +++
configs/eval/generate_large_2.yaml | 15 -------
configs/eval/generate_large_3.yaml | 15 -------
configs/inference/gptj.yaml | 4 +-
configs/train/finetune_gptj.yaml | 4 +-
eval_figures.py | 12 ++---
eval_self_instruct.py | 45 ++++---------------
figs/perplexity_hist.png | Bin 15322 -> 26205 bytes
requirements.txt | 3 +-
14 files changed, 50 insertions(+), 116 deletions(-)
create mode 100644 GPTJ.md
delete mode 100644 configs/eval/generate_full.yaml
create mode 100644 configs/eval/generate_gpt4all_gptj.yaml
create mode 100644 configs/eval/generate_gpt4all_gptj_lora.yaml
delete mode 100644 configs/eval/generate_large_2.yaml
delete mode 100644 configs/eval/generate_large_3.yaml
diff --git a/GPTJ.md b/GPTJ.md
new file mode 100644
index 00000000..670869f5
--- /dev/null
+++ b/GPTJ.md
@@ -0,0 +1,17 @@
+# Inference on Training Data
+
+
+## Run Inference
+
+```bash
+torchrun --master_port=29085 --nproc-per-node 8 inference.py --config=configs/inference/gptj.yaml
+```
+
+
+## Visualizations
+
+```bash
+python build_map.py
+```
+
+will build a map in `Atlas`, one using the internal clustering algorithm provided by Nomic and one using the embeddings generated by the finetuned model.
\ No newline at end of file
diff --git a/configs/eval/generate.yaml b/configs/eval/generate.yaml
index a29f2b2a..b06137d6 100644
--- a/configs/eval/generate.yaml
+++ b/configs/eval/generate.yaml
@@ -1,15 +1,5 @@
# model/tokenizer
-model_name: # update with llama 7b
-tokenizer_name: # update with llama 7b
+model_name: # update with llama model name
+tokenizer_name: # update with llama model name
lora: true
lora_path: "nomic-ai/gpt4all-lora"
-
-max_new_tokens: 512
-temperature: 0.001
-prompt: |
- #this code prints a string reversed
- my_string = "hello how are you"
- print(len(my_string))
-
-
- My code above does not work. Can you help me?
diff --git a/configs/eval/generate_baseline.yaml b/configs/eval/generate_baseline.yaml
index 7e8aa9c0..d409d3ab 100644
--- a/configs/eval/generate_baseline.yaml
+++ b/configs/eval/generate_baseline.yaml
@@ -2,16 +2,4 @@
model_name: # update with llama model name
tokenizer_name: # update with llama model name
lora: true
-lora_path: "tloen/alpaca-lora-7b"
-
-
-
-max_new_tokens: 512
-temperature: 0.001
-prompt: |
- #this code prints a string reversed
- my_string = "hello how are you"
- print(len(my_string))
-
-
- My code above does not work. Can you help me?
+lora_path: "tloen/alpaca-lora-7b"
\ No newline at end of file
diff --git a/configs/eval/generate_full.yaml b/configs/eval/generate_full.yaml
deleted file mode 100644
index 972286ae..00000000
--- a/configs/eval/generate_full.yaml
+++ /dev/null
@@ -1,14 +0,0 @@
-# model/tokenizer
-model_name: # update
-tokenizer_name: # update
-lora_path: "no-lora"
-
-max_new_tokens: 512
-temperature: 0.001
-prompt: |
- #this code prints a string reversed
- my_string = "hello how are you"
- print(len(my_string))
-
-
- My code above does not work. Can you help me?
diff --git a/configs/eval/generate_gpt4all_gptj.yaml b/configs/eval/generate_gpt4all_gptj.yaml
new file mode 100644
index 00000000..496ce25f
--- /dev/null
+++ b/configs/eval/generate_gpt4all_gptj.yaml
@@ -0,0 +1,4 @@
+# model/tokenizer
+model_name: "nomic-ai/gpt4all-warmup-lr-epoch_1"
+tokenizer_name: "EleutherAI/gpt-j-6b"
+lora: false
diff --git a/configs/eval/generate_gpt4all_gptj_lora.yaml b/configs/eval/generate_gpt4all_gptj_lora.yaml
new file mode 100644
index 00000000..f27feb09
--- /dev/null
+++ b/configs/eval/generate_gpt4all_gptj_lora.yaml
@@ -0,0 +1,5 @@
+# model/tokenizer
+model_name: "EleutherAI/gpt-j-6b"
+tokenizer_name: "EleutherAI/gpt-j-6B"
+lora: true
+lora_path: "nomic-ai/gpt4all-gptj-lora-epoch_1"
diff --git a/configs/eval/generate_large_2.yaml b/configs/eval/generate_large_2.yaml
deleted file mode 100644
index 5b909905..00000000
--- a/configs/eval/generate_large_2.yaml
+++ /dev/null
@@ -1,15 +0,0 @@
-# model/tokenizer
-model_name: # update
-tokenizer_name: # update
-lora: true
-lora_path: # update
-
-max_new_tokens: 512
-temperature: 0.001
-prompt: |
- #this code prints a string reversed
- my_string = "hello how are you"
- print(len(my_string))
-
-
- My code above does not work. Can you help me?
diff --git a/configs/eval/generate_large_3.yaml b/configs/eval/generate_large_3.yaml
deleted file mode 100644
index 5b909905..00000000
--- a/configs/eval/generate_large_3.yaml
+++ /dev/null
@@ -1,15 +0,0 @@
-# model/tokenizer
-model_name: # update
-tokenizer_name: # update
-lora: true
-lora_path: # update
-
-max_new_tokens: 512
-temperature: 0.001
-prompt: |
- #this code prints a string reversed
- my_string = "hello how are you"
- print(len(my_string))
-
-
- My code above does not work. Can you help me?
diff --git a/configs/inference/gptj.yaml b/configs/inference/gptj.yaml
index 8c20efac..8b744fdb 100644
--- a/configs/inference/gptj.yaml
+++ b/configs/inference/gptj.yaml
@@ -1,11 +1,11 @@
# model/tokenizer
-model_name: "nomic-ai/gpt4all-gptj-multinode-deepspeed-finetuned-epoch_0"
+model_name: "nomic-ai/gpt4all-warmup-lr-epoch_1"
tokenizer_name: "EleutherAI/gpt-j-6B"
# dataset
streaming: false
num_proc: 64
-dataset_path: "data_multiplus"
+dataset_path: "nomic-ai/turbo-500k-multi"
max_length: 1024
batch_size: 32
diff --git a/configs/train/finetune_gptj.yaml b/configs/train/finetune_gptj.yaml
index ce6feef7..1ff6a7bb 100644
--- a/configs/train/finetune_gptj.yaml
+++ b/configs/train/finetune_gptj.yaml
@@ -2,14 +2,14 @@
model_name: "EleutherAI/gpt-j-6B"
tokenizer_name: "EleutherAI/gpt-j-6B"
gradient_checkpointing: true
-save_name: "nomic-ai/gpt4all-mosaic"
+save_name: "nomic-ai/gpt4all-warmup-lr"
# dataset
streaming: false
num_proc: 64
dataset_path: "nomic-ai/turbo-500k-multi"
max_length: 1024
-batch_size: 8
+batch_size: 32
# train dynamics
lr: 2.0e-5
diff --git a/eval_figures.py b/eval_figures.py
index e1b50bbe..f7fca1c6 100644
--- a/eval_figures.py
+++ b/eval_figures.py
@@ -6,18 +6,20 @@ from matplotlib import pyplot as plt
plt.figure()
for fpath in glob.glob('./eval_data/*.pkl'):
parts = fpath.split('__')
- model_name = parts[1].replace('model-', '').replace('.pkl', '')
- lora_name = parts[2].replace('lora-', '').replace('.pkl', '')
+ model_name = "-".join(fpath.replace(".pkl", "").split("_")[2:])
with open(fpath, 'rb') as f:
data = pickle.load(f)
perplexities = data['perplexities']
perplexities = np.nan_to_num(perplexities, 100)
perplexities = np.clip(perplexities, 0, 100)
- if 'nomic' in fpath:
- label = 'GPT4all-lora'
+ if 'alpaca' not in fpath:
+ identifier = model_name = "-".join(fpath.replace(".pkl", "").split("eval__model-")[1:])
+ label = 'GPT4all-'
+ label += identifier
+
else:
label = 'alpaca-lora'
- plt.hist(perplexities, label=label, alpha=.5)
+ plt.hist(perplexities, label=label, alpha=.5, bins=50)
plt.xlabel('Perplexity')
plt.ylabel('Frequency')
diff --git a/eval_self_instruct.py b/eval_self_instruct.py
index e0dbded1..e05a68e4 100644
--- a/eval_self_instruct.py
+++ b/eval_self_instruct.py
@@ -49,28 +49,6 @@ def eval_example(model, tokenizer, example, config):
input = tokenizer(prompt, return_tensors="pt")
input = {k: v.to(model.device) for k, v in input.items()}
- continuations = []
- tokenized_continuations = []
- trajectories = []
- for i in range(1):
- with torch.no_grad():
- outputs = model.generate(input_ids=input['input_ids'],
- max_new_tokens=config["max_new_tokens"],
- min_new_tokens=5,
- temperature=config["temperature"],
- repetition_penalty=1.0,
- do_sample=True)
- decoded = tokenizer.decode(outputs[0], skip_special_tokens=True).strip()
-
- y = model(input_ids=outputs)
- trajectory = y.hidden_states[0].detach().cpu().numpy()[0]
- trajectory = trajectory / np.linalg.norm(trajectory, axis=1, keepdims=True)
- trajectory = np.cumsum(trajectory, axis=0) / np.arange(1, trajectory.shape[0]+1).reshape(-1, 1)
-
- trajectories.append(trajectory)
- continuations.append(decoded)
- tokenized_continuations.append(tokenizer.tokenize(decoded))
-
#compute the ground truth perplexity
gt_input = tokenizer(gt, return_tensors="pt")
gt_input = {k: v.to(model.device) for k, v in gt_input.items()}
@@ -101,30 +79,23 @@ def eval_example(model, tokenizer, example, config):
print(prompt)
print(80*'-')
- for continuation in continuations:
- print(continuation)
- print(80*'-')
+
- return ppl, trajectories, continuations, tokenized_continuations
+ return ppl
def do_eval(config):
eval_data = read_jsonl_file('eval_data/user_oriented_instructions.jsonl')
model, tokenizer = setup_model(config)
- all_trajectories = []
all_perplexities = []
- all_continuations = []
- all_tokenized_continuations = []
for example in tqdm(eval_data):
- gt_perplexity, trajectories, continuations, tokenized_continuations = eval_example(model, tokenizer, example, config)
- all_trajectories.append(trajectories)
+ gt_perplexity = eval_example(model, tokenizer, example, config)
all_perplexities.append(gt_perplexity)
- all_continuations.append(continuations)
- with open('eval_data/eval__model-{}__lora-{}.pkl'.format(config['model_name'].replace('/', '_'), config['lora_path'].replace('/', '_')), 'wb') as f:
- r = {'trajectories': all_trajectories,
- 'perplexities': all_perplexities,
- 'continuations': all_continuations,
- 'tokenized_continuations': all_tokenized_continuations}
+
+ name = f"eval_data/eval__model-{config['model_name'].replace('/', '_')}{'__lora-' + config['lora_path'].replace('/', '_') if config['lora'] else ''}.pkl"
+
+ with open(name, 'wb') as f:
+ r = {'perplexities': all_perplexities}
pickle.dump(r, f)
diff --git a/figs/perplexity_hist.png b/figs/perplexity_hist.png
index be3780b07bea2c62e34b20e68cf763bd3415de4f..08fc18e2e23605bc176c07c6adca037afe318d9f 100644
GIT binary patch
literal 26205
zcmeFZc{G;o-Zy?qB$N~x%g|&fQyGdRnTiIpLP#W;OJq)nN>WKm5oIhx88b#GG$%sF
zl*}Y#d_RYK?|biO@8^BizSp~cYdwGbTJ7DfxUTCw&*S)hKhyUJIj~QC2_rWnMNvz3
zYpCc@)B-<>qG@BG$KOc5YyOHqww_WoI;HDqeag+k#fsW%amvZg@s!>1BmAeWTuvT$
zbl5B=D<-*--{#aQr<2>n#qIz61~EsMW8zJW4CnD8i=8wMo}?%i3-Ujjq#cRJDaxvP
zx5`dE_vrpFZboc|b23Ao;Vlo+VuLkU*;Jkhamy<1UWJ1@-So{lzRjA{AC8niY5m%5DTC~>
z>ODuLqL@|iYb`f-ikM$8exVzF|BLR1P0CZho`+dkYzGe>
zl#r70R}s{9-HPXTe_@nX(1!hYODiksxw$){nJg(U7JN^VVfx*a8h$~+I5(%MG5e2?
zj1B9ok`~godE%!R40Z6lXlO91s9T0hlo}ZsIgoyMd;54ErzBN;J~GieG&Gcp3c;G-
zk~Aw`_j*MuSt^2+lm=NbQp|qyU+P+>(?sv^Rb08zh(EWsqJl0`+>D-LV`DR~zAoHQ
zDawNlT@bJx>H_e;#oV)AO`CdnFjlZC?JMhAV
z3r^z$rV5^8T6GVWX-mhZr-yBG?NKr|HpbUQPqC-o}F%>;A+}*f1
z!R8WQK3RumziZd7+3*Q!)tQAZ^)pN}IU5?f78w9O}zthz%ls-+L
zVVvtwR6DRf&kISKOwHWvmtOtmO;z@ZdWlor2i(t`
z*{P|?l$n{?-CDAP;@`e~Ra8{e(D1Of^Mm_K4OZ7Ay`Nq0>g2552n@6+^`mj=tzN;#7Lb%A
z)b4FzA#f1OJ;W$evuZ7)<@FZJ2z`npSJ40j__E2Q({L-$aPukT232V+m5n8CQdmgeiKvE_O>>^mLktJ>(|rc
z@|BI(tXZS+Ku1nq-g0L0*V_*tcAYx4W#gtz*k1T;VQo!MWjubo{6^Hxn+#KOr@EJ2
zUcLRon;mn7TANr_E0o6WJZFC51e3S7cV0n(Wqq3Ip!cO?Z|(*rCT<`bsJ}mycF|HL
z!Ek7e3bMP6QBiDR#M**G{##+_%3t*zI4j1Mf?wQCnA
z507PA>Dd=AU$TBGQFt}|tKKYr=Q;W^)5P;Ee3j?C|fb#LCj9gmNgce?*j;3qW|
zX~WPvG|fex<2?1fJv~Qo6O2lHyxH|t^qf~KxckMchaDO2sF0A55Xybkd-qsUa`NI$
zo5)5M{$jRo-%_IqV8~2@CBu^ytx8*WORqZ0z4&
z-?=zCdW!P9cTd2NsH#3ghY%JP=9IQ&yL9Q21oorb
zwgkh78#fr+2R|n3y3bB;t-QSIqMsiXt#C%i#_d;+{`62eOGBEl+2#T_meZ$Cmp#6Eb-7lQ6kV#pqq0}GWt|6_w=SfoKl03X`};4A?=)@@96my=O$n-f
z_39m#$wTAZrJYq_zWuI#;s=e4O5eR3zB75~iS-i2slLta-UbFMr>Cbgj=fne&=$S*
zBt15JRaLt-{*&JtA*NXn8TxXy*ijt-1xUS_Qq7{j}M6v
zX`#WvG*^;tHD*}QHf9~Cb#!#BjoHDsdi83wG$k)D1u?T?W^wU!#vk8XpBycnbBMEP
z$PnoH^oWUaa&ajeZ}km(a_mjq>HbEe%p-IzE-u%X`gL(BxJ%%31-^?{xBONc^`f?x
zZ9vJ#mHuRVS)hch?D_0$Y5X%C5Ka5pT6FstZ{Dce*=@WfW6y~CjZCl!?6x?z@cQ`pbOWE<*GVqh|NOx7h6c`e@7`(b+2fD?
zLoO8sbnT_-x3E4P`YoOn_3smoKlp8&Ut%^+Z5G06#xJb>qejqa3>^otbg8
z?`!M#mH~>j{rC}rn|K7UbkJ2R{i?>=iQy_f?Sj+xStsh378e&gIXRW0>$c?G2@0Yo
zD>xuv;g;j?7m}aH?hwtA5ls2KRw_y>p+_(!wXjeDSeIW`HjTN^5g%NxTr
zzp~Nd+Vg!pJUryijI7;ux-VEgZ1uB`ANA_G!Zn7zt5qE6GA*5+d8J#ba=$Xiv32R9
zMT;658wIy+<=SjrTM8^t)BIIgMaA(v&6VJf2B%NUkham+XNT&)h?btYG*0!Rp6N8i)ah?J6jDtcQdc$MNtqK$O2XIz;*6JPy@5RW2o
z52ORIl~cmpulec7LOMDD5s^LaW=2NAH#QlQ>n5vXbkr;HX{wH%9vk54@WilXeixS5
zIXp_+3(L(|-HipxHP1)YC$K2Z5;DH+TQhHwmUeD0qhsckTb7oVhBc)$*iyv0Vg)Ubz`aM|;jBP&
z(Rc1-`pkHA|M(%WZ5ubP5E~G`mHB~O|DT~5!;{@0904kVI{{e~6cl1pQ!g4kIwT-1
zt#7mgCGG3CZ_0wem27KiQXX=$;(2^4dgLWNPm4MEdmu(Y%!cWLX^tuFR)@$sbS5Zq8!w!nL8EW&uM
z@AKwMoRSOF)aL0K@c4s_G3gVZ2u{ImJmxA{s8){jSX@LwdN+x*m^hk|3r>sM?Ndfr${lvu1hD=L2_aS~NHZPA8Z=$hp
zU*%caMNU7z?OnKNQAuXWiTbpA_#fdGvlE>xpc#u2_s58RXhkV{mvvkhwCD(`{!KY&
z!ON@U=_#R2n^bXQhlkpCMQ?Luth&6a{8w*1>#}A1c6=*VoGbR8it`*F(9qISK@U)z
zXrs+n6kM~0Ao}OeRW5ODrk|S`^bNmzcXnT=hDy)bM8A-fd#Q%$%ImK!-mFEsaBOTW
zMM<3QGumkSRL{X%TbmU>AAbAxMKCYnx=5|{yPTXPl$Dj+@RGM>9d8`BJaIyl<6&hO
zU)UP;%WVaN#iA?LU(W`$2WPNr%vub>;B@k2xtZ@wASigmad~jIQ$OC*Q5p74G7v&I
zrp5=W2s1izWI>3^f@HnPeXi35ck%4(T8m8?zToB){7sPKi4!L%zq4m)if1QJHG*|Lofu3cO4>EXe1;8ldFzpbmIqeuekaMU~jk%62l=6;H9@&+X1D_u8Q2Z#<+S=MKuD^f(z6X5^
zGK$IKUH0kIC-5ngy8wcn&G{$a
zzIzvLJb<-TTB>B%l(VEJQoO3CE|qLQP}TC%QVIZkPmJ~9Jg0W~m|Kq+1Ox;?kz@ljNq8Px>~xu?Yk
zq&sNRntFKVu<)ByH%-m8%W=CFP-KnaM|O1OtP%WGL22kMi2?)Fj1)7e1&{fgUb?DMg*sfQ0YPW=9D@#gM!o6nEeQKXx7e<_fKD6*ifBKsH@
z7Z-MeZVC%ftI@%OFE^cpE|A;mbFr*US-`8Zsw%R;+r%V#*##_)fJ>K{A&V`bva+)S
zMih|)j)J6jg5D=AzHuxeNh}I{n
z|CX%dWqfSm>8BMH6%l9VMjA{_O$nTBF7`I=p85mPW)5W!4-F}yW(2YD%~tvxuKM(8
z><%uyu0z;7&epa(XbEAYg+)b~z~lEHJP?2=1PZeWi1yNYhUtbnCbLkm+fmm*kIEBcd=7ov_aRr>eaIYl`3Tq-`RnjC>H+ds%(&LZ86#Aq{V|Hv;b!66F)?A-pwlzst?pxepXKXTaB?!>Ita*+mS*3uFS-W;4;=CX%gSx#8hQil)o+Z_H_63q;+?m`3mAl
zG$?XI)~$O`n;k#cfz=8qs|-9h`};c|-bX-K_{Sp$MZvi0axA0&5hm-49M!vyPZ)pZ
z=Qq$3FqKuHj)Bxz8i+CLs#T%baPA%+M^Mvl%DahCFHAWYD3lV?9tsN!uiUv4+TCr_
zzO4`{#hSHi7t+ws5Q_ZRs-`7*HP`^}R+oh=e2OX;S8f`5r_OipUp(kIT61rn}#@)RQX!OO47fRrc?;al9Tzf}8xAw;mVcmo_Xj3+y9&!<`
z_RuJE0X7+RcwrZ6Br7VKTEXf5w%%SQU|u|W^@d_xN<)F0xXNqVU%&pvt5@u+qc!U^
zp=3ZKI1AWueKGrepVNH}0t)r}_wQ!`CQ?CD9PWO%=H{(i#dXC{0N>`HJeL2VsV)>!
z13+i%y$K-P~wDjgcIfL16aM2bG
z85V>cD^7ODGp1Y*PVyi)JzLs)@|0GL;!Jk0{Hbog+@cE)u+U%B)OfbO|EPS1rE9&Q
z4*g%n_bF!~+bj-I@y8<3kBmFu{7k#gQgYk2DXXh5LV0lE
zdnIF8$&9|c3*WbN>C&5259d
z);@G-veBL>B>x52DJzKz3He1va=_BKf`UW%HCU;f?`&a)HwY{WDUmBpo}O9^2{#}+
zJ3FK0#R~>B$GlTNmK&rW4u!s9(N!Jc=xT!Zt-?L?vv102@2I%wgmxDb!?#+|OQ_xf
z=ne9{5*jj4`W4hMK*%*RGAlubx*M}ar)OruqoRHr%&cGyIjb_3!*_qT|DU9j1KJPF
zh?brISno=ZGK>zdN~f8g8Z5Cmemn%I9){lH
z@$Ddxb!ha5?#DD!9$Y^s6p4923$=k^$&v~nssPKaPHhzQ+?mSR+gpzNU%9gE5|8Y@
zWRH;R!%t&)Mo$lIxx2|OCMs$%)<}D{`*4R;
zlUc#(CFp)O9}+?{v|w%Uc@3RC@LjW|rNw3N=g*u048av}GSZvH_u?BSB?j)!ypP
zA)zhZH8>bH`K#CRL&DyS%*>s~j|<t*Q;e=r~?M+W-IIb1io9HX3~;%cBS
z$L2AB83xcrY$}?2rd_;SBkSuf^0>2B^X>Fd_SPp6@b}#d0)^d;Cd_t
zc*ol@tz5aXY}d^@(a}Tb<(+5_ggqHNG&&m{%_~6lK+A_lOWYNJo_D~m#Ow2&{k;L+
z&f*OlHXOCKCJWo;B4M56ZW9CZoE|TkBMT7}o?c#Fo-}h*uJRY*Yg+Rjfdzm8
z5Ro371lRCARQCBY))20AEW;qcrwq?=hqY_h#_qnh{w6%m8{%eZP6ZH$wW8%%42%>ClRrH=)&)1%CP$H~(P=18dB?
z!-b0jG!||<)lD7l->IS^?o59L)kxzg78U+f34-?YKYy8`f(u>7%`H+0s$Z3&pZYOj
zwd;c(=ahYN1}fE^C#jI?d$sCp`uE*t^5UUhZ3Op1kH|aueW{j~R$F&>u)wY$ejyYeuLr0LQ%dIr|}5*orbJ-TU!y_cBuUKd#(nH+!3;-)(8snhiiPDZ4?(0
zDg^~6>krj{+;r#=DWX@qH<_0Bta^1zx*TOc(|z~|cyInI?)fNu)XM4}K)XoF)$aaK
zkacWI@SFuG+YBiCa&mI9Zi6ksAt7f=N=n{-LN|i{^!oK{f?44jFI%;03F;zzLHxcM
z6-7eQI(EG$UniDm`;Q-s0jvRVJ3-2?Xsj(`eg%38
z$7dPTEx1h(Z{qY*4It66e*%`|&Ck!rcj5O)cU=*
z+S;C-E1N`DrvWVtx_o(QUw=O<7ngAE41Bh>uU{A94ULSAJ4wy6^u_ijgdZ0K+mseE
zP<-<(#ou37fX$UbD!#MLRoTVmvB_;9T>=51FJ!>mLQM*EZT0%O1RN!BFa+{^CvGZ5
zMQPEit5->xbpjP(_rp5>m7NW}iyOT<5Eeh+`gx){Lf1+3{+s;C_sv>vdk^2|0~+
zYhPL=zmlGwzAGnOV{MmM{l||1H*Ty%(&GAn_=XLO;?+XSWLv#xy}f6odYKp*0hI!8
z-sHTWkU*FfCtSzZF+0*v@ugS);bB5Ebsp|eC6|aI+m&-i#-1DM4lHgKtQ(Y@c&Jp^
zpq1F>=HP<0wh(>vlV|ad?;aV;5))>g^O-&kKnWwpJ9XvC6+fVZj?S(gsqRSR#Ep^y&}FOj2CTqY(aUvC)yA;Fbi9Gru*S^Y>{L`3=M(X}ooPrggm-SA-Y>dT{~1tS}Q
zh3+BxMhx(&b+V{_Fhp1*_Ni7dI=PouVhczrKS_*^qU71+nzMeG4j?on*p_5&3Ys
zrSMGm`v**vIiN(6ajqCiFW@1TJ%7Fs2wx|8^wH#8SEQK}6f7{?XP-ZFQD3ctNt4Dy
zu>>-pD3U?}GKM^(4m}WNkcGLqA8eH}6eVzO^(WS^4mX(rlaRs#$u%DO9OZ{rR$f=9
zZ*y{T^flbUiIgZb%=$;P5Ht0i-&p_92TaIH)LDQL!BCIS%s}b
zL)Aje6E#f3BHmCvZ6>h1fLopxy08P02{dnP6n-N1k0g+>gS~wyU|RjalZ=dI2$#5s
z3%dSgb&$!^{x|K&xv9`8`kNq6N$(5+6WIaBpVECHtaOuSptY&vVdH-;=cwbee
zK9L(0MV8f-8#fa4TB!^)y8KxPhQtwn>e`E_fj(3{0*jj;%h)%zK@ovbvNKZ5^Z?Ky
z3Ya+<5&GvMJXv}Q|8d^Lw`69F2f!RbvBjuc7QUg}(pRUpMt~_#PkXgDc15P+2DUY3
zpP&v$r}a1I|2z>U{qLG%S$#FE;q4xyrSOKRLb
z+!E&Hl_BW(9pe6l3vDu0(tJkf`popXbM&=>6RqekZLHNW%pUf2JrYI{z;_y
z7}QB3BY>IRI5;jPRGd5juwMos2#JL}3&ecD7#Hu@u_HgdZbOWhKS=TsEa8TyuA;yw
z0lITL=B77;S9eX4P!}|}3n;|be|&iTdSAPH%T%}1|E_b!r$l&t6_<79dJK3cBN
z4y%}`{7~=9QJKs4HnOg8MCci$=JSGQL(tWzq6wwAeQJn2;MLx?gKy9*3lWMoPAYW8*tnF
z@W{|mDe^in|Fs7;!W8MQ
zY1;sZeQN{+cFNHq2Pif^aSM6HHQ3WYWI(av<3=F!%XoM$qB}$$bo)JGrQkk9l(@1t
zY0zG;AFnU+bmQdYG>7XT=QSbWGxN*P!W7n6O_pOz;Z`xzr}u|E`|3_SBv+As4Ij^;
zwKx=gkty30Wfh!zWpP-`DZ6SIo8F22!M1VSToh!8OJMu?RwQ?YN-hC0g(MbX?Ss~-
z!p5g5oY;9gD(c`jwu+J;-QCxt=mc~kKimKTR)s|2=EYnKdM=U~Nlk;Oe%-{Zv
z;dOsVWbxy0i~08bJCQo@?t#U{KKZhb3sYeFk+9q3_+S{jr~xt`tjMVb!N!ERBWYF5
zMj#zNBb19$!~ilJzHGB=$e^K|hT3+T0J7r~?ZW?t(JD;M?*qq>&EqV^0YU=Ab6crW8M?s09N5cZTUM#B)WNS$pU~~RvZW+7aupP6UZ(?>-
zLH8h1A4!cO!D&=*dK30A(S@LPNo5nPL*h|POy{m$`@rKXwm@7=lH}k}W6tka{EfOm
zPh=s25?p&{$F*LOzVi986?uE|mk6&2u3xVvCj-Ng6p^|V{rQO3B`#u4l{Yox$ywUm%!~g%`;X)!K~)+W
z8l^++{n@-#@LIh?Mckr5(s<9h{3yAe!_8pbF0f*0;X@-|3tM&)=fmUIE(07+`%W-AoC7U$JF*A
z|AcIm67bB>=VLJz6&Khc%&)lIk2h2yJUz`+9Yt^ah1p*tDe)ym}aWfWfCrMgCHCKkJMPg4Dh@pOUHab2q<=W2e{XFOmL1THo_-jReO5(2Dq!+!&4EnB%V2q@;oBhmc_4lIL87%&dcfH;9y
zuU!jA77HaS5kg^iAm4k5#xpg@3FZ=W37AEXgi
zZ%$+z1EueBX9pV
zs0x?hE_9#)+}!4xqBE20*t#0CC&5Y-+uBs1x)k}&Zr;Huv`hC&CaLC(h#7kg;UN@oW{GegPlC1(A(%hV%P*D(J
z8VZpoMqpOrgJ8wX8=AknIP(2+0X&FQ17t*05aM#8OjQ5MG=H{$gr9t;hZm8!2s9wL
zF+-24{DQ0Jldx%rix{C$LCeVR0z4+!WRhU9NKQ&(AY}*XdI&`K_00Y`p6!-TC_gbR
zV5O4IgGHte_%!^*t+i<8*^DpotFXjjd9{J}V{us^jS9I()^RnGE(-ubNVEb7pIicR4m?MqKv-EHJ$LS$B+N=u#gUZ(*78l?
zArjo}3qjoW3U>DzlosqUH4BS-8UOEbaaRA{8owK05YbP)eqf+L$jy_FkFUKIr4|>@
zY8`)_h|lw!+yb*Hcw!x(QSg}0!h%r?TTN11i1&BGg|}%fF3Jdtd~zRh2AQ-0LPJrg
zxwP|)KqKaCz%a`^K_c_=^LIiQarg|xd0jCfvSo|RoZ>h5ag>sai_}WQymK8sCTWup
z8%VvufDf`g`K`#I0~+c%oPG+dPBLa59^0@-&m+w;AMnZcba#g>2T>Hhssuf5(8?ZA
zyC&l>rtpU0^~-b`t1Y=0o3wg5Oz0x6U|Aa
zqJm)($EBvIsEC!5GZ><*#qW_WlAuSJkq#>rB#Iw+{>O(}AVs~&k9JmQkYti3a2!%Q
zb=YJBO2qE&?(SGH3liAJqJ$0~O13;anKBtHCz6XHbRC#t7p!Qo0;|f{Lfx_S76!p^
zjcQYaLx&hCWr^!AkCsi!-udKx$JcDv#7Jcf1Pv%q6lk01huKz(Y45_xncv$SxzCXR
zfSk3}xVp+5$#nc*qCr9@y(1oz_!-dv-DCdbyNj?V7E=Eu(k}NP3MqA9{Rfkn+wqTx
zh-j8VQSh4T&n4lpvSWxbV#S{l?ns@Cr!c0mAndo3|8{%W_B;HTF}uU2otVy#rW7?uhud{A8f7qTY?Er2wd0AT3}T0Z{9-6QaCfR
z5DLjNd_(IAJBSAXM1O&;kBeaScXoBRUZ<_!A&m~NhoBaLnPz5Yn6|q|l+(Zo0)77b
z{QVD}T|rX+ZM4YmaiSnxG*Ub9gam+s5z)sOCOwtq@P!+4BM7?Rg9qzCtrJcyfvG_d
zDU_*qN!n{Dab*M(lAC||QW>~Z1=y4jeyhl=g+*C_)RSB6WOiphPgYn+tVNF3v;BK5
zbyuUc@+!D5LyjJ`ded9rxrZkCk83YA`O`5Gj}{j}iam*!ESvcC>le|x$!*wlD63SR
z&pUvq(_fytmW^T0lQ}F%8h(I5^K)Y^UDcjpns{s^#-yUI-ibPkxKWhE=G`iDYrI+MBaeR2YjD|TG);kAT13+y@k5E
zA95Qb!@^kR^9XHvUy9x|2%)qPe|miB?>cr6mP;#6jtSX)kW`4(nOxa4I5;xICc&Eg
z2qJSiS~w6q$p%A$fNoeizAc;Rp%8nfe}9hw2v?>v&{Wa$#w>VvWA*FTl~{1$2z#OB
znS*0lt=5>%AtVa>l#r+itY(oU9l^MsKr&nK3FKQ617`JUgP6K#856mo=sn3VE<~{{-zM$1))vYL3DsaS2ao#Rqen9aeAm=e
zIs~r0Jb(dO1bNYUIz!*+uGMj4#cJ{;RDWg#ZDpE(KU}%t+H?x~UC&sf@(i3Frhv0B%%)
z{1<-hT721s`DtOb&?SB(^rND(aQX7(P8d}n(SI^Rh2FK8A~6SOHzcx7V&s@mhx8e9
zA$2`cPbprvQpDQyA%!9ZSME7j)!J${FoS8744_1WSw_A6<-;_Jz7s!(Ac~
zalrgU5I7NY?HVm&@QpK&4g|<}<`+m~-YnN}QvF(2!<37_r2Fz7vX?3`m?41;zY}FL1du(BJeRTM_${EsCptF%ldOVa2#QZUyx?KG-??0)K|2BV(
z^-$+-%RzK2WFge}kt%aV+95YIB7$AUv1Ph)2Hx*}Iej~f03cyN!Z1+55nnLLUyd3=
zh9mxQc!A!;Fvh}IlXa1>H
zFYsXmUL=Y9RFbuJ0c8WCkBN==+7ZmI19zmxlAnsZrP{xs4ZT7FVa8}V=LJ~rL>z#S
zMNDO{>EWsru`u!)jk*N|e&yxmaB%kmVRPNOC0;GMX;TGsKldf%;16xO0|ivlqer1I
zs>wJiG&hWSpF@`gwo31LPdpP7e23bI+b-$jktb5l4c8OY1{$vT0s4m<5kMHi^M?*3eB$o|$>TRC0B^0ig
zk6LpbV=jmQR|uwX;r_x*iF0f%CgD@OdZkVQl?TgSKk`855UV&I-l(Ik+Y)|?j3OfF
z$wUqVAW(?95)lyyRL5?pMxO9Y@)zbdNYHPyPZ$CYl4Xz8a~9JLWX2b{davOMjuf$Q
z@*anD3%E)_Im;lp!=U%Sd-rZeAwB~96MYLSyqb#ixyD!2ER+E`{dtCtWw1nwmN>
z)~=(iKmYIN@$Y1JVOXSg1WdaWsZu(2+)>uLjkp*8B!R(f)Wx31`?LxUJ821;*Dtt?B5l{tnJD*|2eOsd;(mQIL}~Br6esU
z59ZcxJXd%GT`eFNB<%x4M@IAK&z5=Tse?5McCmIGz9EL>wkb{z?P^(w8ATO68a%EW
zRRG4gPAlt0q(FD~>tCc83iBk_ii&Ffu^jY0@xa>l@0eLTei+gLRuC}^AwI;%rF^WP
zr-;o3n@ufvQlHj{#PO1KK#vz5`|&}2ph!jVEC3~8@wlUPY-n97dgqc%DL@T<*TwVK
zbSOK2KED<$5`?D!H}&9;90cZ)p6|iu3ASC2C5^BaVjL>Y&XSO0DGKB#Nk4TZ0wrQu
z*EH6qi2QvSP~DjR!YDpbLI7XshdSEd5?>AC2eF~p4M}`XuPE7+0=s#Rlvv=cf%R9A-9==vq;>>os1w1df2tXn-8WfY(ien_`R-2UzP1F9<~e6)9xhpQre
z5la-+_y}&u{C2?!EF`$OO8&IopPTW{g2Op>T)@?r!2e^>B1zU77$yWNZwDyoKk6x0
zDQXx$Cc{a598u&@=!$XhjL32TEz&n#O+HR`qQPcAXxt=wT|CjjL?W?Cm1j`^z6H-n2!4b-rSstRo{$T!Fgzznh!;-UeNXIC>
z80KS)v;HtRTsE5U*1CFcgU-HQlD$N-g#9)<`4{5V0K*2Fzhq`p*93%a^~nVehQ~1o?=lJ#o}d<=n-*k
z2S{+~Kk4EE8=>ZlA+oXf?1A+t;i6AY;b-IaXj{_8YOOEFkAqkG@b2*8UBbQPit!)@
zByhF~G!DiZXl^7M!p9fmGM*eCPYb$`&p{T8tFGmKnlP{hbgbv_yctmX+Mg=I&qCPz
z8G>IJG<*(5OOB4g4_)Hhu^KX2A?;04E3&wG2zFzj=H0Uq>}^3nI4Z-XYmbeM4H3#+
zSp4Uy!%MIgtTV`@h#wEN|J2csQF_+NU}e|eMt@4wuu&!H5N{^Lgpd(N4JVJ(Z>^s?-O8-DL%
zdSHu-7p!Z?2*ZDVU>YB7eC57NW(j%P=l6*38q$i7FO#Ws8)eVgYB;H6!NXT!D)8&>
zgEATVclGQ!ze`(pIv>N6lx8^gpT@`uzn)CzQD%bzEF+_xJd8
zQM;dKOLnpA0mqEa%F+JIp=q>jz;=un4Z!d_05B%KXkB`(q`1go45%$iu;9Q?^IWdVyut6xVK&`4VEy0k}!-G%g&Af)`;4Trxx)f#Lv$k$5XEDRDVrZV8GlkhX%cmnV)^|2n_B~C=&@)*_n73vkL4mrRq
zqZXSK;_h>DfLQ@C+5bUn_u?!huWJb3Q+|j+HM_tDurNJyW8*>8pp|;I_H(D>+d@!;
ze(GFrxHdJ#p?mh><``kSlfBl_Wism;@37}&Zk;T`(|xTIXiql$a_!etaVqj6D71{}
zkd;&a;03wst0|#~;+w-@BjLt(uU%X7P&5n;1rlP{-x#(KtvR9zs9~%;Jfa(Mcv3mS
zL`y;T*$w8=`fpMOFkB33oH8_v>m%oWAf*UV2vXn$Naf5hIEZY7!6{lE9v+BS>aMCQ
za3l54q7IQFK}`BFQ*hyxL0}U0$6Q9YC#h%m=?O`KTX_||I1vp`5h2r8Nqasv
zPz^Z!;>FR&^=XkBYmG2y3NOScP#>?$ntWFXqae8L%by^}sUdBKx>6B(ePU^t_zUfZ
zguvy$ggXDM4m*qR6A@wmGG;NaN~s9`ed^+0zVu>d)@I*RbSybzuX^Ane4rMVzq)gD
zG~aAV8L$xY;5u`GM>=I5ai%c-#aFp$`})Vjf)vdIC(h8lNqv|ONmB2H&ueuvY&%-t
zMS=Dh(=uf3!a!sm<7XD!7?!`3uo#k*P}$?|G3ps;|M>2XM^}pNUW>kaaUe?lHD{5d
zFm>HCT5GSjo|fwNtLK9?zF;yOcj7-CMF~4Api>s=zX7eq#^7zMZkVI~3kRLSg9a~r
zZW(<^zcQoOwMPH>bT_vopFFGZUwb@%e_G!uozO9APVb;u;2TX*YxOu$m`XC_P31mcM8mzscv_3oJkl2ag(==)sYs~iFk8V
zFG$Ml!{v4w{R4}`-T(TvZkA=dD`aOmTlJIihiLDhCeWPwz{8BtQ+A6griOXr^I~l(
z_6{g`B;>ubtgzQHs1mi))tn0|li5FbP{i(oN{D)elHu@^6EbV<4>YK4?!R6pb9d~E
z=>2rbFcY4gS_r>KfHhr}+({1=R
z@H>^UiB0rm-_r|x+v6fmn^^EzZr@blz1^@QT)|+H@wUxcA)l{WkDfDKn9@;h{soq0
zqd2$M;`WzCj-qxSKL50=pxa*jAme5JD_!=SVP^@$hRXJWLA0xTsS}3rpP$@eY%&k-
zsQRkMX0^IC?$U(ay5cHEJ9e3aQxYafxK^{NLaj_>bf2$uqHz7BaIW}IFi4?5dBh^c|>?1
z=RJTP=$Ufkg|r{A_M4U>{DY95l2OB)V#c9eyJ*p0a8#E`eiM>ZP%jR+$K0OkSelN3
z?mfxFF>%Tjp}{S?b|-g~&|JJ5X`o?7M|WY$!(c7$BcIT$@37vk;V%VqY@!32y74N`
zX6r5tF7O!%Q*qv|O2^pr@x72igBn*L-EaI%LSFhY)oR{?j)I&R5$C>jb3(IEzi-zs
z&H%<&>qzq&YaV|KjHcCu}?{HbGlZ-
z#MHfnt1nbMMj;&|qgp{GhdMjgY=r8w4We*htM6PyqlUTUMI7e{4H$g)^G#~?|bN);TYg^kx2iGkiC(Qk(
zO<;72PNlEPxBc56dY|NO-@K_ep_sbjUCY8aV17i~=K(c_m`NjZ+e3hGQUYr~Ge4zu
zSAh$HBS|4F;`8!@=G_k1CB@m!dh(q_)H@^#fDR^M&*N$C*wDYIGo!nURxY;CdXN
z=NV=vtD{{KE>t+Dar1spdG_ZYCUHef^hPn0JpPA~FOMP-DF#h~cKi12NIZ66RzT!o-XzjD&tPSf
zQ7v*d5F`j3U=lgiXLR{z!EYpa+99)`izA7}E9a!-P$bnkiL+}EfhDtCm|i66$H-51
z-%9Bj7TVZbSVt^4*6`vRyyYz#g+-jAXV3bFU;+i;)@#F>}wr
zi34SDewQ+)G|AZwFg%G84k1|AqOQ&u2d@$75boiNP2c58O5iWv1A2q5Q3i3+>BswZ
zI0uoOx~6;^*;NZfH7Afj9fJQ%&Kp7MUPwsjfJIkb4bI9S5o2<09mXbbilQ*?dJse*
zMtix}gWs?S$mhk(nAw1bRi!kJ+#{2#!;GZy$pdWG2e*2jb
zm2Y3au7$09^C8j?kI6L3QK`!(Rih*=gZdD`_3EF*){4>-iP&UTY+^Kq@1y!=yZo--{DeRJuAE8vg&>=NCo
zjK~Rg<3%Va^Jx~6^ME=*gSxO19~N{TM<@njPW5di!ryQXCNNKwTKKJJ&&QaBaSkN%
zfWsZH|6aQs2-&3A>!7NB-A2gc7~W9Dd*T?$6_nCHxk#V>aFIlk9#1yHwIXF7=9^S6
zK6NLcKZypCwK_LdGPethY`xY^%HG}{UeRL9gDr1ZFFzTvejObGMR1GlM}}-0i4;OM$Z((RdTq2=qr2_}9Tu4pbii0}kAxO}I?5kh`fsFfLfEEnL
zDLPrLP1s!eh=s*j7_AZcqEeV6)JfLEo#4fHmLn43-0^%dIiT7|EMku~Y~89bzMAas
zc_nwCpb$R+UjBL5T;v2atWm;z$WTO1&Nn$jKQJbwz4kgpL5{&t7>yFA<}B}-9T=W3
zhgLU?(^kmz5(b}1{x<^)p5pyp9ao#T=fQg^Sv5rn{TM0z+E7AAuO$8NWteH6Mi>;~
z+-jUIu#_Ae1<~XkN>Bc;Kb)?sJmWZc6)u2FbE`6>O%kNU;AeZ+A14a6xAJ};uSe^a
z3X{Z)i4rOV4FwU!4?BeCJ08?D;+MSNb8+17W*eIqa~B>$J4Oqjaf%kbDAn16lcjMt
z)?hBghNPEpgbkTi)<1kQ`$12^0gV@l1o)|YRmEx+(
z7bIvT0f%!NXW4}Jq4qw8O8NqttQE}FM{gm%D5JxPOyLe&A#Hg6AqIsX<9L)89OnL$pfwqZAO;d^S!H7!?Cf+%hA=kqv1+R##4VW3#>>l$
zGtTfEb%I0?k=I5JCO<{fQxt#!XEiKqfu76=af$`bT6D~g!B8NYtBsA`Olu($+i2RUBR(O)Edju{_j7Q+z{SZ~4`fI+5EYt)
zQvo!Opp^r-;V3;CasoHZT*js%h5kt#mh=pHO{`*ah971-u|Cd0MkvK`cz|Lgovy9@
z?!*aqUEokUay&0$ikq@ZU_ihq$*`(fnSbc;tS978vE?}c76$zoV#Ya==!JI)LBLvUDiDfkKn&K}M!k5dPkmn_jXF}VZ?K#WE{?@?Cx_-%cC8|!mm
zk^Jg+xFo}j>AJ`CAjT0Y|
zDHk*EU+hGD!QmY^`7U&bF?#nFBR?1vwfPu1pn#e#BHTr`05Z>;OsU5gwq$
zp%J?>DuD~CmKWRx+5*}U0NODJJjxz)7zuFg;r`%w
z;0(?&;7KXKSz)$BwnE^{*biV!7qkPj===5k_4S}l^1!;K1=!{Umiek{J_FD51-82%
zJb%tEEj@d?w20E8?FtitRdA@dqf3Y=aJww%6jxB={_4^x+V=MHz@==UYyqCqEb3Un
z_E<-S#H87+=7Q3kji4DNr0dA?-OEfJ(^cG8+g?VgQu&X%Q~loCIC2y
BF|q&v
literal 15322
zcmeHuXINC(*5(0PR7}@aj0gryAi;nV#f$|4O3tVVN(RZ9)>hkuiUdWHl8Yoc6v+e%
z0ul<4ARr(~K*^bTx7~g3{bu@$^JDIhd7f^4q>8F@_Fj9fH=UcO6=arhtmB|4Y60J+5J^Y-wccaPG1pC4bJ=%FNQ%%=r99d&A2%#+DX)
z1rG@B-nH?Pt*w=fxR8+f-`^l;dHJGH1IL0J_>hHGCpB#-YS}sRKTEW9lrcp~$k2`*
zR&fmMYjV+1Rh^ytK6bc$+kwNap69*Kn>>s3d9Xi>cGWnt+h^66t=oz!2fbsal^7b&
z{M0@b<;~k~py@6bWt+NtU(nB%jyp>}$R1bzkmK29wS08oUS3f59slYUu5-D;_ln#l
zB~A5fI^$%Pxn+#%P1WrxN&X2}WZ;;|z+Y76Dwe-dRDtkMhwzVAf1_9^YVlDD?+{-6
z|6lmOS(3KyepY(&;NayA2VFH-SXh!jn)M6Lrzjovl4RS-VdLW`P87d-^*&3OwZ+nE
z{M)(X$B$<*7E_cJ=TdIT=*f{@24iu?MrxKFzw7Njxon59@P|?+MR}H-Iej`bI(kP@
ze%rRqcwM@Abjjkyyc8>S`pu*1algk0^2jEh_~%WOWLrQ*uPQDsR?*Vx>EC0uB>GI)
zbxA3yyY7VpwPe#(_0FApx^#m$Z%J~aE=R(0cX#(&zx{UK-`}4hm$hv
zX=!O@ytcn4&nr#82ygG7H%Qo@ni}m>K6UC&l8{T@O4f~3s4TyLfUlt@T{|Pe)y=I*
zcWDAkUS1wuGbM5q1C?tw{JG!-hTU~ZVJGga
zX&rBTe(q@QWKZmBDfhYA8LVP`l7UP_?REZFdp#*nlWt$8i%xy>TZtz+OsA}@ELnMZ
z<}FTs+pZ6r=FOWIW8GHRQ=1sVzIe?>T(^i>-Rb#Dx$X~F_0+7l$UMAr=g#bbys5xZ
z-M5}BY(Bxk{o^`1I&t=W^~RlLLG^iV46O_cH&weZlPy@@mTk=*bGh{ymWeGbXXS$>
zHItH(21iG0{Po>q<}F$kt(WVZ-5Kn{ODmV2%i(rvU(UI5<)z8t?w+csdp*q9C;{#C
z=(xC3-l^wav5)jOX@C6q(W^OnZs6K(gTgy!E?fw=?{Dm`7jp5Hm6dh$mD}2~N|>T7
z2I~y&y{oEH*3PhC&WtrD8kGigPxdCN-{Y1_J^zNoZs1dswb`LVo?q=50sD-fiP?O4
zTU)En<2L-DrzW1C(Im)dO1OXjZ!{$hjYnYuJ9fOazU#}c-V-u69i`_!(J^Eq=02Ut
z?byP?j8#`8+kW>hQ(_=TxwYA;Dcdf)!%w@tuP^?$-xl0me?S4NV)JzXTXTKM`pD&5
zPDCpNiOZfk<-=S0X!lVp2VFJR@7c3w!?@;{=g*(N?A<$$bddw1_>S3$au1WP4^Q!)
zntAuK>WYmN6&0Crn#xriQZmQB=`(kD>|J~h+bBo^@A6@k>kVJ2YAs8IBlg`mrzNir#ee
z`VK)sg{iRt>48ViBR$5$T@_yC_oQbt56+Go~f4~0JtHm2B
zs`h^AlLMwjUK_H%7p_;;(fN9bN5VdCdS<4hvok72Im)EiXD4>P!BE$xCQeh_Z8B9!
zNlDQ86i)9p`GD}cI*n_;{`$7LS)a#aI_dJ|%Yw1@aP{~>bCT}c{w4+|56{59l#h9C
zE=7tHGjZk(>77AS@3x0T<09zliNUO_ti#xc?(y$!;x40K4Vm*VlG)K1ilc_pSzK0T
zZ<0Sd*~`qdZW~%<
zqD$5YAMzfqloRSstJSNjOEO5a@6+n5OFDY<=1qUMCleE$A+=r`B)A$Zzc**R9vf&*
zWDMkK<5*Qs4df@r#qBD6yjR9Mp=-bB>{aao%(laqE=Bj}3@MZbOKD>>#*c@vO=|{;
z+i|_z_cR9MF4qs(A
zsd#qHz_hYGsxxi9h)Mebz1_QaYvnqp?Ki6nZDKg|mV3;m3=R(Rh0ldZjn)~6JAMo0
z_L#PI8!A08In_Lua532F1U@XgT`@z%qERCvA_A-Jmp3=-Le7@muB9)Sn|^e&yX!!j!ePwkGdAagRHm_HbKPoZt7hqFDJS`*FloP;rJC4a}5_v$Q)4)4#<+og!4CB$T%ylkdhB27TT<0D
zkD^ZRtjQkFI4=>AnJ(RO;4C}3RVF5vxI=#+qePhFNB7c9Gw~I-?FvQPzjpn4Pd{Uz
z#-Z8G;e~qC^$Lzfiyq<&$aUbm0>^aK5_DqH%^Qjv8ni>)Mjz)*^-GL&RUB7KVq;~+7)>BQOUxY&(DRtF%N;D{S0BzTyt6iJtl2}K^j}~sIm)SAfX=#vDO*xIJLy1LG$gmD~X8*jTO
zO}A*uo*XO@>KGVEM75bW=jq04XVgT6xYN<^jlX|>Lk2c)H?EAXpPyG*Rb@9fGp7G7
z0DGPhu-}}Hd9N9z7@~&pDB?0|Hq))>ksTL+VQMI#m!p9>o!hmWZT`|~G~-mWk9&f|
zY)+(^)~Y!>XLGxLJ4{nDHjY`h-z*yAvnwVl__C|NVToUi%sp=ZuldJsuOep&)74&c
zWcKT`Io;^NxtRoxM=U>41@fc)O|kR29S_y&d*tGtnLtLg?DjkAuC6(4<(oEb%5J^B
zKGkK+f~NGB&A!c<;V__PZJmVTcCF~mpXz6rH0R|4IAD6<>^c@rV2x2c4rb`o2M4Cn
z6cx}u$eY6Afjh_y
zTX*jGlp0kf8n>i`T*{DnN5~z`aqXfuhz&Xfztf;>9XfR?pE#
zR5diZK3lRb1yE3K&-a2is_!VrFYniq=4kWfy_jV)=60HS!+!Zk!iO8u
z&8z0~xQ2fHYSv^^E~QWeAk!V2m{$p~m*zTgIbJ_s@9O+z_DprOs(#E%%*Go0#mus~
zC}LW3YP(|aG;?UbMPt|a(7T7*(=L{ClOKBzW6s5*9td#NN;8Qd?Z9Ppy3_jEqep8d
zCMWr}Y-zhv`|0_)s|k#LajBW%s#~{j6Zj2m-BLHu-5pbScfCaV{rmTa5^&Fn%_VM*Dq}Idp@}gsG>We3`Pf<=VA@6@T+faW^=67b?CfUzz+l}^rSQr4WA}0Z#{qhds)nel4=y(Y|1Kdgp({A78_LHPE^PNBW>{Q<8I&Ot(Q3EfT>opuXSha88bQhU5|v%T?uv_q{#=0o+%9wQ4TriW;@(L@a);9aKh
z;0bo=7n8!ZugYF*E4;PH&>s%?^yw2ARzhk37_H7ztuktsYnF6DOtxcBpFUNXYHMq=
z>#5!e+$B~IxElt(jTIP;w$DAQYd9b5IRZT6ju~CO6ohw|0N+UbrFQ_~GcFVm>v(poN0s?5cuc|XF
z_0=dgaL+`vI*blFkd9Y5AQy-N5N!FusknpF-;UChdTZi?%Cs5w@#I2iTB(;F5ZZ(r
zEWz;2bs4L9|9G#ik~C0_1p3VQ(7y3EI%O}vpjdNfMjJRaS5XCb>Em(&r(cW@v{uqP
z2#&O=jy`j=k~!zPjvfUpCOg^f@SAlWpBNxgK{*&bVF7>=G3d*K-_BaRy5MKn{qa^Q
z=4Ve%cBc#N_3$TRg32OpZf@peREy`W*H^KJ{~Zwa--m7gLcbCDR}(F8RNF5Cl#Orm
z<|7z6vL{b|_>}7s*(L?z6d=kLe8Ilu{_w@Jz(bcojcVJ6%WQ>p2>LwbaHZYCkzkUj`TIClgSU}
zoMu?E3Iv4ajTsz~pv&m_@boBVaZX&`A`Xs~D_4f&l+I&kR~vUIaW$~Lrk|4hW_$iu
zpeZUdym)qu!58g~W1DQ*e6Zuo7rsrKUR12YFF`NaiGk}4N*9BhmKlE-{_C$lIiWG4
z;jYfL2jl=!3*3)Fr$^S$DV%9;iA4tgmMyO;)S#uI`RzP-P|tfM4^JYF@=kt!
zz<2N4BPhtMP8d5fmaL8*LaC^#?x6rkRl!OxyK;+|
z`0yGRc(TYUD%L*QZD0%Nv4mSvBT6Ai2@E>bgvkeTBwD8V*R5n$7Z<7%Uz|VxLy+UL
zmI=pY8>tqpA{Q#Qm9!2jbKn1rLj4P!#s3Q9`%f=7CY|LuggCNGgdDiJF@cig1NPDU
z48^>XcFAGznR$pp1qtZvTM0#q+z`AL!}3OG|W!
zKA^S~OfyTeN%xT9N8mQ+;jX9HbecEe0S=#>WS$(b^c$}F+6?qaC!8DT&pSNhld~My
zoL0KoCZFWGR}C
zEDm9j7>Kq>Z9>qVy%;rHa2FDpuI$gDxJ=FrcqF#BUzquR&jZX=F=BYHMWZMr%V@|?
zbjZU%G{c{73juD^yzy!LG>&D<9K?krB^#3l@;QNkGBfMGXN87_Dw#mmjy@$A!qS*-
z{sfmBb0+M>JBy4TN<~|`*>PtGAw&isQ>`9vStY0wO-K}|U`~%|I`zav?ZZWr1_$wxuv3D+pf<>Q
z1J3g96cQ@$#w?R8%A521NAGyS5rZyT;MDYUAuOz*9=G~Ats;O+JqeCA!ng^
ziksseRPQ#hw$#LHk3b3Ci4g%n_MxjHYy`@W@kkGo+?Ii1KeOjkE}t`*ETmKs>Ci
zjF2O+x5;H7*P9p3op8i!tMxwI;Bk)@%AZz(l%bBs;yn6QlcgC_MN57A;PH(F?P+NR
zB&Vb>p@`;=)MyjpV~{S03+?Xi26$|>?70Yd1dw}$+vV%-4N}f30O^xMA#-s=cJx}i
z*XT}Jpjg<8tMhwN(ijU@F(Fiq7b%ndn76;zU;1t*=|SSMIb8QsdBLx^8X{{s5(qK)vMTJ%FAk
zAOWg3=eg*R4bR)*8+^F8vxlXoeWhhz&038{mvIuUWGV4e;)m79|@3>CBuN$6MIc$p}5W
z603}%O-e4IpRD^FSzQ5vQl>6IOUr7V*~Ou}K@tuMA+CcrX9xZByN0`~yu-0*1ler(
zu^(>$P2sDY23RKK5Q-TzM>G=5hB@$iTZX?;oM9V^)_-4lR~mhW6CLDwsb{(rwZfniCyzE@b`6tIG~Xqq!5+$GJ1e
zLFbSC=cWL*5^={SpmmezfGQdq(P)%l_hJGZ=)s3NtehwxUbq*~Im$QaVvMgT$BxHm
ze9CU0ot>Ft5Q;+@*HgJwt?VN7A(b&GMbo}AYK~9_qarzB}CGx(wKkOE){$G
zbjW%In^9iWYO4}UmbnZtefh_GI&wZ9#?Mr8OgB@q(gKUc&^8z)_(}9AwxtDKp=sc=BICzq}!$Opeex^|{#3jnp
zlX^J8SpyCH;8|7SP8UGAz|`6Be1R?6k^YDHn*-G{*1lUMEk=8L404q4nYrgE0|*N>Fj=$rqQ7g
z5a6W7f-B4PEMYr)`0y{#`0e$k!10Ms`j2Zyg{og=Us6dlm!mh9c4POQq0E8J$3bMx
z%I3m#v}I1OEkL!I!hmu9_F}%Ome!>5^59@u1%(zYvI?HP#<)5g;stJ(VjgF3f*@G+0tjU~S_3iD0EvhL-@*VB%(V8hnUZV%yCNE(R
zIY;>(L48D#%NiRSd!<9>ED8}Qsr(P%h4v=9M_=HjHtN3QT<>&og@ZpXUM|a({Tp!#
zBNc+^u%so@ieA6&D)@zsv!Cj?p=Ec10Ob4UDOZ}E0VHBSo&3Dp9Mq4h*@
z^hUvsA+`pzJ>KrfCl-}pJ^MUnUBT2_=n0Pl0_e~*5?-3bd0CWdw)Fp;@Uo;|5bNuI
z8G{O+k_zpr$*xuxFsK5Ey!zSkMMT#FPl$i+4*8%u$H^utov5G~NZuDR8n)7)w<#1s
z>k%8B$$+r}MlO*K9Z*dp7oVGeMqh^y>HC->LjVnIT!Qj}3FOErgNncx7XsCrUqYfG
z#=JaODjSfFcP4e?>)#nmQa;QRLKa7@a
z8{>ADOH3)F$^OYU#o#!&WcomwHCN^>!fh#L>@eT|1Rnd-)q0MHG23r>dDZ;7aK%b!
zq&VYhCIrXyw^vdcs4%zhZ&nL-_rj>E0Y>o!n1Xz$hXL$7!c_P5@o5)@pSoKdTg$$T
zI}WQ8gA+>BK}hd9zvIrjxZ?4JUUF;1E{9{+MGT8qs@rSt1Tp-bcTUS@Z3vtw?BdAZ
zR<=6iJ({5~?T7o5>B{*HCv_O#-2-+@w3_m%B`ts~nSYFwU6ece0AS8j(6tDN#3v_>
z&lC#5nNjy84R4qKyaF^-(k3woeTS(zKRIX%|y470g0_pw_AqN
z$2c9MX5YUtRm3WM^UzsEneT6KDj)y;o(He?W_?|Is3dtGkdTv$i=6kZTetRIczb_H
zjl}~^CxMT2x|4#7JlWV6TTM15>PxHJppE3kkaVR`p{xwrK
zj@cw2)WT}2+@$-$x7X)$KL=gF=)WBhTC<#rI`w$-?_gH6o~)E@*J-G~#qgjpe$`<7
zPu(J9{(%V(m0|YrS^ji~kjcm72<0iXw-HFV#_znhB8U-$Y5Jqn$H?NsYGZ0x5HUCn
zp~QJ+q*j$-Z!Y5kGw73J+Y(|w!k@nZvJsn_y3f4s`Qh~kENl%3yYNlanqTnr=neQ>
z7Eo=hOv<6$Vz?>0K&&8koS~6XcJ?W})1Od&Uqhe(LRuhmU;dN_=>o?ajz?XL6x3@7
zyQ_1Jpl<-rYzTpd!WXB9-&BwB10Y()UHz@~RT_v7NV`g%owBkr=v8|i*ahF$db}+`
zNr3IFnwp;%w-g;!`xay*I5?QxsZm+rPT-4cH*Um+haUs|zl>;^kRrt5XSMP)+8NxZ
zfrR6?vs*n@pnz_v3s|x^^fzkiyH7+YH-sL(TAgaFe0PI{+L<$F$Yj~GXHPZ5!IGu~
zqyftxLWf61ud|CwNkaoe$&|1+Gv*$O{imOzVL&OE!|hPQfD4m?ypD#X1fdjm(+JEn
ze&{5irLhvS?AWl!CTl67BL2jj&Ctb3q6N|tq~>Zk-A%fHl7}MpZll~MyQ36B++1Dv
zT`c2*L0=>_4@I9zOVjxY_|o$>N!I+=oZSDE*Z3*0&;b`vN#E9N}Ps5UyC#R_KRI&s@*xJl6n@}TNfJjP$~kkL}K
zdy9-$wG5A2I2hI?ypNDiWK=o|>DUOqkSNBY=v}dD@s}q9!A`)lAf|Ua(#<0cx^D`f~;sn%^Lnw)#%`**BI`>a%LSX(gX2g8KMv2fu7UJ`-8w=Z0{
z(93|hrErRN?A<$br4|7hHGsP+RL*H8x@=^roLAJp5;7PbU=3j^4M|
zQyRSC5i$RW8R17XP5QG&%Te0yf;oncwm0H*fei<1KhW?1&osPl!r|yy?xg9@2
zT{^re6^uBADU6(02!b%)h-~l2Yk2V@KgRv{@8A1(tff37SmotT(-<;oQw09Gg@J9&AJ{qoB%&ePwEkhv-6IExF2
zS8bp~AurLC;6Vn}S5;P$5ksr|%QL2{v$L`9e}in%FMr%ZulhY)Tt%Rm4LNnIRY-d^
z7e3D^jB=U1`hTV-V$yFP@+`44En~%PaO&ARuuz_Sch(-l+yCEYLVpY)b-|K##(
zxBe8P=WhIdX(SlJd{?jHpl;P69+AmlEf9VG&+Wv5h0W@wy4rxX!$Nhl@Xo1zK^BT{
z-VZJGuM#90Kd}~U{R0Lzjy&y8K)LXj=w6Q;8
zDXoa?6~t?7TACUBeXl1R<1??WFX?D&3j;7A$+dpMfkKtmO0kU-{Qyv%sKTh_D6m#l
z#GG)})nL0ZO4ul!FpZ7B(#yKm769sWJ&@t9AdMYxlh_3a>N&&iY|@1ma42H8J^ORa
z2~AB+SsKkc70UaLO`8rse*Ac~WK?FR2HLV$IjE<1II2%J=KK5n+a5ExYI;oZuf1
zT>_U<3-Y3GeNkF*9pxy3pxHi?>K#R$V3$EK_2JOLS>>!($FaxIv;Ed`n3xM)Y2SJ7
zq?ihqCA<)2u(y3(=POnQmkP21tcbV|y=q$v*$c$&dZJI3SvEcMzV7%&Jv}`VV8Qg`GDo*Wp5!#566^Va
z`l59T%P84O)g7Blg7t1bv2tC{{2*4IfBB@iu-C=Olj6c7=Nq7xR
zJ{E=VcWBsQ>`Eyh`
zE(zFs(F-#04jvvJIF|~*=k^XXRV@<}x<%fE5qigtUAs;pgho_?s@-rzNSlYLPU0(Y
zoI=LDt#lsmRE~TLQ}^|6%f0p&W}Tn@_G%eP50JO@_QoRfXWo>}OlnTvMGWTU%a;$1
zj1YnbE48=qR%|eYFcN4$uOlyD9yDaEG=>pvFC6_YM+4HWlw9*=gvrmkpSz4SoXJ!~Xfx$B}-r
zHMtn*!T4+HtPaAtq3G0;{qoq!>gkCg2o>9ybX7T=E)e3&1|6y;M795SMEu2Vu69HT
zh5TuZzgIVUh_way_=J<_Xsmi-4P4?h_ZernGIgr@Po1Az$!8%F5r=*+e$e07R}-?T
z8uo5|EL{(<&V-f>5%?ltyiaJZy)G+imFRrru_1rl8RE?BOKom?ck}&TbNoM37ODl{wV!R+?84l#c
zrNMYDT7?Be2Pz6#+u57|1+6o;@Y0`h2@RbrxuH*?P3L!ecXye3%O5Vz_GC;IJ|Q9h
z->2@~_@@9#hlsjnzuq<0i?>$`rIWJ;AH5ertaQj5X7lFH*Tz=&3&s~a
z4_C;Xdf_#gMGD>Z$Y^Eo-6=YR+N
zqN1WkATetpq?5KvKYLo-*=smFWXa4cP#K07;>udVO;{TdUgWPWk+`pL2U
zxxT{USU(mvb`9u8DFC?^2t}1owp?8nhqtO_&6AO*3DWi(a?R6NiYC+tU0F;>z<~y@Q1#$kf%aEt5PuF*T(PNKgZu6o*@9v^J>d
z?k|~qV_FL0aJujNKO?n22k?IoAkP1QCr$3I+5PNgftf`c_)5@}3>@qqnLh}m
zBymDCSePw$xuvvF2+GKhR43?4xPftd4Y}v_P2w2|Vs?Pzs3|CTkp~qZ>=7uZjDwCy
z1bJ>DIy}7nnTQTO5wVjd$F{rcBpsi(jg#mn9y}r#Gst9OqP$#cERZ~&@~G()q6x2-
zNyj02-w@Lby%8+6`|g2H(T`sIhwW~vcEAa8zap2)sh2m%gYtLNevumc2&j$C}HH%h&BrJq2lB;;WYE5y?xq5pG1ro
z3FU-JBQ8vOH+c>Udil(i=2v)J2l3g#;bEU93xE6vmKau;k0h#>0SKUNv)kCa=H~p(
zH-4sgce5>MS(lD%DN3Did>l;Ta*U7+8*zFo!*5n4p|m)bEU|MP9UN3b+_Riv1a{AL
z?zA_W5Tdow(4u>?dlPKLM@xkARWvm}CXSB4IE3V87$D`64jZC7b>K3IHCI9@#)C3z
z@HNIA#lGmxrJVZnDZ%+He=87)g~}Unz%rPO*tKhkT$PJNF-Z<2Cnpn!D5}jvO#2JU
z_(|#LlS!dnyAU%XQWtrm0;^ija7aW_^YRbzV3;x*ibh3^?gHfdml49U1b@)Qc79kC8q
z;z=J)ezhIQPyp^C`d{l*YX+T~Jm~XU
Date: Mon, 10 Apr 2023 02:15:31 +0000
Subject: [PATCH 37/63] fix: num training steps for lr decay
---
train.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/train.py b/train.py
index 189df41c..8605af11 100644
--- a/train.py
+++ b/train.py
@@ -100,7 +100,7 @@ def train(accelerator, config):
name="cosine",
optimizer=optimizer,
num_warmup_steps=config["warmup_steps"] * accelerator.num_processes,
- num_training_steps=total_num_steps * accelerator.num_processes,
+ num_training_steps=total_num_steps,
)
else:
scheduler = DummyScheduler(
From 84de913bafac04dece14846bcf8020bda2b79638 Mon Sep 17 00:00:00 2001
From: Zach Nussbaum
Date: Mon, 10 Apr 2023 02:16:05 +0000
Subject: [PATCH 38/63] chore: submodule ff
---
peft | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/peft b/peft
index 098962fa..deff03f2 160000
--- a/peft
+++ b/peft
@@ -1 +1 @@
-Subproject commit 098962fa6515f2e4fe83a757f5995d3ffbb1c373
+Subproject commit deff03f2c251534fffd2511fc2d440e84cc54b1b
From 70d75951d15ddddd28c6a80c798c3d05003dbdfe Mon Sep 17 00:00:00 2001
From: Zach Nussbaum
Date: Tue, 11 Apr 2023 12:39:07 +0000
Subject: [PATCH 39/63] chore: remove not needed
---
head_node_setup.sh | 19 -------------------
worker_node_setup.sh | 6 ------
2 files changed, 25 deletions(-)
delete mode 100644 head_node_setup.sh
delete mode 100644 worker_node_setup.sh
diff --git a/head_node_setup.sh b/head_node_setup.sh
deleted file mode 100644
index 7a813d1e..00000000
--- a/head_node_setup.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/sh
-WORKER_IP=$1
-N_GPUS=$2
-
-sudo apt install -y nfs-kernel-server
-sudo mkdir -p ./data_multiplus
-sudo chmod 777 ./data_multiplus
-printf "${PWD}/data_multiplus ${WORKER_IP}(rw,sync,no_subtree_check)" | sudo tee -a /etc/exports
-sudo systemctl restart nfs-kernel-server
-
-sudo apt-get install -y pdsh
-export DSHPATH=$PATH
-export PDSH_RCMD_TYPE=ssh
-
-ssh-keygen -t rsa -N ''
-cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
-
-sudo mkdir -p /job
-printf "localhost slots=$N_GPUS\n$WORKER_IP slots=$N_GPUS" | sudo tee /job/hostfile
\ No newline at end of file
diff --git a/worker_node_setup.sh b/worker_node_setup.sh
deleted file mode 100644
index 2c50e29d..00000000
--- a/worker_node_setup.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/sh
-HEAD_IP=$1
-
-sudo apt install -y nfs-common
-sudo mkdir -p ./data_multiplus
-sudo mount ${HEAD_IP}:${PWD}/data_multiplus ./data_multiplus
\ No newline at end of file
From 3c11e68ec8f31fac86a945506f1b5af730ac4cc8 Mon Sep 17 00:00:00 2001
From: Zach Nussbaum
Date: Tue, 11 Apr 2023 19:14:29 +0000
Subject: [PATCH 40/63] feat: commits for eval + generation
---
configs/eval/generate.yaml | 5 -----
configs/eval/generate_baseline.yaml | 4 ++--
configs/eval/generate_gpt4all_gptj.yaml | 2 +-
configs/eval/generate_gpt4all_llama_lora.yaml | 5 +++++
configs/generate/generate.yaml | 4 ++--
configs/generate/generate_gptj.yaml | 15 +++++++++++++++
configs/generate/generate_gptj_lora.yaml | 15 +++++++++++++++
7 files changed, 40 insertions(+), 10 deletions(-)
delete mode 100644 configs/eval/generate.yaml
create mode 100644 configs/eval/generate_gpt4all_llama_lora.yaml
create mode 100644 configs/generate/generate_gptj.yaml
create mode 100644 configs/generate/generate_gptj_lora.yaml
diff --git a/configs/eval/generate.yaml b/configs/eval/generate.yaml
deleted file mode 100644
index b06137d6..00000000
--- a/configs/eval/generate.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-# model/tokenizer
-model_name: # update with llama model name
-tokenizer_name: # update with llama model name
-lora: true
-lora_path: "nomic-ai/gpt4all-lora"
diff --git a/configs/eval/generate_baseline.yaml b/configs/eval/generate_baseline.yaml
index d409d3ab..7c70c814 100644
--- a/configs/eval/generate_baseline.yaml
+++ b/configs/eval/generate_baseline.yaml
@@ -1,5 +1,5 @@
# model/tokenizer
-model_name: # update with llama model name
-tokenizer_name: # update with llama model name
+model_name: "zpn/llama-7b"
+tokenizer_name: "zpn/llama-7b"
lora: true
lora_path: "tloen/alpaca-lora-7b"
\ No newline at end of file
diff --git a/configs/eval/generate_gpt4all_gptj.yaml b/configs/eval/generate_gpt4all_gptj.yaml
index 496ce25f..fc0df450 100644
--- a/configs/eval/generate_gpt4all_gptj.yaml
+++ b/configs/eval/generate_gpt4all_gptj.yaml
@@ -1,4 +1,4 @@
# model/tokenizer
-model_name: "nomic-ai/gpt4all-warmup-lr-epoch_1"
+model_name: "nomic-ai/gpt4all-warmup-lr-epoch_0"
tokenizer_name: "EleutherAI/gpt-j-6b"
lora: false
diff --git a/configs/eval/generate_gpt4all_llama_lora.yaml b/configs/eval/generate_gpt4all_llama_lora.yaml
new file mode 100644
index 00000000..e1b68263
--- /dev/null
+++ b/configs/eval/generate_gpt4all_llama_lora.yaml
@@ -0,0 +1,5 @@
+# model/tokenizer
+model_name: "zpn/llama-7b"
+tokenizer_name: "zpn/llama-7b"
+lora: true
+lora_path: "nomic-ai/gpt4all-lora"
diff --git a/configs/generate/generate.yaml b/configs/generate/generate.yaml
index f81ca3f9..3953d07b 100644
--- a/configs/generate/generate.yaml
+++ b/configs/generate/generate.yaml
@@ -1,6 +1,6 @@
# model/tokenizer
-model_name: # REPLACE HERE with the base llama model
-tokenizer_name: # REPLACE HERE with the llama tokenizer
+model_name: "zpn/llama-7b"
+tokenizer_name: "zpn/llama-7b"
lora: true
lora_path: "nomic-ai/gpt4all-lora"
diff --git a/configs/generate/generate_gptj.yaml b/configs/generate/generate_gptj.yaml
new file mode 100644
index 00000000..6c9cad42
--- /dev/null
+++ b/configs/generate/generate_gptj.yaml
@@ -0,0 +1,15 @@
+# model/tokenizer
+model_name: "nomic-ai/gpt4all-warmup-lr-epoch_1"
+tokenizer_name: "EleutherAI/gpt-j-6b"
+lora: false
+
+
+max_new_tokens: 512
+temperature: 0.001
+prompt: |
+ #this code prints a string reversed
+ my_string = "hello how are you"
+ print(len(my_string))
+
+
+ My code above does not work. Can you help me?
diff --git a/configs/generate/generate_gptj_lora.yaml b/configs/generate/generate_gptj_lora.yaml
new file mode 100644
index 00000000..4444e194
--- /dev/null
+++ b/configs/generate/generate_gptj_lora.yaml
@@ -0,0 +1,15 @@
+# model/tokenizer
+model_name: "EleutherAI/gpt-j-6b"
+tokenizer_name: "EleutherAI/gpt-j-6b"
+lora: true
+lora_path: "nomic-ai/gpt4all-gptj-lora-epoch_0"
+
+max_new_tokens: 512
+temperature: 0
+prompt: |
+ #this code prints a string reversed
+ my_string = "hello how are you"
+ print(len(my_string))
+
+
+ My code above does not work. Can you help me?
\ No newline at end of file
From 60155de2a69630e0f8536d32c6d5d7481005d00a Mon Sep 17 00:00:00 2001
From: Zach Nussbaum
Date: Wed, 12 Apr 2023 03:50:54 +0000
Subject: [PATCH 41/63] chore: peft
---
configs/train/finetune_gptj.yaml | 12 ++++++------
configs/train/finetune_gptj_lora.yaml | 10 +++++-----
configs/train/finetune_lora.yaml | 2 +-
peft | 2 +-
4 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/configs/train/finetune_gptj.yaml b/configs/train/finetune_gptj.yaml
index 1ff6a7bb..ef9802d6 100644
--- a/configs/train/finetune_gptj.yaml
+++ b/configs/train/finetune_gptj.yaml
@@ -2,12 +2,12 @@
model_name: "EleutherAI/gpt-j-6B"
tokenizer_name: "EleutherAI/gpt-j-6B"
gradient_checkpointing: true
-save_name: "nomic-ai/gpt4all-warmup-lr"
+save_name: # CHANGE
# dataset
streaming: false
num_proc: 64
-dataset_path: "nomic-ai/turbo-500k-multi"
+dataset_path: # CHANGE
max_length: 1024
batch_size: 32
@@ -18,8 +18,8 @@ weight_decay: 0.0
eval_every: 500
eval_steps: 105
save_every: 500
-log_grads_every: 500
-output_dir: "ckpts/gpt4all-gptj-multinode"
+log_grads_every: 100
+output_dir: # CHANGE
checkpoint: null
lora: false
warmup_steps: 500
@@ -27,7 +27,7 @@ num_epochs: 2
# logging
wandb: true
-wandb_entity: vicuna
-wandb_project_name: vicuna
+wandb_entity: # CHANGE
+wandb_project_name: # CHANGE
seed: 42
diff --git a/configs/train/finetune_gptj_lora.yaml b/configs/train/finetune_gptj_lora.yaml
index 325c30f7..c2668ddd 100644
--- a/configs/train/finetune_gptj_lora.yaml
+++ b/configs/train/finetune_gptj_lora.yaml
@@ -2,12 +2,12 @@
model_name: "EleutherAI/gpt-j-6b"
tokenizer_name: "EleutherAI/gpt-j-6b"
gradient_checkpointing: false
-save_name: "nomic-ai/gpt4all-mosaic"
+save_name: # CHANGE
# dataset
streaming: false
num_proc: 64
-dataset_path: "nomic-ai/turbo-500k-multi"
+dataset_path: # CHANGE
max_length: 1024
batch_size: 1
@@ -19,7 +19,7 @@ eval_every: 500
eval_steps: 105
save_every: 500
log_grads_every: 500
-output_dir: "ckpts/gpt4all-gptj-multinode"
+output_dir: # CHANGE
checkpoint: null
lora: true
warmup_steps: 500
@@ -27,7 +27,7 @@ num_epochs: 2
# logging
wandb: true
-wandb_entity: zanussbaum
-wandb_project_name: mosaic
+wandb_entity: # CHANGE
+wandb_project_name: # CHANGE
seed: 42
diff --git a/configs/train/finetune_lora.yaml b/configs/train/finetune_lora.yaml
index cf916d3b..7f270d85 100644
--- a/configs/train/finetune_lora.yaml
+++ b/configs/train/finetune_lora.yaml
@@ -2,7 +2,7 @@
model_name: # update
tokenizer_name: # update
gradient_checkpointing: false
-save_name: "nomic-ai/gpt4all-lora-multi-turn"
+save_name: # CHANGE
# dataset
streaming: false
diff --git a/peft b/peft
index deff03f2..098962fa 160000
--- a/peft
+++ b/peft
@@ -1 +1 @@
-Subproject commit deff03f2c251534fffd2511fc2d440e84cc54b1b
+Subproject commit 098962fa6515f2e4fe83a757f5995d3ffbb1c373
From b1e361882d4b30df26724ba427b0e4663b67147d Mon Sep 17 00:00:00 2001
From: Zach Nussbaum
Date: Wed, 12 Apr 2023 03:51:29 +0000
Subject: [PATCH 42/63] fix: multi-turn data breaks
---
data.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/data.py b/data.py
index 7d61154d..dc404af1 100644
--- a/data.py
+++ b/data.py
@@ -15,8 +15,8 @@ def tokenize_inputs(config, tokenizer, examples):
out = {"labels": [], "input_ids": []}
for prompt, response in zip(examples["prompt"], examples["response"]):
if different_eos:
- if response.count("
") > 0:
- response = response.replace("", tokenizer.eos_token)
+ if response.count(" \n") > 0:
+ response = response.replace(" \n", f"{tokenizer.eos_token} \n")
prompt_len = len(tokenizer(prompt + "\n", return_tensors="pt")["input_ids"][0])
From d7395ee37a7599bd7c7c879498e57a5e43f466f2 Mon Sep 17 00:00:00 2001
From: Zach Nussbaum
Date: Thu, 13 Apr 2023 15:16:31 +0000
Subject: [PATCH 43/63] Merge: main into gptj
---
.gitignore | 7 +-
LICENSE.txt | 19 +++++
README.md | 202 ++++++++++++++++++++++++++++++++++++------------
TRAINING_LOG.md | 2 +-
data.py | 3 +-
launcher.sh | 88 +++++++++++++++++++++
6 files changed, 267 insertions(+), 54 deletions(-)
create mode 100644 LICENSE.txt
create mode 100644 launcher.sh
diff --git a/.gitignore b/.gitignore
index bfce2c0c..2952a641 100644
--- a/.gitignore
+++ b/.gitignore
@@ -164,4 +164,9 @@ cython_debug/
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
\ No newline at end of file
+#.idea/
+
+
+# vs code
+.vscode
+*.bin
\ No newline at end of file
diff --git a/LICENSE.txt b/LICENSE.txt
new file mode 100644
index 00000000..51aef442
--- /dev/null
+++ b/LICENSE.txt
@@ -0,0 +1,19 @@
+Copyright (c) 2023 Nomic, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/README.md b/README.md
index 770a2806..821eb0ff 100644
--- a/README.md
+++ b/README.md
@@ -1,13 +1,33 @@
GPT4All
-Demo, data and code to train an assistant-style large language model with ~800k GPT-3.5-Turbo Generations based on LLaMa
+Demo, data, and code to train an assistant-style large language model with ~800k GPT-3.5-Turbo Generations based on LLaMa
:green_book: Technical Report
+
-Discord
+:snake: Official Python Bindings
+
+:computer: Official Typescript Bindings
+
+
+
+:speech_balloon: Official Chat Interface
+
+
+
+🦜️🔗 Official Langchain Backend
+
+
+
+
+Discord
+
+
+
+

@@ -16,20 +36,99 @@ Run on M1 Mac (not sped up!)
# Try it yourself
-Download the CPU quantized gpt4all model checkpoint: [gpt4all-lora-quantized.bin](https://the-eye.eu/public/AI/models/nomic-ai/gpt4all/gpt4all-lora-quantized.bin).
+Here's how to get started with the CPU quantized GPT4All model checkpoint:
+1. Download the `gpt4all-lora-quantized.bin` file from [Direct Link](https://the-eye.eu/public/AI/models/nomic-ai/gpt4all/gpt4all-lora-quantized.bin) or [[Torrent-Magnet]](https://tinyurl.com/gpt4all-lora-quantized).
+2. Clone this repository, navigate to `chat`, and place the downloaded file there.
+3. Run the appropriate command for your OS:
+ - M1 Mac/OSX: `cd chat;./gpt4all-lora-quantized-OSX-m1`
+ - Linux: `cd chat;./gpt4all-lora-quantized-linux-x86`
+ - Windows (PowerShell): `cd chat;./gpt4all-lora-quantized-win64.exe`
+ - Intel Mac/OSX: `cd chat;./gpt4all-lora-quantized-OSX-intel`
-Clone this repository down and place the quantized model in the `chat` directory and start chatting by running:
+For custom hardware compilation, see our [llama.cpp](https://github.com/zanussbaum/gpt4all.cpp) fork.
-- `cd chat;./gpt4all-lora-quantized-OSX-m1` on M1 Mac/OSX
-- `cd chat;./gpt4all-lora-quantized-linux-x86` on Linux
-- `cd chat;./gpt4all-lora-quantized-win64.exe` on Windows (PowerShell)
-- `cd chat;./gpt4all-lora-quantized-OSX-intel` on Intel Mac/OSX
+-----------
+Find all compatible models in the GPT4All Ecosystem section.
-To compile for custom hardware, see our fork of the [Alpaca C++](https://github.com/zanussbaum/gpt4all.cpp) repo.
+[Secret Unfiltered Checkpoint](https://the-eye.eu/public/AI/models/nomic-ai/gpt4all/gpt4all-lora-unfiltered-quantized.bin) - [[Torrent]](https://the-eye.eu/public/AI/models/nomic-ai/gpt4all/gpt4all-lora-unfiltered-quantized.bin.torrent)
+This model had all refusal to answer responses removed from training. Try it with:
+- M1 Mac/OSX: `cd chat;./gpt4all-lora-quantized-OSX-m1 -m gpt4all-lora-unfiltered-quantized.bin`
+- Linux: `cd chat;./gpt4all-lora-quantized-linux-x86 -m gpt4all-lora-unfiltered-quantized.bin`
+- Windows (PowerShell): `cd chat;./gpt4all-lora-quantized-win64.exe -m gpt4all-lora-unfiltered-quantized.bin`
+- Intel Mac/OSX: `cd chat;./gpt4all-lora-quantized-OSX-intel -m gpt4all-lora-unfiltered-quantized.bin`
+-----------
Note: the full model on GPU (16GB of RAM required) performs much better in our qualitative evaluations.
+# Python Client
+## CPU Interface
+To run GPT4All in python, see the new [official Python bindings](https://github.com/nomic-ai/pyllamacpp).
+
+The old bindings are still available but now deprecated. They will not work in a notebook environment.
+To get running using the python client with the CPU interface, first install the [nomic client](https://github.com/nomic-ai/nomic) using `pip install nomic`
+Then, you can use the following script to interact with GPT4All:
+```
+from nomic.gpt4all import GPT4All
+m = GPT4All()
+m.open()
+m.prompt('write me a story about a lonely computer')
+```
+
+## GPU Interface
+There are two ways to get up and running with this model on GPU.
+The setup here is slightly more involved than the CPU model.
+1. clone the nomic client [repo](https://github.com/nomic-ai/nomic) and run `pip install .[GPT4All]` in the home dir.
+2. run `pip install nomic` and install the additional deps from the wheels built [here](https://github.com/nomic-ai/nomic/tree/main/bin)
+
+Once this is done, you can run the model on GPU with a script like the following:
+```
+from nomic.gpt4all import GPT4AllGPU
+m = GPT4AllGPU(LLAMA_PATH)
+config = {'num_beams': 2,
+ 'min_new_tokens': 10,
+ 'max_length': 100,
+ 'repetition_penalty': 2.0}
+out = m.generate('write me a story about a lonely computer', config)
+print(out)
+```
+Where LLAMA_PATH is the path to a Huggingface Automodel compliant LLAMA model.
+Nomic is unable to distribute this file at this time.
+We are working on a GPT4All that does not have this limitation right now.
+
+You can pass any of the [huggingface generation config params](https://huggingface.co/docs/transformers/main_classes/text_generation#transformers.GenerationConfig) in the config.
+
+# GPT4All Compatibility Ecosystem
+Edge models in the GPT4All Ecosystem. Please PR as the [community grows](https://huggingface.co/models?sort=modified&search=4bit).
+Feel free to convert this to a more structured table.
+
+- [gpt4all](https://the-eye.eu/public/AI/models/nomic-ai/gpt4all/gpt4all-lora-quantized.bin) [[MD5 Signature](https://the-eye.eu/public/AI/models/nomic-ai/gpt4all/gpt4all-lora-quantized.bin.md5)]
+ - [gpt4all-ggml-converted](https://the-eye.eu/public/AI/models/nomic-ai/gpt4all/gpt4all-lora-quantized-ggml.bin) [[MD5 Signature](https://the-eye.eu/public/AI/models/nomic-ai/gpt4all/gpt4all-lora-quantized-ggml.bin.md5)]
+- [gpt4all-unfiltered](https://the-eye.eu/public/AI/models/nomic-ai/gpt4all/gpt4all-lora-unfiltered-quantized.bin) [[MD5 Signature](https://the-eye.eu/public/AI/models/nomic-ai/gpt4all/gpt4all-lora-unfiltered-quantized.bin.md5)]
+- [ggml-vicuna-7b-4bit](https://huggingface.co/eachadea/ggml-vicuna-7b-4bit)
+- [vicuna-13b-GPTQ-4bit-128g](https://huggingface.co/anon8231489123/vicuna-13b-GPTQ-4bit-128g)
+- [LLaMa-Storytelling-4Bit](https://huggingface.co/GamerUntouch/LLaMa-Storytelling-4Bit)
+- [Alpaca Native 4bit](https://huggingface.co/Sosaka/Alpaca-native-4bit-ggml/tree/main)
+
+
+# Roadmap
+## Short Term
+ - (IN PROGRESS) Train a GPT4All model based on GPTJ to alleviate llama distribution issues.
+ - (IN PROGRESS) Create improved CPU and GPU interfaces for this model.
+ - (Done) [Integrate llama.cpp bindings](https://github.com/nomic-ai/pyllamacpp)
+ - (Done) [Create a good conversational chat interface for the model.](https://github.com/nomic-ai/gpt4all-ui)
+ - (Done) [Allow users to opt in and submit their chats for subsequent training runs](https://github.com/nomic-ai/gpt4all-ui)
+
+## Medium Term
+ - (NOT STARTED) Integrate GPT4All with [Atlas](https://atlas.nomic.ai) to allow for document retrieval.
+ - BLOCKED by GPT4All based on GPTJ
+ - (NOT STARTED) Integrate GPT4All with Langchain.
+ - (IN PROGRESS) Build easy custom training scripts to allow users to fine tune models.
+
+## Long Term
+ - (NOT STARTED) Allow anyone to curate training data for subsequent GPT4All releases using Atlas.
+ - (IN PROGRESS) Democratize AI.
+
# Reproducibility
Trained LoRa Weights:
@@ -37,9 +136,9 @@ Trained LoRa Weights:
- gpt4all-lora-epoch-2 (three full epochs of training) https://huggingface.co/nomic-ai/gpt4all-lora-epoch-2
Raw Data:
-- [Training Data Without P3](https://s3.amazonaws.com/static.nomic.ai/gpt4all/2022_03_27/gpt4all_curated_data_without_p3_2022_03_27.tar.gz)
+- [Training Data Without P3](https://huggingface.co/datasets/nomic-ai/gpt4all_prompt_generations)
- Explorer: https://atlas.nomic.ai/map/gpt4all_data_clean_without_p3
-- [Full Dataset with P3](https://s3.amazonaws.com/static.nomic.ai/gpt4all/2022_03_27/gpt4all_curated_data_full_2022_03_27.tar.gz)
+- [Full Dataset with P3](https://huggingface.co/datasets/nomic-ai/gpt4all_prompt_generations_with_p3)
- Explorer: https://atlas.nomic.ai/map/gpt4all_data_clean
We are not distributing a LLaMa 7B checkpoint.
@@ -50,9 +149,10 @@ You can reproduce our trained model by doing the following:
Clone the repo
-`git clone --recurse-submodules https://github.com/nomic-ai/gpt4all.git`
-
-`git submodule configure && git submodule update`
+```
+git clone --recurse-submodules https://github.com/nomic-ai/gpt4all.git
+git submodule update --init
+```
Setup the environment
@@ -78,6 +178,10 @@ accelerate launch --dynamo_backend=inductor --num_processes=8 --num_machines=1 -
python generate.py --config configs/generate/generate.yaml --prompt "Write a script to reverse a string in Python"
```
+## Need Help?
+
+Join the Discord and ask for help in `#gpt4all-help`
+
# Sample Generations
### Provide instructions for the given exercise. Leg Raises
@@ -104,7 +208,8 @@ python generate.py --config configs/generate/generate.yaml --prompt "Write a scr
### Come up with an interesting idea for a new movie plot. Your plot should be described with a title and a summary.
->1. The title of your movie plot should be "The Last Stand". 2. The summary of your movie plot should be: A group of mercenaries are hired to protect a small town from a dangerous criminal who is on the run. However, the town's sheriff is not willing to let them take over and decides to take matters into his own hands.
+> 1. The title of your movie plot should be "The Last Stand".
+> 2. The summary of your movie plot should be: A group of mercenaries are hired to protect a small town from a dangerous criminal who is on the run. However, the town's sheriff is not willing to let them take over and decides to take matters into his own hands.
### Reverse a string in python.
> In Python, you can reverse a list or tuple by using the reversed() function on it. Here's an example of how to use this method with strings:
@@ -114,42 +219,43 @@ python generate.py --config configs/generate/generate.yaml --prompt "Write a scr
> print(reversed_str) # Output will be reversed version of the inputted string.
### List 10 dogs.
->1. Labrador Retriever
->2. Golden Retriever
->3. Beagle
->4. Poodle
->5. German Shepherd
->6. Bulldog
->7. Rottweiler
->8. Boxer
->9. Shih Tzu
->10. Pug
+> 1. Labrador Retriever
+> 2. Golden Retriever
+> 3. Beagle
+> 4. Poodle
+> 5. German Shepherd
+> 6. Bulldog
+> 7. Rottweiler
+> 8. Boxer
+> 9. Shih Tzu
+> 10. Pug
-### Write me a poem about the fall of Julius Ceasar into a ceasar salad in iambic pentameter.
->The fall of Julius Ceasar into a ceasar salad
->In iambic pentameter,
->The mighty Roman emperor
->Fell into a salad of lettuce and croutons,
->His empire crumbling around him,
->As he was devoured by the hungry diners.
->The once mighty emperor
->Was now just a salad topping,
->His legacy reduced to a mere garnish,
->As he was consumed by the masses.
->The fall of Julius Ceasar into a ceasar salad,
->In iambic pentameter,
->Was a tragic end to a great man,
->Who had once ruled the world with an iron fist.
->But now, he was just a salad topping,
->His legacy reduced to a mere garnish,
->As he was consumed by the masses.
+### Write me a poem about the fall of Julius Caesar into a caesar salad in iambic pentameter.
+
+> The fall of Julius Caesar into a caesar salad
+> In iambic pentameter,
+> The mighty Roman emperor
+> Fell into a salad of lettuce and croutons,
+> His empire crumbling around him,
+> As he was devoured by the hungry diners.
+> The once mighty emperor
+> Was now just a salad topping,
+> His legacy reduced to a mere garnish,
+> As he was consumed by the masses.
+> The fall of Julius Caesar into a caesar salad,
+> In iambic pentameter,
+> Was a tragic end to a great man,
+> Who had once ruled the world with an iron fist.
+> But now, he was just a salad topping,
+> His legacy reduced to a mere garnish,
+> As he was consumed by the masses.
### What is a three word topic describing the following keywords: baseball, football, soccer:
->Sports, athletics, games
+> Sports, athletics, games
+## Citation
-
-If you utilize this reposistory, models or data in a downstream project, please consider citing it with:
+If you utilize this repository, models or data in a downstream project, please consider citing it with:
```
@misc{gpt4all,
author = {Yuvanesh Anand and Zach Nussbaum and Brandon Duderstadt and Benjamin Schmidt and Andriy Mulyar},
@@ -160,7 +266,3 @@ If you utilize this reposistory, models or data in a downstream project, please
howpublished = {\url{https://github.com/nomic-ai/gpt4all}},
}
```
-
-### Alternative Download Locations
-#### gpt4all-lora-quantized.bin Backup Torrent Link
-magnet:?xt=urn:btih:1F11A9691EE06C18F0040E359361DCA0479BCB5A&dn=gpt4all-lora-quantized.bin&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337%2Fannounce&tr=udp%3A%2F%2Fopentracker.i2p.rocks%3A6969%2Fannounce
diff --git a/TRAINING_LOG.md b/TRAINING_LOG.md
index 31b9bb21..50469645 100644
--- a/TRAINING_LOG.md
+++ b/TRAINING_LOG.md
@@ -160,7 +160,7 @@ We realized that we had two bugs however:
- We accidentally duplicated data and effectively trained for 2 epochs instead of 1
- We added an eos token to every sequence, even those that we truncated (e.g. long code that exceeds the 1024).
-## Conditonal EOS and 1 Epoch
+## Conditional EOS and 1 Epoch
Using the same parameters, we then trained a model using a "conditional" eos token where we only add an `eos` when the inputs are less than the maximum sequence length for one epoch.
diff --git a/data.py b/data.py
index dc404af1..8227de00 100644
--- a/data.py
+++ b/data.py
@@ -62,7 +62,6 @@ def load_data(config, tokenizer):
dataset_path = config["dataset_path"]
if os.path.exists(dataset_path):
- # check if path is a directory
if os.path.isdir(dataset_path):
files = glob.glob(os.path.join(dataset_path, "*_clean.jsonl"))
else:
@@ -92,7 +91,7 @@ def load_data(config, tokenizer):
**kwargs
)
val_dataset = val_dataset.map(
- lambda ele: tokenize_inputs(config, tokenizer, ele),
+ lambda ele: tokenize_inputs(config, tokenizer, ele),
batched=True,
remove_columns=["source", "prompt"],
**kwargs
diff --git a/launcher.sh b/launcher.sh
new file mode 100644
index 00000000..ed7b99cd
--- /dev/null
+++ b/launcher.sh
@@ -0,0 +1,88 @@
+#!/bin/bash
+
+# Display header
+echo "=========================================================="
+echo " ██████ ██████ ████████ ██ ██ █████ ██ ██ "
+echo "██ ██ ██ ██ ██ ██ ██ ██ ██ ██ "
+echo "██ ███ ██████ ██ ███████ ███████ ██ ██ "
+echo "██ ██ ██ ██ ██ ██ ██ ██ ██ "
+echo " ██████ ██ ██ ██ ██ ██ ███████ ███████ "
+echo " └─> https://github.com/nomic-ai/gpt4all"
+
+# Function to detect macOS architecture and set the binary filename
+detect_mac_arch() {
+ local mac_arch
+ mac_arch=$(uname -m)
+ case "$mac_arch" in
+ arm64)
+ os_type="M1 Mac/OSX"
+ binary_filename="gpt4all-lora-quantized-OSX-m1"
+ ;;
+ x86_64)
+ os_type="Intel Mac/OSX"
+ binary_filename="gpt4all-lora-quantized-OSX-intel"
+ ;;
+ *)
+ echo "Unknown macOS architecture"
+ exit 1
+ ;;
+ esac
+}
+
+# Detect operating system and set the binary filename
+case "$(uname -s)" in
+ Darwin*)
+ detect_mac_arch
+ ;;
+ Linux*)
+ if grep -q Microsoft /proc/version; then
+ os_type="Windows (WSL)"
+ binary_filename="gpt4all-lora-quantized-win64.exe"
+ else
+ os_type="Linux"
+ binary_filename="gpt4all-lora-quantized-linux-x86"
+ fi
+ ;;
+ CYGWIN*|MINGW32*|MSYS*|MINGW*)
+ os_type="Windows (Cygwin/MSYS/MINGW)"
+ binary_filename="gpt4all-lora-quantized-win64.exe"
+ ;;
+ *)
+ echo "Unknown operating system"
+ exit 1
+ ;;
+esac
+echo "================================"
+echo "== You are using $os_type."
+
+
+# Change to the chat directory
+cd chat
+
+# List .bin files and prompt user to select one
+bin_files=(*.bin)
+echo "== Available .bin files:"
+for i in "${!bin_files[@]}"; do
+ echo " [$((i+1))] ${bin_files[i]}"
+done
+
+# Function to get user input and validate it
+get_valid_user_input() {
+ local input_valid=false
+
+ while ! $input_valid; do
+ echo "==> Please enter a number:"
+ read -r user_selection
+ if [[ $user_selection =~ ^[0-9]+$ ]] && (( user_selection >= 1 && user_selection <= ${#bin_files[@]} )); then
+ input_valid=true
+ else
+ echo "Invalid input. Please enter a number between 1 and ${#bin_files[@]}."
+ fi
+ done
+}
+
+get_valid_user_input
+selected_bin_file="${bin_files[$((user_selection-1))]}"
+
+# Run the selected .bin file with the appropriate command
+./"$binary_filename" -m "$selected_bin_file"
From 053f2e4f245b4f86523e27b1e78e382d560383f7 Mon Sep 17 00:00:00 2001
From: Andriy Mulyar
Date: Thu, 13 Apr 2023 12:56:08 -0400
Subject: [PATCH 44/63] Update README.md
---
README.md | 48 ++++++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 46 insertions(+), 2 deletions(-)
diff --git a/README.md b/README.md
index 821eb0ff..3b86f94b 100644
--- a/README.md
+++ b/README.md
@@ -1,8 +1,12 @@
GPT4All
-Demo, data, and code to train an assistant-style large language model with ~800k GPT-3.5-Turbo Generations based on LLaMa
+Demo, data, and code to train open-source assistant-style large language model based on GPT-J and LLaMa
-:green_book: Technical Report
+:green_book: Technical Report 2: GPT4All-J
+
+
+
+:green_book: Technical Report 1: GPT4All
@@ -27,9 +31,49 @@
+## GPT4All-J: An Apache-2 Licensed GPT4All Model
+### GPT4All-J Chat UI Installers
+Installs a native chat-client with auto-update functionality that runs on your desktop with the GPT4All-J model baked into it.
+
+[Mac/OSX]()
+
+[Windows]()
+
+[Ubuntu](https://gpt4all.io/installers/gpt4all-0.1.0-Linux.run)
+
+These files are not yet cert signed by Windows/Apple so you will see security warnings on initial installation. We did not want to delay release while waiting for their process to complete.
+
+Find the most up-to-date information on the [GPT4All Website](https://gpt4all.io/index.html)
+
+### Raw Model
+[ggml Model Download Link](https://pub-7382deb27b3e46cd959cc60ce3efaa5a.r2.dev/ggml-gpt4all-j.bin)
+
+Note this model is only compatible with the C++ bindings found [here](https://github.com/nomic-ai/gpt4all-chat). It will not work with any existing llama.cpp bindings as we had to do a large fork of llama.cpp. GPT4All will support the ecosystem around this new C++ backend going forward.
+
+Python bindings are imminent and will be integrated into this [repository](https://github.com/nomic-ai/pyllamacpp). Stay tuned on the [GPT4All discord](https://discord.gg/mGZE39AS3e) for updates.
+
+## Training GPT4All-J
+Please see [GPT4All-J Technical Report]() for details.
+-- Zach this is for you.
+-- Please upload our cleaned training data to huggingface and link it below.
+--
+
+### GPT4All-J Training Data
+
+- We are releasing the curated training data for anyone to replicate GPT4All-J here: [GPT4All-J Training Data]()
+ - [Atlas Map of Prompts](https://atlas.nomic.ai/map/gpt4all-j-prompts)
+ - [Atlas Map of Responses](https://atlas.nomic.ai/map/gpt4all-j-responses)
+
+### GPT4All-J Training Instructions
+
+
+
+
+# Original GPT4All Model (based on GPL Licensed LLaMa)
+

Run on M1 Mac (not sped up!)
From 5467ec78ea6eb431ecaf257c94f4541414a5381c Mon Sep 17 00:00:00 2001
From: Andriy Mulyar
Date: Thu, 13 Apr 2023 12:57:12 -0400
Subject: [PATCH 45/63] Update README.md
---
README.md | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/README.md b/README.md
index 3b86f94b..0071ea05 100644
--- a/README.md
+++ b/README.md
@@ -56,10 +56,13 @@ Note this model is only compatible with the C++ bindings found [here](https://gi
Python bindings are imminent and will be integrated into this [repository](https://github.com/nomic-ai/pyllamacpp). Stay tuned on the [GPT4All discord](https://discord.gg/mGZE39AS3e) for updates.
## Training GPT4All-J
-Please see [GPT4All-J Technical Report]() for details.
--- Zach this is for you.
--- Please upload our cleaned training data to huggingface and link it below.
---
+
+
+
+ Please see [GPT4All-J Technical Report]() for details.
### GPT4All-J Training Data
From 855bc17b1df867fcbe3c05aaf08bb8690d3f7483 Mon Sep 17 00:00:00 2001
From: Zach Nussbaum
Date: Thu, 13 Apr 2023 17:41:07 +0000
Subject: [PATCH 46/63] fix: gpt-j data link
---
README.md | 9 ++-------
1 file changed, 2 insertions(+), 7 deletions(-)
diff --git a/README.md b/README.md
index 0071ea05..cb991346 100644
--- a/README.md
+++ b/README.md
@@ -57,16 +57,11 @@ Python bindings are imminent and will be integrated into this [repository](https
## Training GPT4All-J
-
-
- Please see [GPT4All-J Technical Report]() for details.
+Please see [GPT4All-J Technical Report]() for details.
### GPT4All-J Training Data
-- We are releasing the curated training data for anyone to replicate GPT4All-J here: [GPT4All-J Training Data]()
+- We are releasing the curated training data for anyone to replicate GPT4All-J here: [GPT4All-J Training Data](https://huggingface.co/datasets/nomic-ai/gpt4all-j-prompt-generations)
- [Atlas Map of Prompts](https://atlas.nomic.ai/map/gpt4all-j-prompts)
- [Atlas Map of Responses](https://atlas.nomic.ai/map/gpt4all-j-responses)
From a93457199c80e1038a0aeb94159da895fce71696 Mon Sep 17 00:00:00 2001
From: Zach Nussbaum
Date: Thu, 13 Apr 2023 17:52:04 +0000
Subject: [PATCH 47/63] fix: map links
---
README.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/README.md b/README.md
index cb991346..f77b4993 100644
--- a/README.md
+++ b/README.md
@@ -62,8 +62,8 @@ Please see [GPT4All-J Technical Report]() for details.
### GPT4All-J Training Data
- We are releasing the curated training data for anyone to replicate GPT4All-J here: [GPT4All-J Training Data](https://huggingface.co/datasets/nomic-ai/gpt4all-j-prompt-generations)
- - [Atlas Map of Prompts](https://atlas.nomic.ai/map/gpt4all-j-prompts)
- - [Atlas Map of Responses](https://atlas.nomic.ai/map/gpt4all-j-responses)
+ - [Atlas Map of Prompts](https://atlas.nomic.ai/map/gpt4all-j-prompts-curated)
+ - [Atlas Map of Responses](https://atlas.nomic.ai/map/gpt4all-j-response-curated)
### GPT4All-J Training Instructions
From f10f8099db51b0067e9068b6d90143922e2f4e8c Mon Sep 17 00:00:00 2001
From: Zach Nussbaum
Date: Thu, 13 Apr 2023 17:58:59 +0000
Subject: [PATCH 48/63] fix: update config
---
configs/deepspeed/ds_config_gptj.json | 24 ++++++++++++++++++++++--
1 file changed, 22 insertions(+), 2 deletions(-)
diff --git a/configs/deepspeed/ds_config_gptj.json b/configs/deepspeed/ds_config_gptj.json
index 3e933966..6f9b2961 100644
--- a/configs/deepspeed/ds_config_gptj.json
+++ b/configs/deepspeed/ds_config_gptj.json
@@ -24,5 +24,25 @@
"allgather_partitions": true,
"allgather_bucket_size": 5e8,
"contiguous_gradients": true
- }
- }
\ No newline at end of file
+ },
+ "optimizer": {
+ "type": "AdamW",
+ "params": {
+ "lr": "auto",
+ "betas": [
+ 0.9,
+ 0.999
+ ],
+ "eps": 1e-08
+ }
+ },
+ "scheduler": {
+ "type": "WarmupLR",
+ "params": {
+ "warmup_min_lr": 0,
+ "warmup_max_lr": "auto",
+ "warmup_num_steps": "auto",
+ "warmup_type": "linear"
+ }
+ }
+}
\ No newline at end of file
From e1199b0f674da1748d8945cd9adae193333e62e4 Mon Sep 17 00:00:00 2001
From: Zach Nussbaum
Date: Thu, 13 Apr 2023 17:59:19 +0000
Subject: [PATCH 49/63] fix: train gpt-j command
---
README.md | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/README.md b/README.md
index f77b4993..254695bd 100644
--- a/README.md
+++ b/README.md
@@ -67,7 +67,9 @@ Please see [GPT4All-J Technical Report]() for details.
### GPT4All-J Training Instructions
-
+```bash
+accelerate launch --dynamo_backend=inductor --num_processes=8 --num_machines=1 --machine_rank=0 --deepspeed_multinode_launcher standard --mixed_precision=bf16 --use_deepspeed --deepspeed_config_file=configs/deepspeed/ds_config_gptj.json train.py --config configs/train/finetune_gptj.yaml
+```
# Original GPT4All Model (based on GPL Licensed LLaMa)
From 3154e6333e13515fd2e85f08f25cdae69d6872eb Mon Sep 17 00:00:00 2001
From: Andriy Mulyar
Date: Thu, 13 Apr 2023 14:33:52 -0400
Subject: [PATCH 50/63] Compute partner
---
README.md | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/README.md b/README.md
index 254695bd..f7126057 100644
--- a/README.md
+++ b/README.md
@@ -31,6 +31,12 @@
+
+GPT4All is made possible by our compute partner Paperspace.
+
+
+
+
## GPT4All-J: An Apache-2 Licensed GPT4All Model
From f92d9e896f80c5e226cad228feac019f87101ad3 Mon Sep 17 00:00:00 2001
From: Zach Nussbaum
Date: Thu, 13 Apr 2023 18:41:22 +0000
Subject: [PATCH 51/63] fix: figs
---
figs/perplexity_hist.png | Bin 26205 -> 15322 bytes
1 file changed, 0 insertions(+), 0 deletions(-)
diff --git a/figs/perplexity_hist.png b/figs/perplexity_hist.png
index 08fc18e2e23605bc176c07c6adca037afe318d9f..be3780b07bea2c62e34b20e68cf763bd3415de4f 100644
GIT binary patch
literal 15322
zcmeHuXINC(*5(0PR7}@aj0gryAi;nV#f$|4O3tVVN(RZ9)>hkuiUdWHl8Yoc6v+e%
z0ul<4ARr(~K*^bTx7~g3{bu@$^JDIhd7f^4q>8F@_Fj9fH=UcO6=arhtmB|4Y60J+5J^Y-wccaPG1pC4bJ=%FNQ%%=r99d&A2%#+DX)
z1rG@B-nH?Pt*w=fxR8+f-`^l;dHJGH1IL0J_>hHGCpB#-YS}sRKTEW9lrcp~$k2`*
zR&fmMYjV+1Rh^ytK6bc$+kwNap69*Kn>>s3d9Xi>cGWnt+h^66t=oz!2fbsal^7b&
z{M0@b<;~k~py@6bWt+NtU(nB%jyp>}$R1bzkmK29wS08oUS3f59slYUu5-D;_ln#l
zB~A5fI^$%Pxn+#%P1WrxN&X2}WZ;;|z+Y76Dwe-dRDtkMhwzVAf1_9^YVlDD?+{-6
z|6lmOS(3KyepY(&;NayA2VFH-SXh!jn)M6Lrzjovl4RS-VdLW`P87d-^*&3OwZ+nE
z{M)(X$B$<*7E_cJ=TdIT=*f{@24iu?MrxKFzw7Njxon59@P|?+MR}H-Iej`bI(kP@
ze%rRqcwM@Abjjkyyc8>S`pu*1algk0^2jEh_~%WOWLrQ*uPQDsR?*Vx>EC0uB>GI)
zbxA3yyY7VpwPe#(_0FApx^#m$Z%J~aE=R(0cX#(&zx{UK-`}4hm$hv
zX=!O@ytcn4&nr#82ygG7H%Qo@ni}m>K6UC&l8{T@O4f~3s4TyLfUlt@T{|Pe)y=I*
zcWDAkUS1wuGbM5q1C?tw{JG!-hTU~ZVJGga
zX&rBTe(q@QWKZmBDfhYA8LVP`l7UP_?REZFdp#*nlWt$8i%xy>TZtz+OsA}@ELnMZ
z<}FTs+pZ6r=FOWIW8GHRQ=1sVzIe?>T(^i>-Rb#Dx$X~F_0+7l$UMAr=g#bbys5xZ
z-M5}BY(Bxk{o^`1I&t=W^~RlLLG^iV46O_cH&weZlPy@@mTk=*bGh{ymWeGbXXS$>
zHItH(21iG0{Po>q<}F$kt(WVZ-5Kn{ODmV2%i(rvU(UI5<)z8t?w+csdp*q9C;{#C
z=(xC3-l^wav5)jOX@C6q(W^OnZs6K(gTgy!E?fw=?{Dm`7jp5Hm6dh$mD}2~N|>T7
z2I~y&y{oEH*3PhC&WtrD8kGigPxdCN-{Y1_J^zNoZs1dswb`LVo?q=50sD-fiP?O4
zTU)En<2L-DrzW1C(Im)dO1OXjZ!{$hjYnYuJ9fOazU#}c-V-u69i`_!(J^Eq=02Ut
z?byP?j8#`8+kW>hQ(_=TxwYA;Dcdf)!%w@tuP^?$-xl0me?S4NV)JzXTXTKM`pD&5
zPDCpNiOZfk<-=S0X!lVp2VFJR@7c3w!?@;{=g*(N?A<$$bddw1_>S3$au1WP4^Q!)
zntAuK>WYmN6&0Crn#xriQZmQB=`(kD>|J~h+bBo^@A6@k>kVJ2YAs8IBlg`mrzNir#ee
z`VK)sg{iRt>48ViBR$5$T@_yC_oQbt56+Go~f4~0JtHm2B
zs`h^AlLMwjUK_H%7p_;;(fN9bN5VdCdS<4hvok72Im)EiXD4>P!BE$xCQeh_Z8B9!
zNlDQ86i)9p`GD}cI*n_;{`$7LS)a#aI_dJ|%Yw1@aP{~>bCT}c{w4+|56{59l#h9C
zE=7tHGjZk(>77AS@3x0T<09zliNUO_ti#xc?(y$!;x40K4Vm*VlG)K1ilc_pSzK0T
zZ<0Sd*~`qdZW~%<
zqD$5YAMzfqloRSstJSNjOEO5a@6+n5OFDY<=1qUMCleE$A+=r`B)A$Zzc**R9vf&*
zWDMkK<5*Qs4df@r#qBD6yjR9Mp=-bB>{aao%(laqE=Bj}3@MZbOKD>>#*c@vO=|{;
z+i|_z_cR9MF4qs(A
zsd#qHz_hYGsxxi9h)Mebz1_QaYvnqp?Ki6nZDKg|mV3;m3=R(Rh0ldZjn)~6JAMo0
z_L#PI8!A08In_Lua532F1U@XgT`@z%qERCvA_A-Jmp3=-Le7@muB9)Sn|^e&yX!!j!ePwkGdAagRHm_HbKPoZt7hqFDJS`*FloP;rJC4a}5_v$Q)4)4#<+og!4CB$T%ylkdhB27TT<0D
zkD^ZRtjQkFI4=>AnJ(RO;4C}3RVF5vxI=#+qePhFNB7c9Gw~I-?FvQPzjpn4Pd{Uz
z#-Z8G;e~qC^$Lzfiyq<&$aUbm0>^aK5_DqH%^Qjv8ni>)Mjz)*^-GL&RUB7KVq;~+7)>BQOUxY&(DRtF%N;D{S0BzTyt6iJtl2}K^j}~sIm)SAfX=#vDO*xIJLy1LG$gmD~X8*jTO
zO}A*uo*XO@>KGVEM75bW=jq04XVgT6xYN<^jlX|>Lk2c)H?EAXpPyG*Rb@9fGp7G7
z0DGPhu-}}Hd9N9z7@~&pDB?0|Hq))>ksTL+VQMI#m!p9>o!hmWZT`|~G~-mWk9&f|
zY)+(^)~Y!>XLGxLJ4{nDHjY`h-z*yAvnwVl__C|NVToUi%sp=ZuldJsuOep&)74&c
zWcKT`Io;^NxtRoxM=U>41@fc)O|kR29S_y&d*tGtnLtLg?DjkAuC6(4<(oEb%5J^B
zKGkK+f~NGB&A!c<;V__PZJmVTcCF~mpXz6rH0R|4IAD6<>^c@rV2x2c4rb`o2M4Cn
z6cx}u$eY6Afjh_y
zTX*jGlp0kf8n>i`T*{DnN5~z`aqXfuhz&Xfztf;>9XfR?pE#
zR5diZK3lRb1yE3K&-a2is_!VrFYniq=4kWfy_jV)=60HS!+!Zk!iO8u
z&8z0~xQ2fHYSv^^E~QWeAk!V2m{$p~m*zTgIbJ_s@9O+z_DprOs(#E%%*Go0#mus~
zC}LW3YP(|aG;?UbMPt|a(7T7*(=L{ClOKBzW6s5*9td#NN;8Qd?Z9Ppy3_jEqep8d
zCMWr}Y-zhv`|0_)s|k#LajBW%s#~{j6Zj2m-BLHu-5pbScfCaV{rmTa5^&Fn%_VM*Dq}Idp@}gsG>We3`Pf<=VA@6@T+faW^=67b?CfUzz+l}^rSQr4WA}0Z#{qhds)nel4=y(Y|1Kdgp({A78_LHPE^PNBW>{Q<8I&Ot(Q3EfT>opuXSha88bQhU5|v%T?uv_q{#=0o+%9wQ4TriW;@(L@a);9aKh
z;0bo=7n8!ZugYF*E4;PH&>s%?^yw2ARzhk37_H7ztuktsYnF6DOtxcBpFUNXYHMq=
z>#5!e+$B~IxElt(jTIP;w$DAQYd9b5IRZT6ju~CO6ohw|0N+UbrFQ_~GcFVm>v(poN0s?5cuc|XF
z_0=dgaL+`vI*blFkd9Y5AQy-N5N!FusknpF-;UChdTZi?%Cs5w@#I2iTB(;F5ZZ(r
zEWz;2bs4L9|9G#ik~C0_1p3VQ(7y3EI%O}vpjdNfMjJRaS5XCb>Em(&r(cW@v{uqP
z2#&O=jy`j=k~!zPjvfUpCOg^f@SAlWpBNxgK{*&bVF7>=G3d*K-_BaRy5MKn{qa^Q
z=4Ve%cBc#N_3$TRg32OpZf@peREy`W*H^KJ{~Zwa--m7gLcbCDR}(F8RNF5Cl#Orm
z<|7z6vL{b|_>}7s*(L?z6d=kLe8Ilu{_w@Jz(bcojcVJ6%WQ>p2>LwbaHZYCkzkUj`TIClgSU}
zoMu?E3Iv4ajTsz~pv&m_@boBVaZX&`A`Xs~D_4f&l+I&kR~vUIaW$~Lrk|4hW_$iu
zpeZUdym)qu!58g~W1DQ*e6Zuo7rsrKUR12YFF`NaiGk}4N*9BhmKlE-{_C$lIiWG4
z;jYfL2jl=!3*3)Fr$^S$DV%9;iA4tgmMyO;)S#uI`RzP-P|tfM4^JYF@=kt!
zz<2N4BPhtMP8d5fmaL8*LaC^#?x6rkRl!OxyK;+|
z`0yGRc(TYUD%L*QZD0%Nv4mSvBT6Ai2@E>bgvkeTBwD8V*R5n$7Z<7%Uz|VxLy+UL
zmI=pY8>tqpA{Q#Qm9!2jbKn1rLj4P!#s3Q9`%f=7CY|LuggCNGgdDiJF@cig1NPDU
z48^>XcFAGznR$pp1qtZvTM0#q+z`AL!}3OG|W!
zKA^S~OfyTeN%xT9N8mQ+;jX9HbecEe0S=#>WS$(b^c$}F+6?qaC!8DT&pSNhld~My
zoL0KoCZFWGR}C
zEDm9j7>Kq>Z9>qVy%;rHa2FDpuI$gDxJ=FrcqF#BUzquR&jZX=F=BYHMWZMr%V@|?
zbjZU%G{c{73juD^yzy!LG>&D<9K?krB^#3l@;QNkGBfMGXN87_Dw#mmjy@$A!qS*-
z{sfmBb0+M>JBy4TN<~|`*>PtGAw&isQ>`9vStY0wO-K}|U`~%|I`zav?ZZWr1_$wxuv3D+pf<>Q
z1J3g96cQ@$#w?R8%A521NAGyS5rZyT;MDYUAuOz*9=G~Ats;O+JqeCA!ng^
ziksseRPQ#hw$#LHk3b3Ci4g%n_MxjHYy`@W@kkGo+?Ii1KeOjkE}t`*ETmKs>Ci
zjF2O+x5;H7*P9p3op8i!tMxwI;Bk)@%AZz(l%bBs;yn6QlcgC_MN57A;PH(F?P+NR
zB&Vb>p@`;=)MyjpV~{S03+?Xi26$|>?70Yd1dw}$+vV%-4N}f30O^xMA#-s=cJx}i
z*XT}Jpjg<8tMhwN(ijU@F(Fiq7b%ndn76;zU;1t*=|SSMIb8QsdBLx^8X{{s5(qK)vMTJ%FAk
zAOWg3=eg*R4bR)*8+^F8vxlXoeWhhz&038{mvIuUWGV4e;)m79|@3>CBuN$6MIc$p}5W
z603}%O-e4IpRD^FSzQ5vQl>6IOUr7V*~Ou}K@tuMA+CcrX9xZByN0`~yu-0*1ler(
zu^(>$P2sDY23RKK5Q-TzM>G=5hB@$iTZX?;oM9V^)_-4lR~mhW6CLDwsb{(rwZfniCyzE@b`6tIG~Xqq!5+$GJ1e
zLFbSC=cWL*5^={SpmmezfGQdq(P)%l_hJGZ=)s3NtehwxUbq*~Im$QaVvMgT$BxHm
ze9CU0ot>Ft5Q;+@*HgJwt?VN7A(b&GMbo}AYK~9_qarzB}CGx(wKkOE){$G
zbjW%In^9iWYO4}UmbnZtefh_GI&wZ9#?Mr8OgB@q(gKUc&^8z)_(}9AwxtDKp=sc=BICzq}!$Opeex^|{#3jnp
zlX^J8SpyCH;8|7SP8UGAz|`6Be1R?6k^YDHn*-G{*1lUMEk=8L404q4nYrgE0|*N>Fj=$rqQ7g
z5a6W7f-B4PEMYr)`0y{#`0e$k!10Ms`j2Zyg{og=Us6dlm!mh9c4POQq0E8J$3bMx
z%I3m#v}I1OEkL!I!hmu9_F}%Ome!>5^59@u1%(zYvI?HP#<)5g;stJ(VjgF3f*@G+0tjU~S_3iD0EvhL-@*VB%(V8hnUZV%yCNE(R
zIY;>(L48D#%NiRSd!<9>ED8}Qsr(P%h4v=9M_=HjHtN3QT<>&og@ZpXUM|a({Tp!#
zBNc+^u%so@ieA6&D)@zsv!Cj?p=Ec10Ob4UDOZ}E0VHBSo&3Dp9Mq4h*@
z^hUvsA+`pzJ>KrfCl-}pJ^MUnUBT2_=n0Pl0_e~*5?-3bd0CWdw)Fp;@Uo;|5bNuI
z8G{O+k_zpr$*xuxFsK5Ey!zSkMMT#FPl$i+4*8%u$H^utov5G~NZuDR8n)7)w<#1s
z>k%8B$$+r}MlO*K9Z*dp7oVGeMqh^y>HC->LjVnIT!Qj}3FOErgNncx7XsCrUqYfG
z#=JaODjSfFcP4e?>)#nmQa;QRLKa7@a
z8{>ADOH3)F$^OYU#o#!&WcomwHCN^>!fh#L>@eT|1Rnd-)q0MHG23r>dDZ;7aK%b!
zq&VYhCIrXyw^vdcs4%zhZ&nL-_rj>E0Y>o!n1Xz$hXL$7!c_P5@o5)@pSoKdTg$$T
zI}WQ8gA+>BK}hd9zvIrjxZ?4JUUF;1E{9{+MGT8qs@rSt1Tp-bcTUS@Z3vtw?BdAZ
zR<=6iJ({5~?T7o5>B{*HCv_O#-2-+@w3_m%B`ts~nSYFwU6ece0AS8j(6tDN#3v_>
z&lC#5nNjy84R4qKyaF^-(k3woeTS(zKRIX%|y470g0_pw_AqN
z$2c9MX5YUtRm3WM^UzsEneT6KDj)y;o(He?W_?|Is3dtGkdTv$i=6kZTetRIczb_H
zjl}~^CxMT2x|4#7JlWV6TTM15>PxHJppE3kkaVR`p{xwrK
zj@cw2)WT}2+@$-$x7X)$KL=gF=)WBhTC<#rI`w$-?_gH6o~)E@*J-G~#qgjpe$`<7
zPu(J9{(%V(m0|YrS^ji~kjcm72<0iXw-HFV#_znhB8U-$Y5Jqn$H?NsYGZ0x5HUCn
zp~QJ+q*j$-Z!Y5kGw73J+Y(|w!k@nZvJsn_y3f4s`Qh~kENl%3yYNlanqTnr=neQ>
z7Eo=hOv<6$Vz?>0K&&8koS~6XcJ?W})1Od&Uqhe(LRuhmU;dN_=>o?ajz?XL6x3@7
zyQ_1Jpl<-rYzTpd!WXB9-&BwB10Y()UHz@~RT_v7NV`g%owBkr=v8|i*ahF$db}+`
zNr3IFnwp;%w-g;!`xay*I5?QxsZm+rPT-4cH*Um+haUs|zl>;^kRrt5XSMP)+8NxZ
zfrR6?vs*n@pnz_v3s|x^^fzkiyH7+YH-sL(TAgaFe0PI{+L<$F$Yj~GXHPZ5!IGu~
zqyftxLWf61ud|CwNkaoe$&|1+Gv*$O{imOzVL&OE!|hPQfD4m?ypD#X1fdjm(+JEn
ze&{5irLhvS?AWl!CTl67BL2jj&Ctb3q6N|tq~>Zk-A%fHl7}MpZll~MyQ36B++1Dv
zT`c2*L0=>_4@I9zOVjxY_|o$>N!I+=oZSDE*Z3*0&;b`vN#E9N}Ps5UyC#R_KRI&s@*xJl6n@}TNfJjP$~kkL}K
zdy9-$wG5A2I2hI?ypNDiWK=o|>DUOqkSNBY=v}dD@s}q9!A`)lAf|Ua(#<0cx^D`f~;sn%^Lnw)#%`**BI`>a%LSX(gX2g8KMv2fu7UJ`-8w=Z0{
z(93|hrErRN?A<$br4|7hHGsP+RL*H8x@=^roLAJp5;7PbU=3j^4M|
zQyRSC5i$RW8R17XP5QG&%Te0yf;oncwm0H*fei<1KhW?1&osPl!r|yy?xg9@2
zT{^re6^uBADU6(02!b%)h-~l2Yk2V@KgRv{@8A1(tff37SmotT(-<;oQw09Gg@J9&AJ{qoB%&ePwEkhv-6IExF2
zS8bp~AurLC;6Vn}S5;P$5ksr|%QL2{v$L`9e}in%FMr%ZulhY)Tt%Rm4LNnIRY-d^
z7e3D^jB=U1`hTV-V$yFP@+`44En~%PaO&ARuuz_Sch(-l+yCEYLVpY)b-|K##(
zxBe8P=WhIdX(SlJd{?jHpl;P69+AmlEf9VG&+Wv5h0W@wy4rxX!$Nhl@Xo1zK^BT{
z-VZJGuM#90Kd}~U{R0Lzjy&y8K)LXj=w6Q;8
zDXoa?6~t?7TACUBeXl1R<1??WFX?D&3j;7A$+dpMfkKtmO0kU-{Qyv%sKTh_D6m#l
z#GG)})nL0ZO4ul!FpZ7B(#yKm769sWJ&@t9AdMYxlh_3a>N&&iY|@1ma42H8J^ORa
z2~AB+SsKkc70UaLO`8rse*Ac~WK?FR2HLV$IjE<1II2%J=KK5n+a5ExYI;oZuf1
zT>_U<3-Y3GeNkF*9pxy3pxHi?>K#R$V3$EK_2JOLS>>!($FaxIv;Ed`n3xM)Y2SJ7
zq?ihqCA<)2u(y3(=POnQmkP21tcbV|y=q$v*$c$&dZJI3SvEcMzV7%&Jv}`VV8Qg`GDo*Wp5!#566^Va
z`l59T%P84O)g7Blg7t1bv2tC{{2*4IfBB@iu-C=Olj6c7=Nq7xR
zJ{E=VcWBsQ>`Eyh`
zE(zFs(F-#04jvvJIF|~*=k^XXRV@<}x<%fE5qigtUAs;pgho_?s@-rzNSlYLPU0(Y
zoI=LDt#lsmRE~TLQ}^|6%f0p&W}Tn@_G%eP50JO@_QoRfXWo>}OlnTvMGWTU%a;$1
zj1YnbE48=qR%|eYFcN4$uOlyD9yDaEG=>pvFC6_YM+4HWlw9*=gvrmkpSz4SoXJ!~Xfx$B}-r
zHMtn*!T4+HtPaAtq3G0;{qoq!>gkCg2o>9ybX7T=E)e3&1|6y;M795SMEu2Vu69HT
zh5TuZzgIVUh_way_=J<_Xsmi-4P4?h_ZernGIgr@Po1Az$!8%F5r=*+e$e07R}-?T
z8uo5|EL{(<&V-f>5%?ltyiaJZy)G+imFRrru_1rl8RE?BOKom?ck}&TbNoM37ODl{wV!R+?84l#c
zrNMYDT7?Be2Pz6#+u57|1+6o;@Y0`h2@RbrxuH*?P3L!ecXye3%O5Vz_GC;IJ|Q9h
z->2@~_@@9#hlsjnzuq<0i?>$`rIWJ;AH5ertaQj5X7lFH*Tz=&3&s~a
z4_C;Xdf_#gMGD>Z$Y^Eo-6=YR+N
zqN1WkATetpq?5KvKYLo-*=smFWXa4cP#K07;>udVO;{TdUgWPWk+`pL2U
zxxT{USU(mvb`9u8DFC?^2t}1owp?8nhqtO_&6AO*3DWi(a?R6NiYC+tU0F;>z<~y@Q1#$kf%aEt5PuF*T(PNKgZu6o*@9v^J>d
z?k|~qV_FL0aJujNKO?n22k?IoAkP1QCr$3I+5PNgftf`c_)5@}3>@qqnLh}m
zBymDCSePw$xuvvF2+GKhR43?4xPftd4Y}v_P2w2|Vs?Pzs3|CTkp~qZ>=7uZjDwCy
z1bJ>DIy}7nnTQTO5wVjd$F{rcBpsi(jg#mn9y}r#Gst9OqP$#cERZ~&@~G()q6x2-
zNyj02-w@Lby%8+6`|g2H(T`sIhwW~vcEAa8zap2)sh2m%gYtLNevumc2&j$C}HH%h&BrJq2lB;;WYE5y?xq5pG1ro
z3FU-JBQ8vOH+c>Udil(i=2v)J2l3g#;bEU93xE6vmKau;k0h#>0SKUNv)kCa=H~p(
zH-4sgce5>MS(lD%DN3Did>l;Ta*U7+8*zFo!*5n4p|m)bEU|MP9UN3b+_Riv1a{AL
z?zA_W5Tdow(4u>?dlPKLM@xkARWvm}CXSB4IE3V87$D`64jZC7b>K3IHCI9@#)C3z
z@HNIA#lGmxrJVZnDZ%+He=87)g~}Unz%rPO*tKhkT$PJNF-Z<2Cnpn!D5}jvO#2JU
z_(|#LlS!dnyAU%XQWtrm0;^ija7aW_^YRbzV3;x*ibh3^?gHfdml49U1b@)Qc79kC8q
z;z=J)ezhIQPyp^C`d{l*YX+T~Jm~XUWKm5oIhx88b#GG$%sF
zl*}Y#d_RYK?|biO@8^BizSp~cYdwGbTJ7DfxUTCw&*S)hKhyUJIj~QC2_rWnMNvz3
zYpCc@)B-<>qG@BG$KOc5YyOHqww_WoI;HDqeag+k#fsW%amvZg@s!>1BmAeWTuvT$
zbl5B=D<-*--{#aQr<2>n#qIz61~EsMW8zJW4CnD8i=8wMo}?%i3-Ujjq#cRJDaxvP
zx5`dE_vrpFZboc|b23Ao;Vlo+VuLkU*;Jkhamy<1UWJ1@-So{lzRjA{AC8niY5m%5DTC~>
z>ODuLqL@|iYb`f-ikM$8exVzF|BLR1P0CZho`+dkYzGe>
zl#r70R}s{9-HPXTe_@nX(1!hYODiksxw$){nJg(U7JN^VVfx*a8h$~+I5(%MG5e2?
zj1B9ok`~godE%!R40Z6lXlO91s9T0hlo}ZsIgoyMd;54ErzBN;J~GieG&Gcp3c;G-
zk~Aw`_j*MuSt^2+lm=NbQp|qyU+P+>(?sv^Rb08zh(EWsqJl0`+>D-LV`DR~zAoHQ
zDawNlT@bJx>H_e;#oV)AO`CdnFjlZC?JMhAV
z3r^z$rV5^8T6GVWX-mhZr-yBG?NKr|HpbUQPqC-o}F%>;A+}*f1
z!R8WQK3RumziZd7+3*Q!)tQAZ^)pN}IU5?f78w9O}zthz%ls-+L
zVVvtwR6DRf&kISKOwHWvmtOtmO;z@ZdWlor2i(t`
z*{P|?l$n{?-CDAP;@`e~Ra8{e(D1Of^Mm_K4OZ7Ay`Nq0>g2552n@6+^`mj=tzN;#7Lb%A
z)b4FzA#f1OJ;W$evuZ7)<@FZJ2z`npSJ40j__E2Q({L-$aPukT232V+m5n8CQdmgeiKvE_O>>^mLktJ>(|rc
z@|BI(tXZS+Ku1nq-g0L0*V_*tcAYx4W#gtz*k1T;VQo!MWjubo{6^Hxn+#KOr@EJ2
zUcLRon;mn7TANr_E0o6WJZFC51e3S7cV0n(Wqq3Ip!cO?Z|(*rCT<`bsJ}mycF|HL
z!Ek7e3bMP6QBiDR#M**G{##+_%3t*zI4j1Mf?wQCnA
z507PA>Dd=AU$TBGQFt}|tKKYr=Q;W^)5P;Ee3j?C|fb#LCj9gmNgce?*j;3qW|
zX~WPvG|fex<2?1fJv~Qo6O2lHyxH|t^qf~KxckMchaDO2sF0A55Xybkd-qsUa`NI$
zo5)5M{$jRo-%_IqV8~2@CBu^ytx8*WORqZ0z4&
z-?=zCdW!P9cTd2NsH#3ghY%JP=9IQ&yL9Q21oorb
zwgkh78#fr+2R|n3y3bB;t-QSIqMsiXt#C%i#_d;+{`62eOGBEl+2#T_meZ$Cmp#6Eb-7lQ6kV#pqq0}GWt|6_w=SfoKl03X`};4A?=)@@96my=O$n-f
z_39m#$wTAZrJYq_zWuI#;s=e4O5eR3zB75~iS-i2slLta-UbFMr>Cbgj=fne&=$S*
zBt15JRaLt-{*&JtA*NXn8TxXy*ijt-1xUS_Qq7{j}M6v
zX`#WvG*^;tHD*}QHf9~Cb#!#BjoHDsdi83wG$k)D1u?T?W^wU!#vk8XpBycnbBMEP
z$PnoH^oWUaa&ajeZ}km(a_mjq>HbEe%p-IzE-u%X`gL(BxJ%%31-^?{xBONc^`f?x
zZ9vJ#mHuRVS)hch?D_0$Y5X%C5Ka5pT6FstZ{Dce*=@WfW6y~CjZCl!?6x?z@cQ`pbOWE<*GVqh|NOx7h6c`e@7`(b+2fD?
zLoO8sbnT_-x3E4P`YoOn_3smoKlp8&Ut%^+Z5G06#xJb>qejqa3>^otbg8
z?`!M#mH~>j{rC}rn|K7UbkJ2R{i?>=iQy_f?Sj+xStsh378e&gIXRW0>$c?G2@0Yo
zD>xuv;g;j?7m}aH?hwtA5ls2KRw_y>p+_(!wXjeDSeIW`HjTN^5g%NxTr
zzp~Nd+Vg!pJUryijI7;ux-VEgZ1uB`ANA_G!Zn7zt5qE6GA*5+d8J#ba=$Xiv32R9
zMT;658wIy+<=SjrTM8^t)BIIgMaA(v&6VJf2B%NUkham+XNT&)h?btYG*0!Rp6N8i)ah?J6jDtcQdc$MNtqK$O2XIz;*6JPy@5RW2o
z52ORIl~cmpulec7LOMDD5s^LaW=2NAH#QlQ>n5vXbkr;HX{wH%9vk54@WilXeixS5
zIXp_+3(L(|-HipxHP1)YC$K2Z5;DH+TQhHwmUeD0qhsckTb7oVhBc)$*iyv0Vg)Ubz`aM|;jBP&
z(Rc1-`pkHA|M(%WZ5ubP5E~G`mHB~O|DT~5!;{@0904kVI{{e~6cl1pQ!g4kIwT-1
zt#7mgCGG3CZ_0wem27KiQXX=$;(2^4dgLWNPm4MEdmu(Y%!cWLX^tuFR)@$sbS5Zq8!w!nL8EW&uM
z@AKwMoRSOF)aL0K@c4s_G3gVZ2u{ImJmxA{s8){jSX@LwdN+x*m^hk|3r>sM?Ndfr${lvu1hD=L2_aS~NHZPA8Z=$hp
zU*%caMNU7z?OnKNQAuXWiTbpA_#fdGvlE>xpc#u2_s58RXhkV{mvvkhwCD(`{!KY&
z!ON@U=_#R2n^bXQhlkpCMQ?Luth&6a{8w*1>#}A1c6=*VoGbR8it`*F(9qISK@U)z
zXrs+n6kM~0Ao}OeRW5ODrk|S`^bNmzcXnT=hDy)bM8A-fd#Q%$%ImK!-mFEsaBOTW
zMM<3QGumkSRL{X%TbmU>AAbAxMKCYnx=5|{yPTXPl$Dj+@RGM>9d8`BJaIyl<6&hO
zU)UP;%WVaN#iA?LU(W`$2WPNr%vub>;B@k2xtZ@wASigmad~jIQ$OC*Q5p74G7v&I
zrp5=W2s1izWI>3^f@HnPeXi35ck%4(T8m8?zToB){7sPKi4!L%zq4m)if1QJHG*|Lofu3cO4>EXe1;8ldFzpbmIqeuekaMU~jk%62l=6;H9@&+X1D_u8Q2Z#<+S=MKuD^f(z6X5^
zGK$IKUH0kIC-5ngy8wcn&G{$a
zzIzvLJb<-TTB>B%l(VEJQoO3CE|qLQP}TC%QVIZkPmJ~9Jg0W~m|Kq+1Ox;?kz@ljNq8Px>~xu?Yk
zq&sNRntFKVu<)ByH%-m8%W=CFP-KnaM|O1OtP%WGL22kMi2?)Fj1)7e1&{fgUb?DMg*sfQ0YPW=9D@#gM!o6nEeQKXx7e<_fKD6*ifBKsH@
z7Z-MeZVC%ftI@%OFE^cpE|A;mbFr*US-`8Zsw%R;+r%V#*##_)fJ>K{A&V`bva+)S
zMih|)j)J6jg5D=AzHuxeNh}I{n
z|CX%dWqfSm>8BMH6%l9VMjA{_O$nTBF7`I=p85mPW)5W!4-F}yW(2YD%~tvxuKM(8
z><%uyu0z;7&epa(XbEAYg+)b~z~lEHJP?2=1PZeWi1yNYhUtbnCbLkm+fmm*kIEBcd=7ov_aRr>eaIYl`3Tq-`RnjC>H+ds%(&LZ86#Aq{V|Hv;b!66F)?A-pwlzst?pxepXKXTaB?!>Ita*+mS*3uFS-W;4;=CX%gSx#8hQil)o+Z_H_63q;+?m`3mAl
zG$?XI)~$O`n;k#cfz=8qs|-9h`};c|-bX-K_{Sp$MZvi0axA0&5hm-49M!vyPZ)pZ
z=Qq$3FqKuHj)Bxz8i+CLs#T%baPA%+M^Mvl%DahCFHAWYD3lV?9tsN!uiUv4+TCr_
zzO4`{#hSHi7t+ws5Q_ZRs-`7*HP`^}R+oh=e2OX;S8f`5r_OipUp(kIT61rn}#@)RQX!OO47fRrc?;al9Tzf}8xAw;mVcmo_Xj3+y9&!<`
z_RuJE0X7+RcwrZ6Br7VKTEXf5w%%SQU|u|W^@d_xN<)F0xXNqVU%&pvt5@u+qc!U^
zp=3ZKI1AWueKGrepVNH}0t)r}_wQ!`CQ?CD9PWO%=H{(i#dXC{0N>`HJeL2VsV)>!
z13+i%y$K-P~wDjgcIfL16aM2bG
z85V>cD^7ODGp1Y*PVyi)JzLs)@|0GL;!Jk0{Hbog+@cE)u+U%B)OfbO|EPS1rE9&Q
z4*g%n_bF!~+bj-I@y8<3kBmFu{7k#gQgYk2DXXh5LV0lE
zdnIF8$&9|c3*WbN>C&5259d
z);@G-veBL>B>x52DJzKz3He1va=_BKf`UW%HCU;f?`&a)HwY{WDUmBpo}O9^2{#}+
zJ3FK0#R~>B$GlTNmK&rW4u!s9(N!Jc=xT!Zt-?L?vv102@2I%wgmxDb!?#+|OQ_xf
z=ne9{5*jj4`W4hMK*%*RGAlubx*M}ar)OruqoRHr%&cGyIjb_3!*_qT|DU9j1KJPF
zh?brISno=ZGK>zdN~f8g8Z5Cmemn%I9){lH
z@$Ddxb!ha5?#DD!9$Y^s6p4923$=k^$&v~nssPKaPHhzQ+?mSR+gpzNU%9gE5|8Y@
zWRH;R!%t&)Mo$lIxx2|OCMs$%)<}D{`*4R;
zlUc#(CFp)O9}+?{v|w%Uc@3RC@LjW|rNw3N=g*u048av}GSZvH_u?BSB?j)!ypP
zA)zhZH8>bH`K#CRL&DyS%*>s~j|<t*Q;e=r~?M+W-IIb1io9HX3~;%cBS
z$L2AB83xcrY$}?2rd_;SBkSuf^0>2B^X>Fd_SPp6@b}#d0)^d;Cd_t
zc*ol@tz5aXY}d^@(a}Tb<(+5_ggqHNG&&m{%_~6lK+A_lOWYNJo_D~m#Ow2&{k;L+
z&f*OlHXOCKCJWo;B4M56ZW9CZoE|TkBMT7}o?c#Fo-}h*uJRY*Yg+Rjfdzm8
z5Ro371lRCARQCBY))20AEW;qcrwq?=hqY_h#_qnh{w6%m8{%eZP6ZH$wW8%%42%>ClRrH=)&)1%CP$H~(P=18dB?
z!-b0jG!||<)lD7l->IS^?o59L)kxzg78U+f34-?YKYy8`f(u>7%`H+0s$Z3&pZYOj
zwd;c(=ahYN1}fE^C#jI?d$sCp`uE*t^5UUhZ3Op1kH|aueW{j~R$F&>u)wY$ejyYeuLr0LQ%dIr|}5*orbJ-TU!y_cBuUKd#(nH+!3;-)(8snhiiPDZ4?(0
zDg^~6>krj{+;r#=DWX@qH<_0Bta^1zx*TOc(|z~|cyInI?)fNu)XM4}K)XoF)$aaK
zkacWI@SFuG+YBiCa&mI9Zi6ksAt7f=N=n{-LN|i{^!oK{f?44jFI%;03F;zzLHxcM
z6-7eQI(EG$UniDm`;Q-s0jvRVJ3-2?Xsj(`eg%38
z$7dPTEx1h(Z{qY*4It66e*%`|&Ck!rcj5O)cU=*
z+S;C-E1N`DrvWVtx_o(QUw=O<7ngAE41Bh>uU{A94ULSAJ4wy6^u_ijgdZ0K+mseE
zP<-<(#ou37fX$UbD!#MLRoTVmvB_;9T>=51FJ!>mLQM*EZT0%O1RN!BFa+{^CvGZ5
zMQPEit5->xbpjP(_rp5>m7NW}iyOT<5Eeh+`gx){Lf1+3{+s;C_sv>vdk^2|0~+
zYhPL=zmlGwzAGnOV{MmM{l||1H*Ty%(&GAn_=XLO;?+XSWLv#xy}f6odYKp*0hI!8
z-sHTWkU*FfCtSzZF+0*v@ugS);bB5Ebsp|eC6|aI+m&-i#-1DM4lHgKtQ(Y@c&Jp^
zpq1F>=HP<0wh(>vlV|ad?;aV;5))>g^O-&kKnWwpJ9XvC6+fVZj?S(gsqRSR#Ep^y&}FOj2CTqY(aUvC)yA;Fbi9Gru*S^Y>{L`3=M(X}ooPrggm-SA-Y>dT{~1tS}Q
zh3+BxMhx(&b+V{_Fhp1*_Ni7dI=PouVhczrKS_*^qU71+nzMeG4j?on*p_5&3Ys
zrSMGm`v**vIiN(6ajqCiFW@1TJ%7Fs2wx|8^wH#8SEQK}6f7{?XP-ZFQD3ctNt4Dy
zu>>-pD3U?}GKM^(4m}WNkcGLqA8eH}6eVzO^(WS^4mX(rlaRs#$u%DO9OZ{rR$f=9
zZ*y{T^flbUiIgZb%=$;P5Ht0i-&p_92TaIH)LDQL!BCIS%s}b
zL)Aje6E#f3BHmCvZ6>h1fLopxy08P02{dnP6n-N1k0g+>gS~wyU|RjalZ=dI2$#5s
z3%dSgb&$!^{x|K&xv9`8`kNq6N$(5+6WIaBpVECHtaOuSptY&vVdH-;=cwbee
zK9L(0MV8f-8#fa4TB!^)y8KxPhQtwn>e`E_fj(3{0*jj;%h)%zK@ovbvNKZ5^Z?Ky
z3Ya+<5&GvMJXv}Q|8d^Lw`69F2f!RbvBjuc7QUg}(pRUpMt~_#PkXgDc15P+2DUY3
zpP&v$r}a1I|2z>U{qLG%S$#FE;q4xyrSOKRLb
z+!E&Hl_BW(9pe6l3vDu0(tJkf`popXbM&=>6RqekZLHNW%pUf2JrYI{z;_y
z7}QB3BY>IRI5;jPRGd5juwMos2#JL}3&ecD7#Hu@u_HgdZbOWhKS=TsEa8TyuA;yw
z0lITL=B77;S9eX4P!}|}3n;|be|&iTdSAPH%T%}1|E_b!r$l&t6_<79dJK3cBN
z4y%}`{7~=9QJKs4HnOg8MCci$=JSGQL(tWzq6wwAeQJn2;MLx?gKy9*3lWMoPAYW8*tnF
z@W{|mDe^in|Fs7;!W8MQ
zY1;sZeQN{+cFNHq2Pif^aSM6HHQ3WYWI(av<3=F!%XoM$qB}$$bo)JGrQkk9l(@1t
zY0zG;AFnU+bmQdYG>7XT=QSbWGxN*P!W7n6O_pOz;Z`xzr}u|E`|3_SBv+As4Ij^;
zwKx=gkty30Wfh!zWpP-`DZ6SIo8F22!M1VSToh!8OJMu?RwQ?YN-hC0g(MbX?Ss~-
z!p5g5oY;9gD(c`jwu+J;-QCxt=mc~kKimKTR)s|2=EYnKdM=U~Nlk;Oe%-{Zv
z;dOsVWbxy0i~08bJCQo@?t#U{KKZhb3sYeFk+9q3_+S{jr~xt`tjMVb!N!ERBWYF5
zMj#zNBb19$!~ilJzHGB=$e^K|hT3+T0J7r~?ZW?t(JD;M?*qq>&EqV^0YU=Ab6crW8M?s09N5cZTUM#B)WNS$pU~~RvZW+7aupP6UZ(?>-
zLH8h1A4!cO!D&=*dK30A(S@LPNo5nPL*h|POy{m$`@rKXwm@7=lH}k}W6tka{EfOm
zPh=s25?p&{$F*LOzVi986?uE|mk6&2u3xVvCj-Ng6p^|V{rQO3B`#u4l{Yox$ywUm%!~g%`;X)!K~)+W
z8l^++{n@-#@LIh?Mckr5(s<9h{3yAe!_8pbF0f*0;X@-|3tM&)=fmUIE(07+`%W-AoC7U$JF*A
z|AcIm67bB>=VLJz6&Khc%&)lIk2h2yJUz`+9Yt^ah1p*tDe)ym}aWfWfCrMgCHCKkJMPg4Dh@pOUHab2q<=W2e{XFOmL1THo_-jReO5(2Dq!+!&4EnB%V2q@;oBhmc_4lIL87%&dcfH;9y
zuU!jA77HaS5kg^iAm4k5#xpg@3FZ=W37AEXgi
zZ%$+z1EueBX9pV
zs0x?hE_9#)+}!4xqBE20*t#0CC&5Y-+uBs1x)k}&Zr;Huv`hC&CaLC(h#7kg;UN@oW{GegPlC1(A(%hV%P*D(J
z8VZpoMqpOrgJ8wX8=AknIP(2+0X&FQ17t*05aM#8OjQ5MG=H{$gr9t;hZm8!2s9wL
zF+-24{DQ0Jldx%rix{C$LCeVR0z4+!WRhU9NKQ&(AY}*XdI&`K_00Y`p6!-TC_gbR
zV5O4IgGHte_%!^*t+i<8*^DpotFXjjd9{J}V{us^jS9I()^RnGE(-ubNVEb7pIicR4m?MqKv-EHJ$LS$B+N=u#gUZ(*78l?
zArjo}3qjoW3U>DzlosqUH4BS-8UOEbaaRA{8owK05YbP)eqf+L$jy_FkFUKIr4|>@
zY8`)_h|lw!+yb*Hcw!x(QSg}0!h%r?TTN11i1&BGg|}%fF3Jdtd~zRh2AQ-0LPJrg
zxwP|)KqKaCz%a`^K_c_=^LIiQarg|xd0jCfvSo|RoZ>h5ag>sai_}WQymK8sCTWup
z8%VvufDf`g`K`#I0~+c%oPG+dPBLa59^0@-&m+w;AMnZcba#g>2T>Hhssuf5(8?ZA
zyC&l>rtpU0^~-b`t1Y=0o3wg5Oz0x6U|Aa
zqJm)($EBvIsEC!5GZ><*#qW_WlAuSJkq#>rB#Iw+{>O(}AVs~&k9JmQkYti3a2!%Q
zb=YJBO2qE&?(SGH3liAJqJ$0~O13;anKBtHCz6XHbRC#t7p!Qo0;|f{Lfx_S76!p^
zjcQYaLx&hCWr^!AkCsi!-udKx$JcDv#7Jcf1Pv%q6lk01huKz(Y45_xncv$SxzCXR
zfSk3}xVp+5$#nc*qCr9@y(1oz_!-dv-DCdbyNj?V7E=Eu(k}NP3MqA9{Rfkn+wqTx
zh-j8VQSh4T&n4lpvSWxbV#S{l?ns@Cr!c0mAndo3|8{%W_B;HTF}uU2otVy#rW7?uhud{A8f7qTY?Er2wd0AT3}T0Z{9-6QaCfR
z5DLjNd_(IAJBSAXM1O&;kBeaScXoBRUZ<_!A&m~NhoBaLnPz5Yn6|q|l+(Zo0)77b
z{QVD}T|rX+ZM4YmaiSnxG*Ub9gam+s5z)sOCOwtq@P!+4BM7?Rg9qzCtrJcyfvG_d
zDU_*qN!n{Dab*M(lAC||QW>~Z1=y4jeyhl=g+*C_)RSB6WOiphPgYn+tVNF3v;BK5
zbyuUc@+!D5LyjJ`ded9rxrZkCk83YA`O`5Gj}{j}iam*!ESvcC>le|x$!*wlD63SR
z&pUvq(_fytmW^T0lQ}F%8h(I5^K)Y^UDcjpns{s^#-yUI-ibPkxKWhE=G`iDYrI+MBaeR2YjD|TG);kAT13+y@k5E
zA95Qb!@^kR^9XHvUy9x|2%)qPe|miB?>cr6mP;#6jtSX)kW`4(nOxa4I5;xICc&Eg
z2qJSiS~w6q$p%A$fNoeizAc;Rp%8nfe}9hw2v?>v&{Wa$#w>VvWA*FTl~{1$2z#OB
znS*0lt=5>%AtVa>l#r+itY(oU9l^MsKr&nK3FKQ617`JUgP6K#856mo=sn3VE<~{{-zM$1))vYL3DsaS2ao#Rqen9aeAm=e
zIs~r0Jb(dO1bNYUIz!*+uGMj4#cJ{;RDWg#ZDpE(KU}%t+H?x~UC&sf@(i3Frhv0B%%)
z{1<-hT721s`DtOb&?SB(^rND(aQX7(P8d}n(SI^Rh2FK8A~6SOHzcx7V&s@mhx8e9
zA$2`cPbprvQpDQyA%!9ZSME7j)!J${FoS8744_1WSw_A6<-;_Jz7s!(Ac~
zalrgU5I7NY?HVm&@QpK&4g|<}<`+m~-YnN}QvF(2!<37_r2Fz7vX?3`m?41;zY}FL1du(BJeRTM_${EsCptF%ldOVa2#QZUyx?KG-??0)K|2BV(
z^-$+-%RzK2WFge}kt%aV+95YIB7$AUv1Ph)2Hx*}Iej~f03cyN!Z1+55nnLLUyd3=
zh9mxQc!A!;Fvh}IlXa1>H
zFYsXmUL=Y9RFbuJ0c8WCkBN==+7ZmI19zmxlAnsZrP{xs4ZT7FVa8}V=LJ~rL>z#S
zMNDO{>EWsru`u!)jk*N|e&yxmaB%kmVRPNOC0;GMX;TGsKldf%;16xO0|ivlqer1I
zs>wJiG&hWSpF@`gwo31LPdpP7e23bI+b-$jktb5l4c8OY1{$vT0s4m<5kMHi^M?*3eB$o|$>TRC0B^0ig
zk6LpbV=jmQR|uwX;r_x*iF0f%CgD@OdZkVQl?TgSKk`855UV&I-l(Ik+Y)|?j3OfF
z$wUqVAW(?95)lyyRL5?pMxO9Y@)zbdNYHPyPZ$CYl4Xz8a~9JLWX2b{davOMjuf$Q
z@*anD3%E)_Im;lp!=U%Sd-rZeAwB~96MYLSyqb#ixyD!2ER+E`{dtCtWw1nwmN>
z)~=(iKmYIN@$Y1JVOXSg1WdaWsZu(2+)>uLjkp*8B!R(f)Wx31`?LxUJ821;*Dtt?B5l{tnJD*|2eOsd;(mQIL}~Br6esU
z59ZcxJXd%GT`eFNB<%x4M@IAK&z5=Tse?5McCmIGz9EL>wkb{z?P^(w8ATO68a%EW
zRRG4gPAlt0q(FD~>tCc83iBk_ii&Ffu^jY0@xa>l@0eLTei+gLRuC}^AwI;%rF^WP
zr-;o3n@ufvQlHj{#PO1KK#vz5`|&}2ph!jVEC3~8@wlUPY-n97dgqc%DL@T<*TwVK
zbSOK2KED<$5`?D!H}&9;90cZ)p6|iu3ASC2C5^BaVjL>Y&XSO0DGKB#Nk4TZ0wrQu
z*EH6qi2QvSP~DjR!YDpbLI7XshdSEd5?>AC2eF~p4M}`XuPE7+0=s#Rlvv=cf%R9A-9==vq;>>os1w1df2tXn-8WfY(ien_`R-2UzP1F9<~e6)9xhpQre
z5la-+_y}&u{C2?!EF`$OO8&IopPTW{g2Op>T)@?r!2e^>B1zU77$yWNZwDyoKk6x0
zDQXx$Cc{a598u&@=!$XhjL32TEz&n#O+HR`qQPcAXxt=wT|CjjL?W?Cm1j`^z6H-n2!4b-rSstRo{$T!Fgzznh!;-UeNXIC>
z80KS)v;HtRTsE5U*1CFcgU-HQlD$N-g#9)<`4{5V0K*2Fzhq`p*93%a^~nVehQ~1o?=lJ#o}d<=n-*k
z2S{+~Kk4EE8=>ZlA+oXf?1A+t;i6AY;b-IaXj{_8YOOEFkAqkG@b2*8UBbQPit!)@
zByhF~G!DiZXl^7M!p9fmGM*eCPYb$`&p{T8tFGmKnlP{hbgbv_yctmX+Mg=I&qCPz
z8G>IJG<*(5OOB4g4_)Hhu^KX2A?;04E3&wG2zFzj=H0Uq>}^3nI4Z-XYmbeM4H3#+
zSp4Uy!%MIgtTV`@h#wEN|J2csQF_+NU}e|eMt@4wuu&!H5N{^Lgpd(N4JVJ(Z>^s?-O8-DL%
zdSHu-7p!Z?2*ZDVU>YB7eC57NW(j%P=l6*38q$i7FO#Ws8)eVgYB;H6!NXT!D)8&>
zgEATVclGQ!ze`(pIv>N6lx8^gpT@`uzn)CzQD%bzEF+_xJd8
zQM;dKOLnpA0mqEa%F+JIp=q>jz;=un4Z!d_05B%KXkB`(q`1go45%$iu;9Q?^IWdVyut6xVK&`4VEy0k}!-G%g&Af)`;4Trxx)f#Lv$k$5XEDRDVrZV8GlkhX%cmnV)^|2n_B~C=&@)*_n73vkL4mrRq
zqZXSK;_h>DfLQ@C+5bUn_u?!huWJb3Q+|j+HM_tDurNJyW8*>8pp|;I_H(D>+d@!;
ze(GFrxHdJ#p?mh><``kSlfBl_Wism;@37}&Zk;T`(|xTIXiql$a_!etaVqj6D71{}
zkd;&a;03wst0|#~;+w-@BjLt(uU%X7P&5n;1rlP{-x#(KtvR9zs9~%;Jfa(Mcv3mS
zL`y;T*$w8=`fpMOFkB33oH8_v>m%oWAf*UV2vXn$Naf5hIEZY7!6{lE9v+BS>aMCQ
za3l54q7IQFK}`BFQ*hyxL0}U0$6Q9YC#h%m=?O`KTX_||I1vp`5h2r8Nqasv
zPz^Z!;>FR&^=XkBYmG2y3NOScP#>?$ntWFXqae8L%by^}sUdBKx>6B(ePU^t_zUfZ
zguvy$ggXDM4m*qR6A@wmGG;NaN~s9`ed^+0zVu>d)@I*RbSybzuX^Ane4rMVzq)gD
zG~aAV8L$xY;5u`GM>=I5ai%c-#aFp$`})Vjf)vdIC(h8lNqv|ONmB2H&ueuvY&%-t
zMS=Dh(=uf3!a!sm<7XD!7?!`3uo#k*P}$?|G3ps;|M>2XM^}pNUW>kaaUe?lHD{5d
zFm>HCT5GSjo|fwNtLK9?zF;yOcj7-CMF~4Api>s=zX7eq#^7zMZkVI~3kRLSg9a~r
zZW(<^zcQoOwMPH>bT_vopFFGZUwb@%e_G!uozO9APVb;u;2TX*YxOu$m`XC_P31mcM8mzscv_3oJkl2ag(==)sYs~iFk8V
zFG$Ml!{v4w{R4}`-T(TvZkA=dD`aOmTlJIihiLDhCeWPwz{8BtQ+A6griOXr^I~l(
z_6{g`B;>ubtgzQHs1mi))tn0|li5FbP{i(oN{D)elHu@^6EbV<4>YK4?!R6pb9d~E
z=>2rbFcY4gS_r>KfHhr}+({1=R
z@H>^UiB0rm-_r|x+v6fmn^^EzZr@blz1^@QT)|+H@wUxcA)l{WkDfDKn9@;h{soq0
zqd2$M;`WzCj-qxSKL50=pxa*jAme5JD_!=SVP^@$hRXJWLA0xTsS}3rpP$@eY%&k-
zsQRkMX0^IC?$U(ay5cHEJ9e3aQxYafxK^{NLaj_>bf2$uqHz7BaIW}IFi4?5dBh^c|>?1
z=RJTP=$Ufkg|r{A_M4U>{DY95l2OB)V#c9eyJ*p0a8#E`eiM>ZP%jR+$K0OkSelN3
z?mfxFF>%Tjp}{S?b|-g~&|JJ5X`o?7M|WY$!(c7$BcIT$@37vk;V%VqY@!32y74N`
zX6r5tF7O!%Q*qv|O2^pr@x72igBn*L-EaI%LSFhY)oR{?j)I&R5$C>jb3(IEzi-zs
z&H%<&>qzq&YaV|KjHcCu}?{HbGlZ-
z#MHfnt1nbMMj;&|qgp{GhdMjgY=r8w4We*htM6PyqlUTUMI7e{4H$g)^G#~?|bN);TYg^kx2iGkiC(Qk(
zO<;72PNlEPxBc56dY|NO-@K_ep_sbjUCY8aV17i~=K(c_m`NjZ+e3hGQUYr~Ge4zu
zSAh$HBS|4F;`8!@=G_k1CB@m!dh(q_)H@^#fDR^M&*N$C*wDYIGo!nURxY;CdXN
z=NV=vtD{{KE>t+Dar1spdG_ZYCUHef^hPn0JpPA~FOMP-DF#h~cKi12NIZ66RzT!o-XzjD&tPSf
zQ7v*d5F`j3U=lgiXLR{z!EYpa+99)`izA7}E9a!-P$bnkiL+}EfhDtCm|i66$H-51
z-%9Bj7TVZbSVt^4*6`vRyyYz#g+-jAXV3bFU;+i;)@#F>}wr
zi34SDewQ+)G|AZwFg%G84k1|AqOQ&u2d@$75boiNP2c58O5iWv1A2q5Q3i3+>BswZ
zI0uoOx~6;^*;NZfH7Afj9fJQ%&Kp7MUPwsjfJIkb4bI9S5o2<09mXbbilQ*?dJse*
zMtix}gWs?S$mhk(nAw1bRi!kJ+#{2#!;GZy$pdWG2e*2jb
zm2Y3au7$09^C8j?kI6L3QK`!(Rih*=gZdD`_3EF*){4>-iP&UTY+^Kq@1y!=yZo--{DeRJuAE8vg&>=NCo
zjK~Rg<3%Va^Jx~6^ME=*gSxO19~N{TM<@njPW5di!ryQXCNNKwTKKJJ&&QaBaSkN%
zfWsZH|6aQs2-&3A>!7NB-A2gc7~W9Dd*T?$6_nCHxk#V>aFIlk9#1yHwIXF7=9^S6
zK6NLcKZypCwK_LdGPethY`xY^%HG}{UeRL9gDr1ZFFzTvejObGMR1GlM}}-0i4;OM$Z((RdTq2=qr2_}9Tu4pbii0}kAxO}I?5kh`fsFfLfEEnL
zDLPrLP1s!eh=s*j7_AZcqEeV6)JfLEo#4fHmLn43-0^%dIiT7|EMku~Y~89bzMAas
zc_nwCpb$R+UjBL5T;v2atWm;z$WTO1&Nn$jKQJbwz4kgpL5{&t7>yFA<}B}-9T=W3
zhgLU?(^kmz5(b}1{x<^)p5pyp9ao#T=fQg^Sv5rn{TM0z+E7AAuO$8NWteH6Mi>;~
z+-jUIu#_Ae1<~XkN>Bc;Kb)?sJmWZc6)u2FbE`6>O%kNU;AeZ+A14a6xAJ};uSe^a
z3X{Z)i4rOV4FwU!4?BeCJ08?D;+MSNb8+17W*eIqa~B>$J4Oqjaf%kbDAn16lcjMt
z)?hBghNPEpgbkTi)<1kQ`$12^0gV@l1o)|YRmEx+(
z7bIvT0f%!NXW4}Jq4qw8O8NqttQE}FM{gm%D5JxPOyLe&A#Hg6AqIsX<9L)89OnL$pfwqZAO;d^S!H7!?Cf+%hA=kqv1+R##4VW3#>>l$
zGtTfEb%I0?k=I5JCO<{fQxt#!XEiKqfu76=af$`bT6D~g!B8NYtBsA`Olu($+i2RUBR(O)Edju{_j7Q+z{SZ~4`fI+5EYt)
zQvo!Opp^r-;V3;CasoHZT*js%h5kt#mh=pHO{`*ah971-u|Cd0MkvK`cz|Lgovy9@
z?!*aqUEokUay&0$ikq@ZU_ihq$*`(fnSbc;tS978vE?}c76$zoV#Ya==!JI)LBLvUDiDfkKn&K}M!k5dPkmn_jXF}VZ?K#WE{?@?Cx_-%cC8|!mm
zk^Jg+xFo}j>AJ`CAjT0Y|
zDHk*EU+hGD!QmY^`7U&bF?#nFBR?1vwfPu1pn#e#BHTr`05Z>;OsU5gwq$
zp%J?>DuD~CmKWRx+5*}U0NODJJjxz)7zuFg;r`%w
z;0(?&;7KXKSz)$BwnE^{*biV!7qkPj===5k_4S}l^1!;K1=!{Umiek{J_FD51-82%
zJb%tEEj@d?w20E8?FtitRdA@dqf3Y=aJww%6jxB={_4^x+V=MHz@==UYyqCqEb3Un
z_E<-S#H87+=7Q3kji4DNr0dA?-OEfJ(^cG8+g?VgQu&X%Q~loCIC2y
BF|q&v
From 9e01944bc64cab1cf57c264cf7563a62787758d4 Mon Sep 17 00:00:00 2001
From: Zach Nussbaum
Date: Thu, 13 Apr 2023 18:41:39 +0000
Subject: [PATCH 52/63] feat: wip training log
---
TRAINING_LOG.md | 43 ++++++++++++++++++++++++++++++++++++
figs/clustering_overfit.png | Bin 0 -> 2414770 bytes
figs/overfit-gpt-j.png | Bin 0 -> 364829 bytes
3 files changed, 43 insertions(+)
create mode 100644 figs/clustering_overfit.png
create mode 100644 figs/overfit-gpt-j.png
diff --git a/TRAINING_LOG.md b/TRAINING_LOG.md
index 50469645..e06cb65b 100644
--- a/TRAINING_LOG.md
+++ b/TRAINING_LOG.md
@@ -235,3 +235,46 @@ Taking inspiration from [the Alpaca Repo](https://github.com/tatsu-lab/stanford_
Comparing our model LoRa to the [Alpaca LoRa](https://huggingface.co/tloen/alpaca-lora-7b), our model has lower perplexity. Qualitatively, training on 3 epochs performed the best on perplexity as well as qualitative examples.
We tried training a full model using the parameters above, but found that during the second epoch the model diverged and samples generated post training were worse than the first epoch.
+
+
+## GPT-J Training
+
+### Model Training Divergence
+
+We trained multiple [GPT-J models](https://huggingface.co/EleutherAI/gpt-j-6b) with varying success. We found that training the full model lead to diverged post epoch 1. . We release the checkpoint after epoch 1.
+
+
+Using Atlas, we extracted the embeddings and calculated the per sequence level loss. We then uploaded [this to Atlas](https://atlas.nomic.ai/map/gpt4all-j-post-epoch-1-embeddings) and noticed that the higher loss items seem to cluster. On further inspection, the highest density clusters seemded to be of prompt/response pairs that asked for creative-like generations such as `Generate a story about ...` 
+
+
+
+### GPT4All-J Hyperparameters
+
+We varied learning rate, learning rate schedule, and weight decay following suggestions from the [original GPT-J codebase](https://github.com/kingoflolz/mesh-transformer-jax/blob/master/howto_finetune.md) but found no real performance difference (qualitatively or quantitatively) when varying these parameters.
+
+
+
+The final model was trained using the following hyperparameters with a linear warmup followed by constant learning rate:
+
+| Hyperparameter | Value |
+|----------------|-------|
+| Per Device BS | 32 |
+| Global BS | 256 |
+| Learning rate | 2e-5 |
+| Epochs | 2 |
+| Max length | 1024 |
+| Weight decay | 0 |
+| Warmup Steps | 500 |
+
+
+The LoRA model was trained using using the following hyperparameters with a linear warmup followed by constant learning rate:
+
+| Hyperparameter | Value |
+|----------------|-------|
+| Per Device BS | 4 |
+| Global BS | 32 |
+| Learning rate | 2e-5 |
+| Epochs | 2 |
+| Max length | 1024 |
+| Weight decay | 0 |
+| Warmup Steps | 500 |
diff --git a/figs/clustering_overfit.png b/figs/clustering_overfit.png
new file mode 100644
index 0000000000000000000000000000000000000000..30079f565be384335160fc547bf8ec3b1ba52054
GIT binary patch
literal 2414770
zcmb@u1yod9A2&>gNQsDqfGD6KAl;>eh=g<_-635wl%&!n9TL*g4GPlT9ZGjIFvGxi
z##`v)UGMv?`??m);mq0l?EUZfZG+@xC2_BjT|+`b!j*a^rT~0Ek&rOuu`qx;LTcm!
zNJ!TO&7MA$mwNh?THema*v!%h3F%o-j4Gy@V&|n!6^~-+SVi^{dz#eNzqO5*8y{7fB#q64g$PN)ni+?pP5-~#@V<(bGA<oi|)V=Rfbx|EVVu`NIzb`!6j`b+#b`h
z*rh)kXWU*EeO}CtPC+Z2trATdhsBDnfA~c2$^Eb91pG@T?f#6Qwwp$!_X0fb58YP3
zSRrV~5=-~K1}=E4edET5BbMG?ssxrwPQ{KOrSHDz@vrrQp|7k2iUd4)ND36G^;_}x
zu@0hW{8ySu(9juILIsHezZ=8|?6#tNvj&&&h$j&h9<|G$^QN=aJ^Wg9=2y~H;_?Jp
zEBqQ*KwFF8PEH3AOV{_%5S1n0VkAN@TpuK&XX)AA0^#Uy?tpMXd$NN>DAgh;Y+gtc
zj7(4O-B=c+aS1^eU%(GUqxN*sN~>2zx|R;_{-pf60(1Rf?#Wr8jP4IPbx~s<_y}(@uLdqLOgtgdJ*;pjcz#VoJYyRhVyEHOEx;Qg8>}tPx
zSF;!f^zfjLW1dtqzH4cQ>lt#J8ZjQmD*K(GLgtP4W{2gW5qCNtxT5iTS;7~eacXi_
z>D^KZs_9T
zC>UE@ly1s;;8A)vY>#T+Kc%1`@+_1yv>`N=7=-S#UTVYfYBk8@*&EV$^cT0z?_s^C
zAnez>IXpQyiHWd57fx`@KP??&;P^gwfh{X$-%eMTi4he1U3A2o^B3Ct%)CzO@%ay
zU_SQB)xX|@R@N@S%5}>x>(%;uv_#(+{be;`mA9NYlF&wHxiajy}xxP5w$@f5Y5>hgIS;dLx0G#iwFaIp^R7x$tGy0Gi8B(ZY7)r~xL
zk-mR}RE#kL(}N@`Nd0U4*Q~EHU-|TLO5UCZ@`%tx96fWoud8q~NSc8b{nuCv=h#XG^3@%S(5b=niP@
z@Xv_Eqx8l1v%bD8eQ@}2^I=8waCBWXcV1pzcAie&qOxD!q0)FhrMzGcROA>{`K`IM
zks`f9)`-!_?8vPRrVVnk5MwEvw5)+^rroBa<%;F7a)`MDCnjeonXXllhMz{mlu8
zhM^;lV=UD0nC95-_~}O62#lkHESAg?x2TOG0w#`&_2149RU*DU$M8p9iX!3lF8WJW57R!mLI;>>o(cLsLG2Zvsd
zN)5ypDW==M$Vv2OeMCwjfoH(GNWI9nxQNw4SU~7QC@nD(S}pKBdFRO+rvFqPPzimWX$jWCuV)26eg#8v-F{_Z?c~WkRA*=oS&3LOG
zFcE6hW1`$nt|+`Hxonl}{cM~Un=f|bU-k&}4)u<`uShsfBu@-uPc_mp08jf^elv?O
zE9v`Q(8@2(*e%vA9*Y~>U*J&4qav=tsp4E{Hg++lSeRc>JVjt7U{z|xHR&*wQE6Qn
zUrAgUwS6B4%%}bpRVjuD(WF8kJ^Lpl+OS@FJS-~($4qeW8UuB=b>b2E;W{x<5xEkgV=3=dFE%c&9ZJXM`
z*|r)cI}1nH_nmLduZtabj~_#x9e+Hk{+7E6}6LVXWTd~xS9&mi*_;^o^kw?hIU=L0(9R4mmA^bwVY9KzHN$<_4
zy-zWE+&umilsO5*!ea(oTP0TCM)oH5LcZbOv%GhoP-b!9wLyC!r8D;sjl#`I0`v@?
zm-!!d#5Kj`LM|KyUV9v!2Jg%1C+QovCWU0R%LdJo!?>6qTf5FAyKK6cd06Y?84#)O&_E1kPP8(2kzc73mc9=g>
zUhik~U{^6YGKQMOnw(%yHitB;s_Ut1md+VT8twK`_DJ>Qb}M(v^a?Tz-7tDPb@1*W
zd(|;$`z!7Do!mLaaoO#-H_}ea_s*gY$Z^WQNi=GkGr^^xX#>TdGWOu*Z_EO+Eo4b#
zY*O|34LaZDh~E|GH~G%)%C2b=-;*G``NeW<%gR2S*UVmHHKCU&WtyGEVJ_t?`fT#h
zQ*)vE)u>O?EjngP2Chdbs^@Dt1F7cEN1orO<-T?HXsYzK?jx<4lCMUc{GsCrw>KeChZSTwxmE{ypeW-OS
z{xY}BX&O?V?L7Y#lnnDd;U1eBx6U^hiMfM>nTeTl842w^_Wg#biSJ68CusS)`Tz_5
z%uG;0_(NmU-LmvL)|!&dwTTafGD0$vV8>HO=Q1h#R@P*L?l8qr5-KWzmk#=A;xj|e
zY>P`D#vT?r%hD#-PH{CBE;^j9+nYS;OXBP1bN`;STeEUFw;HCiS3h7o2-gSIX~H7r
zXTbbJ-{xL=b}lLyV@U5+{Ugett;oL+d6pz
zcip8OJl~PpZQ){irYdzVrg7Z8Y1MZ%pp1%0SP0Is&AxsJW#Q6IX!Q4N;D680^I+w9
z^+6#L2CkK}655~7
z$O4~;f8oFnV$2_(sFD6in805*fFGAMl)s+Fg{GnYb&V+xe1{~e_*6;?_*68oGcvNa
zH??s{j^Hu}ZeZI!Q@2M#x)g|Ge_bP_=)C
zvU76tTpfDlt6zsI+Z)+EwXp(T>LBzF!~Pt6^~*m83a}vdeuWl)5c`i<
zg6J(O>&{mXN8?vQ<1Ka?*iBY?Kp?6x_thTt#`#^>lp3wY`qjnfGBPsuHCj)eA|ay@
zi2S!-Fmx@m9plSS?*frfu&BNM+b@P!KH!}^@C_1@0IdH$8`i`lU_OnJ{$;yK0gjft
z|8a4^?!2&}U;g_nNGK76X&JhPGo-}-o&NVk2Es^C{yVVTNw|4c1!EUan;1%;vS-@ns5=6Fi;-+AekyA~H4OUE3C
zgZrOV360>B`e!t_C1upN|EiV1S{A5~eZciM|NBgVX-J|?Sl`(>{<~BZjf{+N#C7|j
z{r9P`#6-M;!Gt0v&;C2zA0ZPRNlQx;V({QC9lh!mW~{|9SZL<82g$jnRs?*DRa1TrXZ
zZ!tys;gEC%6EiMB<~`%I>zq0x?n&MUSecOBsrA3q_kK?_&}?>gxElAp^{sT)^O#Jh
z?Wv*a*9K-IA7Az)TR6#@_Qom=7UR|p8BRBHOnUOjgeJcra^9}_Xw4k=Y)&lhCi6n1
zg1DAKdbM<3ToAkNAkXW#x&+5g^?xoHsBiPBd`flJo
z!rw$4Pnc~l&Q`lC#&sCE=UhHSv*-#f!THl$l0CCJ3sMygFNzMU6^Qv?`G#=#UbKaV`3%n|-B>=xWh=_g$dtk!t3(tbW
zWAUrjBJd?L9HO@~QT3|332HA}>>W=N~ieX$HNPl{l-<9tdQe){*KYS=D@>u!&q~2Lc^@!i?=%WW@*16PSUBP}zLpj5|f=nUH-0`7G!JDKg
zmel+A@AG7n=>-z$oDo&|N6+Li7QS@kNXkOH`4MNWbn9D2N5+RYs0
zvEvyx_?u<^!L~l;$EE{mUdrd!Za$!9(yWZ8lSz2CJynsVRb#gTS8Y)B_y3Jb5eRvY
zj41KC?4<#W6;~CdiX3QTfcN?wr#M_;pzCLU-D(fx^e;&gUE#P|Kf5#8`m(}*&}Oif)|%|}kX@C(pOhd>q(4Q_2#D!ZR>z)Hn2#=1PFqJYYu7&0
z(n_jGko?U!Z>(#kr#8)Run$hAZMvr_Ec)L|C30CUB|FdY8Z<dqP(c7G>n3KQ&hUOYF6`9g}7HP*9+P<
ze%>r@Vs7!rgSXhU#5A7l
z@F$v~5$xTVs57bZ5Hj@=Qzy?|
z(Qy&AuKwolu$gpV*cW`$b$e4$Hr`rn#dTG=X6ayRzcR$i2b`Om3uIg-(`gqN34D%u
zkF;x*7?cVgsXDLS4+;vpa5A{_i#$YQCkePJTyPsjTAms-j1=qYPE}fw8?hVm{R`Q>
z8c^I%wj#1+61&o$1_+>9&^Ad$(GeaT27daDtLKpxX#pqmIX+jNWC2XwN`ESg_}g2K
zBb^ii40e$BLoz9e~Y1-nGE++KDN%m4~XiYvesC&88L`?X@Bi%-=#`6Q|
zhgwSlHR~&sN(YAqUm1SVA?gJ5^3wTJ{PwO3kb4#gcA`*aJ!_i6Z2xaraWey8e5gse
zt+i@dv*l8mD=fxkt3v62x8=SYY9;!9<<_%&vzc{{n{@Xi-aV^go=SZe|N6P{Be|dx
zGl|x7jnR-eokF`wHWH$vGNVi+UkX7tB^}VQGB5bN`eY(dU!`iJx^W(Ac}d
z-Kl%Rc`h6pYjn1dL;F4
zOTXy?KOQ%l_@uW^ciDn3{Os&tdARWH7(M^x^QgKx0m8!Jw_UCwJIR{QZ(j3^Xb4n2
zO8ir3n43x~M(V2dlyt7s`TZ=<1WBJuQ*U)AsSjs?;m{$#dMx1Sv;4%5e|AajkLHh0
z#r#1s>?G+WUT6me&P9>PYw$Lzx(MFCx?RqhB93I2e~U~q?k$VO?E!X^~6L*My}Y^
zh&63lmzng(?lxWO6q%2Unvc|X?0h4||JkZiUq{JTh12MIa*AKT<}bU5RkaBH3D<7?
z{M6rs4wUrmrOR0}Y
zz*zhxT6rVCaJp}B(Dk9{Tv
z_uJq4-Q)U_X*TnXY-U6E`>}B;c;t6#Rw?@rO|?$90u0YwbnG}}nKo8OD%5?n#_!)P
zIDDrzU~{7}nV%^cMyXI~sqT(x!trmS
z`~*eN{Uq9KINMdi6NqlJEXIreMU=mWewL&H-ABc^zLzBbryJiF2y62*p-YVv-&DwW
zZT?a?M?(4g@|Vz>Wr@s%N2(wJx?W}XPJ$?J?5Ga3^p?}`)&tpA>3w_
zh+Z}tu%pk5h2WVBHLFnT72cl}%*z)YkCKPlV9rGyaPAD+wGOZM;vYNy8@XMOL2^ex
z+?;|6hHZ0qrV2N$^8Pk!F1fciuInikLdGdT9~~WSSgMdh{Ic4%`y;};ls`yPKayKd
znqfX$n48f{{)i3rb9{r8+;lQ!QEJjpS>yb<1wbPaykb%LB^~5>NzKcvE+GhszR9Ga
zv{Qd5OP_*b|BD6xJA`3llCO0zlahf$hz6^GfC?61W8yYM@jJ5-PKykpdNz+=3a~uM
za<+EVjVvZ?JVbzb$L)3bBP`R9hJK8d&gO_OR5wb}<~t(pEuZc$vb|a`s81(7Kh~G~
ziC~`zQ8aijY9d+yg%n`3T@aD%iI@6CDgU;%lf*oi=ZC|NC4h@`=y&z`%jEt20FWYU
zqR6@ne;gieH`qj(@ZgG~3?59ICz2ERChL{miHeU~86W*?No$M1g<*oU1?JR@Ki(a4
ziTi6N>;f%6ryyL|rUP=*;dh@}l11qKGtVDaww0fOfOu)5#9-zA!AdVfGS$qziD
zJ+SbzoJN34ABB8PaiF7*$XE9@DtJfBC<<{Z(cm1LCeF&jKHx!Uq=C*)Gl-a?VL|=c
zB6NNsh+Qa9_)D@c~Z3=pBAzy7$~RI-N0$szUHrtmH(U{l?D=`AhW00)EIO
z9;dPQU$_3z1|HSVX1kc;G;0vC@vqkzPm{MdEEZTan!$Sb(5&q2>>912viLV1{Gv_a
zQA2lsdI5nl3K~3P!|pNZPpkP`7e0x8Hsf}h6lo^--@S^wKx~z|&KFGTcdladTm~}r
z@g+0!to<(i;Ej`eYpWn-wO{Oz?K>n<;dU(sy5Rra1!6yUZy-o}m(QwJ^zSA7t%P<3
zUc+7DK4u_srNAMD34
z^#6N;AHqiSKxBZ$=k$8W?^i$w$W@3op{Tz6{}Kd<5|d8A;o722%sXi*HfnN=r??lV
zh6Aeey7G^Q^qM|l!hv{YSc_S8;92@q#QzU@`)e(4J9r5LbjmKe+6x|LT7i
zMlFHhvb@t%XOlY1M$x7%mBX^O!@>=c;U6VGE#}P`D_6kM5a&
z{9_GDU+P5STDh@Ey7uc~q3dZ>#Bh{`dbOX5+JpDQC&N?wlGAjTil{}KIwslK{&zma
z#>6BL@+>Y9^zt|IJn_SXr(d5t(`mI;ojt^Zu1|auAepx>OM&%$(V)tcWPtiTj|5-!
z5!3}J>ey%NZvgLcTxvMME82A3b>d(8z^o4z%E{~W_iore|6g)=75><5(zj|`OnAy#
z!UN8(R@WjMp8i($!2TzIsRl|EJ3e5u8zhk3AZX9sM{)sxhlhd|(!c1fr^;U44z&|n
zDcOM{GA?wv+a5a4|K?%~BtvjiHlFw|Wku?PA8a_#@A+RNM_^ltuyWT-c$g8=5U$PB
zPdku5l>5F#nD;ryS;9cf-p1@i60z=RZiYg`{aQ3Dr
z5u(jU?IC=vT4E_>C(?*p>$b2!(hl()PRVo#YRfW*e|r?`xaSoz=AvCg;Be
z^)K6fMZYc>@G7jc2h6P4$HFf8##Gk_eAg$F{EAlTt1VdG6&5@;V{anFg_#Y9>n6rB
z^^_imq@~j8UEUKFhV_#EOPiI35y3#3{MNNA+x%1KfZPzFrPaP*IR}|&vgPjrBu*WY
z^hTpAO8tig%nmj8MfvKQ3QjtG4CB`>O6w+X9NDWLM^#sQ5^nQ!s$GY!%XGAdQN?Lg
zm`klPb5DJ_J;ZwC7@m~$2q-|^-Hz+KB6S_XB9HG1Z*$-B0lO7VQMx(=x?VrgR0+eW
zyxiK69AK&Qq4-HX{#mW;4?81!Xw>-6y}zzR$^nD$!+oabZNz=}0Yd!~8?Yx;gLgCyyF0wh3oRj0PJwwFltEVPfF%ExyMkzX+#lOl9c?L#4n_
zvmiu8k~vRNbz`iM-f>g=Q7^L<f33qYfU3<{Pog*Bst+>&}daLG!7XvIa%}4)2kwH2HPcwSM&VDwYSYj#*D~UF-
zR@0SII?j0CfI4mHmf`I?$N>n~lLjcMd2C2jI(IgUNbGDLxbdzq$m|VF0HxvF63F>Y
z_iT&8+S29+KS&yV20=Pti*v_GZvE44-QC5FJFb+bdglR)nd4T1JuB+-JyCQr%u-dG
z)sVS}3;0fC6un%w<}N&QcJ$6tatYP5i}SMv)x_mYkDB|mIHFCNKIlg6MNNHcaxKaD
zc-5+N9tIBnaGlu;GSb5_wL!4yUknB8-NfC6JFjqBU1YC8*UvDBP7XP*t-P04Gp@w<
z0ml;?cwW(qEAnMq`@;>^avw_mY!I*tKLZsedZ4MpDcWSy1m`MG9wtX&W%wfw=nx#_
zd}$v2_E2koOXc_wcN|notxi|lL^J5IdSipPdd8c@3YoFf&rKx135d>_bWfs=)|{dW
zbT1P*#Sd>=4iw83ryP8N8T9I!PS1X+{rYuyT@iN`myAb5q6r$g&LSkFlc!Wj2V~1Y
zZg5|{<~xac$#;{%@bpSDPK!>HRL_S%--x}x(55x`X2;`jlWh}0N@4KKBaCa%<2o3g
z>*_XluaHLX{sE)L^InaL-g8IU!xa#H03IdBG)r}w&x_RK!f0mgwvCjly@qC*1#*Cf
z)Ps~q!}dplyYGD5<0rXqf+yE~({!DVcy9eIl|K4&vXGVMvaYMX&an(2X1z@zer
zq7Gs4J3`Y^oASWw*sEs3$(M=Y%p4z!@ALQF9bk-2)1{6lC>#%8l>U$s)gB3ru*RLI_kfW>OMjTlMK7HekBzPK&taLUaL1$;8W2f
zi*L<-D=+&m8(}-_*JTno2;p{a_CVI<`U
zZdi?qSq38E8RFHU@{C0A3@N8sz;L+jKAi=%!2E
zg@Cr)nw*RiF7TRw8qdR*8E?L}CO2}^vA378pj-ec@|mKZwh9`@Oippms|1WopoB}%
zQ}nVX43P&huH}$y1{DRFC@l^jYs~)0Gkb|3+_(_4L)F%P%hOd&l{{28s?_8k>}aCM
zG&1o~7vv)I0e6En7-TNp`OwBQJ82Jgs`14cXPDGWpD|lNsqD?GX6Gbz+?Fe@%MI6=
z&yNqAbZ5m?JcjMgHApk4KDgJ}**TQ8DRZzhfzP(Jx=~rUl*n7wBQ)Ow*CFHN6AZYH
ze=zwmz5cDa3hU^(Mn&Uuq08_24(|15ZKgHFp1Ng4R&^%_ohG`O(>pN3lLAti%KXe
zW?=*9)5`2sm=0z{zJKphIc(_u9{tEMQ?#jvgFXO`NHvp1g+aKd`CF^7sR0HMXO|Ik
zIc4rADj+b-Cuv?KJE|*~^ju00po;Rw7o^$W-_M5hV8EfXpn&?rlf)I4H`{d=Y@@L9
zdxb~bo^#IG{TR69(yhTUJ$7j}Rgf)asu0h4OPw0y2BnYktOG^J*d0oi^-u13sqO3o;|!>1@vBORuU|hfU2u82xt}=%{PMRmNFUnoOdikOLnXvLM}s@
zQbSd{>lrNA*w}Vo&`I8R%>XLW@{$axemPdN9$$~t`vo={`B5UTefEoFKIO4OO?pf`
zipK^^y@}(=@gE=D;d2PjFK&buESfJI_j1oi1Fp^$+5t4yqZw6Y&Bu$FfR0v#$Jqh;
zmZYcuocpd|JiAFK_8CxbNqF&WeJCqSA?x{2owLnD%}UG9i<1PD$ny{h2LyZ|5aX=C
z*bhn9CGD#K5Pduw4Z(Dczoz
zFx`HqLcXzbGsVHQSEG5#Ml;o~6|!HDzMdbpX=cI~*z>9((Q5I4y4N}nizoy2%--D{
zpfSd+CxR;qhePHMmZIbmK1b~oH!bQqO&ZMwl)e)gKrz9ACiA>*u0H{~w%ylZmIG