github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-08-31 16:40:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

[format] applied code formatting on changed files in pull request 5067 (#5072)

Browse Source 
Co-authored-by: github-actions <github-actions@github.com>
This commit is contained in:
github-actions[bot]
2023-11-20 19:46:43 +08:00
committed by GitHub
parent fb103cfd6e
commit 8921a73c90
6 changed files with 27 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
examples/language/llama2/benchmark.py
View File

@@ -188,7 +188,9 @@ def main():
model.config.num_hidden_layers,
model.config.hidden_size,
model.config.vocab_size,
args.grad_checkpoint, args.ignore_steps, dp_world_size=dp_size
args.grad_checkpoint,
args.ignore_steps,
dp_world_size=dp_size,
)
optimizer = HybridAdam(model.parameters())

8
examples/language/llama2/performance_evaluator.py
View File

@@ -98,8 +98,12 @@ class PerformanceEvaluator:
batch_size, seq_len = input_ids.shape
self.num_samples += batch_size
checkpoint_activations_factor = (3 + int(self.enable_grad_checkpoint))
self.flop_megatron += (24 * checkpoint_activations_factor * batch_size * seq_len * self.num_layers * (self.hidden_size**2)) * (1. + (seq_len / (6. * self.hidden_size)) + (self.vocab_size / (16. * self.num_layers * self.hidden_size)))
checkpoint_activations_factor = 3 + int(self.enable_grad_checkpoint)
self.flop_megatron += (
24 * checkpoint_activations_factor * batch_size * seq_len * self.num_layers * (self.hidden_size**2)
) * (
1.0 + (seq_len / (6.0 * self.hidden_size)) + (self.vocab_size / (16.0 * self.num_layers * self.hidden_size))
)
self.flop += batch_size * seq_len * self.model_numel * 2 * (3 + int(self.enable_grad_checkpoint))
def on_fit_end(self) -> None: