github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-16 06:30:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

[misc] update pre-commit and run all files (#4752)

Browse Source 
* [misc] update pre-commit

* [misc] run pre-commit

* [misc] remove useless configuration files

* [misc] ignore cuda for clang-format
This commit is contained in:
Hongxin Liu
2023-09-19 14:20:26 +08:00
committed by GitHub
parent 3c6b831c26
commit 079bf3cb26
1268 changed files with 50037 additions and 38444 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
tests/test_legacy/test_amp/test_naive_fp16.py
View File

@@ -13,7 +13,7 @@ def check_equal(a, b):
"""
This function checks if two tensors are equal within tolerance
"""
assert torch.allclose(a.float(), b.float(), rtol=1e-4, atol=1e-3), f'a = {a}, b = {b}'
assert torch.allclose(a.float(), b.float(), rtol=1e-4, atol=1e-3), f"a = {a}, b = {b}"
def run_naive_amp():
@@ -25,7 +25,7 @@ def run_naive_amp():
torch.backends.cudnn.deterministic = True
# create layer
test_models = ['repeated_computed_layers', 'nested_model', 'resnet18']
test_models = ["repeated_computed_layers", "nested_model", "resnet18"]
for test_name in test_models:
get_component_func = non_distributed_component_funcs.get_callable(test_name)
model_builder, train_dataloader, _, optim_class, _ = get_component_func()
@@ -41,9 +41,10 @@ def run_naive_amp():
# inject naive and apex amp
naive_amp_config = dict(initial_scale=128, clip_grad_norm=1.0)
naive_amp_model, naive_amp_optimizer = convert_to_naive_amp(naive_amp_model, naive_amp_optimizer,
naive_amp_config)
apex_amp_config = dict(opt_level='O2', loss_scale=128, keep_batchnorm_fp32=False)
naive_amp_model, naive_amp_optimizer = convert_to_naive_amp(
naive_amp_model, naive_amp_optimizer, naive_amp_config
)
apex_amp_config = dict(opt_level="O2", loss_scale=128, keep_batchnorm_fp32=False)
apex_amp_model, apex_amp_optimizer = convert_to_apex_amp(apex_amp_model, apex_amp_optimizer, apex_amp_config)
# create data
@@ -78,7 +79,7 @@ def run_naive_amp():
def run_dist(rank, world_size, port):
colossalai.legacy.launch(config=dict(), rank=rank, world_size=world_size, port=port, host='localhost')
colossalai.legacy.launch(config=dict(), rank=rank, world_size=world_size, port=port, host="localhost")
run_naive_amp()
@@ -89,5 +90,5 @@ def test_naive_amp():
spawn(run_dist, 1)
if __name__ == '__main__':
if __name__ == "__main__":
test_naive_amp()

16
tests/test_legacy/test_amp/test_torch_fp16.py
View File

@@ -18,7 +18,7 @@ def run_torch_amp():
torch.backends.cudnn.deterministic = True
# create layer
test_models = ['resnet18', 'simple_net']
test_models = ["resnet18", "simple_net"]
for test_name in test_models:
get_component_func = non_distributed_component_funcs.get_callable(test_name)
model_builder, train_dataloader, _, optim_class, _ = get_component_func()
@@ -34,10 +34,10 @@ def run_torch_amp():
# inject torch and apex amp
torch_amp_config = dict(init_scale=128, enabled=True)
torch_amp_model, torch_amp_optimizer, _ = convert_to_torch_amp(torch_amp_model,
torch_amp_optimizer,
amp_config=torch_amp_config)
apex_amp_config = dict(opt_level='O1', loss_scale=128)
torch_amp_model, torch_amp_optimizer, _ = convert_to_torch_amp(
torch_amp_model, torch_amp_optimizer, amp_config=torch_amp_config
)
apex_amp_config = dict(opt_level="O1", loss_scale=128)
apex_amp_model, apex_amp_optimizer = convert_to_apex_amp(apex_amp_model, apex_amp_optimizer, apex_amp_config)
# create data
@@ -61,7 +61,7 @@ def run_torch_amp():
# check grad
# In apex amp, grad is not scaled before backward, but torch amp does
for torch_amp_param, apex_amp_param in zip(torch_amp_model.parameters(), apex_amp_model.parameters()):
assert_close_loose(torch_amp_param.grad, apex_amp_param.grad * apex_amp_config['loss_scale'])
assert_close_loose(torch_amp_param.grad, apex_amp_param.grad * apex_amp_config["loss_scale"])
# clip gradient
apex_amp_optimizer.clip_grad_norm(model=apex_amp_model, max_norm=1.0)
@@ -78,7 +78,7 @@ def run_torch_amp():
def run_dist(rank, world_size, port):
colossalai.legacy.launch(config=dict(), rank=rank, world_size=world_size, port=port, host='localhost')
colossalai.legacy.launch(config=dict(), rank=rank, world_size=world_size, port=port, host="localhost")
run_torch_amp()
@@ -89,5 +89,5 @@ def test_torch_amp():
spawn(run_dist, 1)
if __name__ == '__main__':
if __name__ == "__main__":
test_torch_amp()