mirror of
https://github.com/hpcaitech/ColossalAI.git
synced 2026-07-12 10:54:49 +00:00
27061426f7
* [gemini] remove distributed-related part from colotensor (#4379) * [gemini] remove process group dependency * [gemini] remove tp part from colo tensor * [gemini] patch inplace op * [gemini] fix param op hook and update tests * [test] remove useless tests * [test] remove useless tests * [misc] fix requirements * [test] fix model zoo * [test] fix model zoo * [test] fix model zoo * [test] fix model zoo * [test] fix model zoo * [misc] update requirements * [gemini] refactor gemini optimizer and gemini ddp (#4398) * [gemini] update optimizer interface * [gemini] renaming gemini optimizer * [gemini] refactor gemini ddp class * [example] update gemini related example * [example] update gemini related example * [plugin] fix gemini plugin args * [test] update gemini ckpt tests * [gemini] fix checkpoint io * [example] fix opt example requirements * [example] fix opt example * [example] fix opt example * [example] fix opt example * [gemini] add static placement policy (#4443) * [gemini] add static placement policy * [gemini] fix param offload * [test] update gemini tests * [plugin] update gemini plugin * [plugin] update gemini plugin docstr * [misc] fix flash attn requirement * [test] fix gemini checkpoint io test * [example] update resnet example result (#4457) * [example] update bert example result (#4458) * [doc] update gemini doc (#4468) * [example] update gemini related examples (#4473) * [example] update gpt example * [example] update dreambooth example * [example] update vit * [example] update opt * [example] update palm * [example] update vit and opt benchmark * [hotfix] fix bert in model zoo (#4480) * [hotfix] fix bert in model zoo * [test] remove chatglm gemini test * [test] remove sam gemini test * [test] remove vit gemini test * [hotfix] fix opt tutorial example (#4497) * [hotfix] fix opt tutorial example * [hotfix] fix opt tutorial example
55 lines
1.9 KiB
Python
55 lines
1.9 KiB
Python
from copy import deepcopy
|
|
|
|
import numpy as np
|
|
import pytest
|
|
import torch
|
|
|
|
from colossalai.testing import clear_cache_before_run
|
|
from colossalai.zero.gemini.memory_tracer.runtime_mem_tracer import RuntimeMemTracer
|
|
from tests.components_to_test import run_fwd_bwd
|
|
from tests.components_to_test.registry import non_distributed_component_funcs
|
|
|
|
|
|
@pytest.mark.skip("this is not used")
|
|
@clear_cache_before_run()
|
|
def test_runtime_mem_tracer():
|
|
test_models = ['gpt2', 'bert', 'simple_net', 'repeated_computed_layers', 'nested_model', 'albert']
|
|
|
|
for model_name in test_models:
|
|
get_components_func = non_distributed_component_funcs.get_callable(model_name)
|
|
model_builder, train_dataloader, _, _, criterion = get_components_func()
|
|
|
|
model = model_builder(checkpoint=False).cuda()
|
|
|
|
model_bk = deepcopy(model)
|
|
runtime_mem_tracer = RuntimeMemTracer(model)
|
|
|
|
for i, (data, label) in enumerate(train_dataloader):
|
|
if i > 1:
|
|
break
|
|
data = data.cuda()
|
|
label = label.cuda()
|
|
|
|
run_fwd_bwd(runtime_mem_tracer, data, label, criterion, optimizer=runtime_mem_tracer)
|
|
|
|
for p1, p2 in zip(model_bk.parameters(), model.parameters()):
|
|
torch.allclose(p1.to(torch.half), p2)
|
|
|
|
non_model_data_list = runtime_mem_tracer._memstats.non_model_data_list('cuda')
|
|
cuda_non_model_data_list = np.array(non_model_data_list) / 1024**2
|
|
print("cuda_non_model_data_list", len(cuda_non_model_data_list))
|
|
print(non_model_data_list)
|
|
|
|
cnt1 = 0
|
|
for p in runtime_mem_tracer.parameters_in_runtime_order():
|
|
cnt1 += 1
|
|
cnt2 = 0
|
|
for p in model.parameters():
|
|
cnt2 += 1
|
|
assert cnt2 == cnt1, f'visited param number {cnt1} vs real param number {cnt2}'
|
|
del model
|
|
|
|
|
|
if __name__ == '__main__':
|
|
test_runtime_mem_tracer()
|