mirror of
https://github.com/hpcaitech/ColossalAI.git
synced 2025-09-07 20:10:17 +00:00
Develop/experiments (#59)
* Add gradient accumulation, fix lr scheduler * fix FP16 optimizer and adapted torch amp with tensor parallel (#18) * fixed bugs in compatibility between torch amp and tensor parallel and performed some minor fixes * fixed trainer * Revert "fixed trainer" This reverts commit2e0b0b7699. * improved consistency between trainer, engine and schedule (#23) Co-authored-by: 1SAA <c2h214748@gmail.com> * Split conv2d, class token, positional embedding in 2d, Fix random number in ddp Fix convergence in cifar10, Imagenet1000 * Integrate 1d tensor parallel in Colossal-AI (#39) * fixed 1D and 2D convergence (#38) * optimized 2D operations * fixed 1D ViT convergence problem * Feature/ddp (#49) * remove redundancy func in setup (#19) (#20) * use env to control the language of doc (#24) (#25) * Support TP-compatible Torch AMP and Update trainer API (#27) * Add gradient accumulation, fix lr scheduler * fix FP16 optimizer and adapted torch amp with tensor parallel (#18) * fixed bugs in compatibility between torch amp and tensor parallel and performed some minor fixes * fixed trainer * Revert "fixed trainer" This reverts commit2e0b0b7699. * improved consistency between trainer, engine and schedule (#23) Co-authored-by: 1SAA <c2h214748@gmail.com> Co-authored-by: 1SAA <c2h214748@gmail.com> Co-authored-by: ver217 <lhx0217@gmail.com> * add an example of ViT-B/16 and remove w_norm clipping in LAMB (#29) * add explanation for ViT example (#35) (#36) * support torch ddp * fix loss accumulation * add log for ddp * change seed * modify timing hook Co-authored-by: Frank Lee <somerlee.9@gmail.com> Co-authored-by: 1SAA <c2h214748@gmail.com> Co-authored-by: binmakeswell <binmakeswell@gmail.com> * Feature/pipeline (#40) * remove redundancy func in setup (#19) (#20) * use env to control the language of doc (#24) (#25) * Support TP-compatible Torch AMP and Update trainer API (#27) * Add gradient accumulation, fix lr scheduler * fix FP16 optimizer and adapted torch amp with tensor parallel (#18) * fixed bugs in compatibility between torch amp and tensor parallel and performed some minor fixes * fixed trainer * Revert "fixed trainer" This reverts commit2e0b0b7699. * improved consistency between trainer, engine and schedule (#23) Co-authored-by: 1SAA <c2h214748@gmail.com> Co-authored-by: 1SAA <c2h214748@gmail.com> Co-authored-by: ver217 <lhx0217@gmail.com> * add an example of ViT-B/16 and remove w_norm clipping in LAMB (#29) * add explanation for ViT example (#35) (#36) * optimize communication of pipeline parallel * fix grad clip for pipeline Co-authored-by: Frank Lee <somerlee.9@gmail.com> Co-authored-by: 1SAA <c2h214748@gmail.com> Co-authored-by: binmakeswell <binmakeswell@gmail.com> * optimized 3d layer to fix slow computation ; tested imagenet performance with 3d; reworked lr_scheduler config definition; fixed launch args; fixed some printing issues; simplified apis of 3d layers (#51) * Update 2.5d layer code to get a similar accuracy on imagenet-1k dataset * update api for better usability (#58) update api for better usability Co-authored-by: 1SAA <c2h214748@gmail.com> Co-authored-by: ver217 <lhx0217@gmail.com> Co-authored-by: puck_WCR <46049915+WANG-CR@users.noreply.github.com> Co-authored-by: binmakeswell <binmakeswell@gmail.com> Co-authored-by: アマデウス <kurisusnowdeng@users.noreply.github.com> Co-authored-by: BoxiangW <45734921+BoxiangW@users.noreply.github.com>
This commit is contained in:
Frank Lee
GitHub
@@ -8,19 +8,28 @@ import torch
|
||||
|
||||
from colossalai.context.parallel_mode import ParallelMode
|
||||
from colossalai.core import global_context as gpc
|
||||
from colossalai.logging import get_global_dist_logger
|
||||
from colossalai.logging import get_dist_logger
|
||||
|
||||
|
||||
def bytes_to_GB(val, decimal=2):
|
||||
'''A byte-to-Gigabyte converter, defaultly using binary notation.
|
||||
|
||||
:param val: X bytes to convert
|
||||
:return: X' Gb
|
||||
:param val: X bytes to convert
|
||||
:return: X' GB
|
||||
'''
|
||||
return round(val / (1024 * 1024 * 1024), decimal)
|
||||
|
||||
|
||||
def report_memory_usage(message):
|
||||
def bytes_to_MB(val, decimal=2):
|
||||
'''A byte-to-Megabyte converter, defaultly using binary notation.
|
||||
|
||||
:param val: X bytes to convert
|
||||
:return: X' MB
|
||||
'''
|
||||
return round(val / (1024 * 1024), decimal)
|
||||
|
||||
|
||||
def report_memory_usage(message, logger=None, report_cpu=False):
|
||||
'''Calculate and print RAM usage (in GB)
|
||||
|
||||
:param message: a prefix message to add in the log
|
||||
@@ -30,19 +39,24 @@ def report_memory_usage(message):
|
||||
if not gpc.is_initialized(ParallelMode.GLOBAL):
|
||||
raise EnvironmentError("No distributed environment is initialized")
|
||||
|
||||
# python doesn't do real-time garbage collection so do it explicitly to get the correct RAM reports
|
||||
gc.collect()
|
||||
vm_stats = psutil.virtual_memory()
|
||||
vm_used = bytes_to_GB(vm_stats.total - vm_stats.available)
|
||||
gpu_allocated = bytes_to_MB(torch.cuda.memory_allocated())
|
||||
gpu_max_allocated = bytes_to_MB(torch.cuda.max_memory_allocated())
|
||||
gpu_cached = bytes_to_MB(torch.cuda.memory_reserved())
|
||||
gpu_max_cached = bytes_to_MB(torch.cuda.max_memory_reserved())
|
||||
|
||||
gpu_allocated = bytes_to_GB(torch.cuda.memory_allocated())
|
||||
gpu_max_allocated = bytes_to_GB(torch.cuda.max_memory_allocated())
|
||||
gpu_cached = bytes_to_GB(torch.cuda.memory_cached())
|
||||
gpu_max_cached = bytes_to_GB(torch.cuda.max_memory_cached())
|
||||
full_log = f"{message} - GPU: allocated {gpu_allocated} MB, max allocated {gpu_max_allocated} MB, \
|
||||
cached: {gpu_cached} MB, max cached: {gpu_max_cached} MB"
|
||||
|
||||
get_global_dist_logger().info(
|
||||
f"{message} - GPU: allocated {gpu_allocated}GB, max allocated {gpu_max_allocated}GB, cached: {gpu_cached} GB, "
|
||||
f"max cached: {gpu_max_cached}GB, CPU Virtual Memory: used = {vm_used}GB, percent = {vm_stats.percent}%")
|
||||
if report_cpu:
|
||||
# python doesn't do real-time garbage collection so do it explicitly to get the correct RAM reports
|
||||
gc.collect()
|
||||
vm_stats=psutil.virtual_memory()
|
||||
vm_used=bytes_to_MB(vm_stats.total - vm_stats.available)
|
||||
full_log += f", CPU Virtual Memory: used = {vm_used} MB, percent = {vm_stats.percent}%"
|
||||
|
||||
if logger is None:
|
||||
logger = get_dist_logger()
|
||||
logger.info(full_log)
|
||||
|
||||
# get the peak memory to report correct data, so reset the counter for the next call
|
||||
if hasattr(torch.cuda, "reset_peak_memory_stats"): # pytorch 1.4+
|
||||
|
||||
Reference in New Issue
Block a user