github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-23 10:30:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix issue #1080 (#1071)

Browse Source
This commit is contained in:
Jiarui Fang
2022-06-07 17:21:11 +08:00
committed by GitHub
parent 1b17859328
commit bcab249565
2 changed files with 28 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
colossalai/trainer/hooks/_metric_hook.py
View File

@@ -52,7 +52,7 @@ class Metric(ABC):
pass
@abstractmethod
def get_last_step_value(self) -> str:
def get_last_step_value(self) -> float:
"""Returns the metric value in the last iteration.
"""
pass
@@ -121,10 +121,10 @@ class LossMetric(Metric):
self.accum_loss.div_(self.count)
return self.accum_loss.item()
def get_last_step_value(self) -> str:
def get_last_step_value(self) -> float:
"""Returns :attr:`last_step_loss`.
"""
return str(self.last_step_loss.cpu().item())
return self.last_step_loss.cpu().item()
@staticmethod
def is_better(a, b):
@@ -149,8 +149,8 @@ class LearningRateMetric(Metric):
def update(self, lr) -> None:
self.lr = lr
def get_last_step_value(self) -> str:
return str(self.lr)
def get_last_step_value(self) -> float:
return self.lr
def get_accumulated_value(self):
return self.lr
@@ -204,10 +204,10 @@ class AccuracyMetric(Metric):
self.accumulated_sum += self.last_step_sum
self.accumulated_correct += self.last_step_correct
def get_last_step_value(self) -> str:
def get_last_step_value(self) -> float:
self.last_step_sum = all_reduce(self.last_step_sum, ParallelMode.DATA)
self.last_step_correct = all_reduce(self.last_step_correct, ParallelMode.DATA)
return str(_format_number((self.last_step_correct / self.last_step_sum).cpu().item()))
return _format_number((self.last_step_correct / self.last_step_sum).cpu().item())
def get_accumulated_value(self):
self.accumulated_sum = all_reduce(self.accumulated_sum, ParallelMode.DATA)
@@ -350,7 +350,18 @@ class ThroughputMetric(Metric):
self.accumulated_num_samples += self.last_step_num_samples
self.accumulated_used_time += self.last_step_used_time
def get_last_step_value(self) -> str:
def get_last_step_value(self) -> float:
if self._use_local:
self.last_step_num_samples *= gpc.get_world_size(ParallelMode.DATA)
else:
self.last_step_used_time = all_reduce(self.last_step_used_time, ParallelMode.DATA) / \
gpc.get_world_size(ParallelMode.DATA)
self.last_step_num_samples = all_reduce(self.last_step_num_samples, ParallelMode.DATA)
sample_per_sec = _format_number(self.last_step_num_samples / (self.last_step_used_time + 1e-12).item())
return sample_per_sec
def get_last_step_info(self) -> str:
if self._use_local:
self.last_step_num_samples *= gpc.get_world_size(ParallelMode.DATA)
else: