mirror of
https://github.com/hpcaitech/ColossalAI.git
synced 2025-09-16 14:41:53 +00:00
[legacy] move communication and nn to legacy and refactor logger (#4671)
* [legacy] move communication to legacy (#4640) * [legacy] refactor logger and clean up legacy codes (#4654) * [legacy] make logger independent to gpc * [legacy] make optim independent to registry * [legacy] move test engine to legacy * [legacy] move nn to legacy (#4656) * [legacy] move nn to legacy * [checkpointio] fix save hf config * [test] remove useledd rpc pp test * [legacy] fix nn init * [example] skip tutorial hybriad parallel example * [devops] test doc check * [devops] test doc check
This commit is contained in:
Hongxin Liu
GitHub
@@ -1,41 +1 @@
|
||||
from colossalai.global_variables import tensor_parallel_env as env
|
||||
from colossalai.nn.layer.utils import get_tensor_parallel_mode
|
||||
from torch import nn
|
||||
from torch.nn.modules.loss import *
|
||||
from torch.nn.modules.loss import _Loss
|
||||
|
||||
from .loss_1d import VocabParallelCrossEntropyLoss1D
|
||||
from .loss_2d import CrossEntropyLoss2D, VocabParallelCrossEntropyLoss2D
|
||||
from .loss_2p5d import CrossEntropyLoss2p5D, VocabParallelCrossEntropyLoss2p5D
|
||||
from .loss_3d import CrossEntropyLoss3D, VocabParallelCrossEntropyLoss3D
|
||||
from .loss_moe import MoeCrossEntropyLoss, MoeLoss
|
||||
|
||||
_parallel_cross_entropy = {
|
||||
'2d': CrossEntropyLoss2D,
|
||||
'2.5d': CrossEntropyLoss2p5D,
|
||||
'3d': CrossEntropyLoss3D,
|
||||
}
|
||||
|
||||
_vocab_parallel_cross_entropy = {
|
||||
'1d': VocabParallelCrossEntropyLoss1D,
|
||||
'2d': VocabParallelCrossEntropyLoss2D,
|
||||
'2.5d': VocabParallelCrossEntropyLoss2p5D,
|
||||
'3d': VocabParallelCrossEntropyLoss3D,
|
||||
}
|
||||
|
||||
|
||||
class CrossEntropyLoss(_Loss):
|
||||
|
||||
def __init__(self, reduction: bool = True, *args, **kwargs):
|
||||
super().__init__()
|
||||
tensor_parallel = get_tensor_parallel_mode()
|
||||
if tensor_parallel is not None and env.vocab_parallel:
|
||||
self.loss = _vocab_parallel_cross_entropy[tensor_parallel](reduction=reduction, *args, **kwargs)
|
||||
elif tensor_parallel is None or tensor_parallel == '1d':
|
||||
reduction = 'mean' if reduction else 'none'
|
||||
self.loss = nn.CrossEntropyLoss(reduction=reduction, *args, **kwargs)
|
||||
else:
|
||||
self.loss = _parallel_cross_entropy[tensor_parallel](reduction=reduction, *args, **kwargs)
|
||||
|
||||
def forward(self, *args):
|
||||
return self.loss(*args)
|
||||
|
||||
@@ -1,106 +0,0 @@
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
from torch.cuda.amp import custom_bwd, custom_fwd
|
||||
from torch.nn.modules.loss import _Loss
|
||||
|
||||
from colossalai.context import ParallelMode
|
||||
from colossalai.core import global_context as gpc
|
||||
from colossalai.legacy.registry import LOSSES
|
||||
|
||||
|
||||
class _VocabParallelCrossEntropy1D(torch.autograd.Function):
|
||||
|
||||
@staticmethod
|
||||
@custom_fwd(cast_inputs=torch.float32)
|
||||
def forward(ctx, vocab_parallel_logits, targets, process_group):
|
||||
if process_group is None:
|
||||
process_group = gpc.get_group(ParallelMode.PARALLEL_1D)
|
||||
|
||||
# Maximum value along vocab dimension across all GPUs.
|
||||
logits_max = torch.max(vocab_parallel_logits, dim=-1)[0]
|
||||
torch.distributed.all_reduce(logits_max, op=torch.distributed.ReduceOp.MAX, group=process_group)
|
||||
# Subtract the maximum value.
|
||||
vocab_parallel_logits.sub_(logits_max.unsqueeze(dim=-1))
|
||||
|
||||
# Get the partition's vocab indices
|
||||
partition_vocab_size = vocab_parallel_logits.size()[-1]
|
||||
rank = dist.get_rank(process_group)
|
||||
vocab_start_index = partition_vocab_size * rank
|
||||
vocab_end_index = vocab_start_index + partition_vocab_size
|
||||
|
||||
# Create a mask of valid vocab ids (1 means it needs to be masked).
|
||||
target_mask = (targets < vocab_start_index) | (targets >= vocab_end_index)
|
||||
masked_target = targets.clone() - vocab_start_index
|
||||
masked_target[target_mask] = 0
|
||||
|
||||
# Get predicted-logits = logits[target].
|
||||
# For Simplicity, we convert logits to a 2-D tensor with size
|
||||
# [*, partition-vocab-size] and target to a 1-D tensor of size [*].
|
||||
logits_2d = vocab_parallel_logits.view(-1, partition_vocab_size)
|
||||
masked_target_1d = masked_target.view(-1)
|
||||
arange_1d = torch.arange(start=0, end=logits_2d.size()[0], device=logits_2d.device)
|
||||
predicted_logits_1d = logits_2d[arange_1d, masked_target_1d]
|
||||
predicted_logits_1d = predicted_logits_1d.clone().contiguous()
|
||||
predicted_logits = predicted_logits_1d.view_as(targets)
|
||||
predicted_logits[target_mask] = 0.0
|
||||
# All reduce is needed to get the chunks from other GPUs.
|
||||
torch.distributed.all_reduce(predicted_logits, op=torch.distributed.ReduceOp.SUM, group=process_group)
|
||||
|
||||
# Sum of exponential of logits along vocab dimension across all GPUs.
|
||||
exp_logits = torch.exp(vocab_parallel_logits)
|
||||
sum_exp_logits = exp_logits.sum(dim=-1)
|
||||
torch.distributed.all_reduce(sum_exp_logits, op=torch.distributed.ReduceOp.SUM, group=process_group)
|
||||
|
||||
# Loss = log(sum(exp(logits))) - predicted-logit.
|
||||
loss = torch.log(sum_exp_logits) - predicted_logits
|
||||
# Store softmax, target-mask and masked-target for backward pass.
|
||||
exp_logits.div_(sum_exp_logits.unsqueeze(dim=-1))
|
||||
ctx.save_for_backward(exp_logits, target_mask, masked_target_1d)
|
||||
return loss
|
||||
|
||||
@staticmethod
|
||||
@custom_bwd
|
||||
def backward(ctx, grad_output):
|
||||
|
||||
# Retrieve tensors from the forward path.
|
||||
softmax, target_mask, masked_target_1d = ctx.saved_tensors
|
||||
|
||||
# All the inputs have softmax as their gradient.
|
||||
grad_input = softmax
|
||||
# For simplicity, work with the 2D gradient.
|
||||
partition_vocab_size = softmax.size()[-1]
|
||||
grad_2d = grad_input.view(-1, partition_vocab_size)
|
||||
|
||||
# Add the gradient from matching classes.
|
||||
arange_1d = torch.arange(start=0, end=grad_2d.size()[0], device=grad_2d.device)
|
||||
grad_2d[arange_1d, masked_target_1d] -= (1.0 - target_mask.view(-1).float())
|
||||
|
||||
# Finally elementwise multiplication with the output gradients.
|
||||
grad_input.mul_(grad_output.unsqueeze(dim=-1))
|
||||
|
||||
return grad_input, None, None
|
||||
|
||||
|
||||
@LOSSES.register_module
|
||||
class VocabParallelCrossEntropyLoss1D(_Loss):
|
||||
"""Vocab parallel cross entropy loss for 1D parallelism.
|
||||
|
||||
Args:
|
||||
reduction (bool, optional): whether to average the loss, defaults to True.
|
||||
"""
|
||||
|
||||
def __init__(self, reduction=True):
|
||||
super().__init__()
|
||||
self.reduction_mean = reduction
|
||||
|
||||
def forward(self, logits, targets, process_group=None):
|
||||
"""Calculate loss between logits and targets.
|
||||
|
||||
Args:
|
||||
logits (:class:`torch.tensor`): Predicted unnormalized scores (often referred to as logits).
|
||||
targets (:class:`torch.tensor`): Ground truth class indices or class probabilities.
|
||||
"""
|
||||
loss = _VocabParallelCrossEntropy1D.apply(logits, targets, process_group)
|
||||
if self.reduction_mean:
|
||||
loss = loss.mean()
|
||||
return loss
|
||||
@@ -1,157 +0,0 @@
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
from torch.cuda.amp import custom_bwd, custom_fwd
|
||||
from torch.nn.functional import cross_entropy
|
||||
from torch.nn.modules.loss import _Loss
|
||||
|
||||
from colossalai.context import ParallelMode
|
||||
from colossalai.core import global_context as gpc
|
||||
from colossalai.legacy.registry import LOSSES
|
||||
from colossalai.nn.layer.parallel_2d import reduce_by_batch_2d, split_batch_2d
|
||||
from colossalai.nn.layer.parallel_2d._utils import assert_summa_initialization
|
||||
from colossalai.utils import get_current_device
|
||||
|
||||
|
||||
@LOSSES.register_module
|
||||
class CrossEntropyLoss2D(_Loss):
|
||||
r"""Cross entropy loss for 2D parallelism
|
||||
|
||||
Args:
|
||||
reduction (bool, optional): whether to average the loss, defaults to True.
|
||||
|
||||
The ``args`` and ``kwargs`` should include parameters below:
|
||||
::
|
||||
|
||||
weight (Tensor, optional)
|
||||
size_average (bool, optional)
|
||||
ignore_index (int, optional)
|
||||
reduce (bool, optional)
|
||||
label_smoothing (float, optional)
|
||||
|
||||
More details about ``args``, ``kwargs`` and ``torch.nn.functional.cross_entropy`` could be found in
|
||||
`Cross_entropy <https://pytorch.org/docs/stable/generated/torch.nn.functional.cross_entropy.html#torch.nn.functional.cross_entropy>`_.
|
||||
"""
|
||||
|
||||
def __init__(self, reduction=True, *args, **kwargs):
|
||||
super().__init__()
|
||||
assert_summa_initialization()
|
||||
self.reduction_mean = reduction
|
||||
self.loss_args = args
|
||||
self.loss_kwargs = kwargs
|
||||
|
||||
def forward(self, logits, targets):
|
||||
"""Calculate loss between logits and targets.
|
||||
|
||||
Args:
|
||||
logits (:class:`torch.tensor`): Predicted unnormalized scores (often referred to as logits).
|
||||
targets (:class:`torch.tensor`): Ground truth class indices or class probabilities.
|
||||
|
||||
Returns:
|
||||
float: the loss between logits and targets.
|
||||
"""
|
||||
targets = split_batch_2d(targets)
|
||||
loss = cross_entropy(logits, targets, reduction='none', *self.loss_args, **self.loss_kwargs)
|
||||
if self.reduction_mean:
|
||||
loss = loss.mean()
|
||||
loss = reduce_by_batch_2d(loss, True)
|
||||
return loss
|
||||
|
||||
|
||||
class _VocabParallelCrossEntropy2D(torch.autograd.Function):
|
||||
### Modified based on megatron.mpu.cross_entropy ###
|
||||
|
||||
@staticmethod
|
||||
@custom_fwd(cast_inputs=torch.float32)
|
||||
def forward(ctx, logits, targets):
|
||||
# logits: [b/q, h/q]
|
||||
# labels: [b/q]
|
||||
# loss: [b/q]
|
||||
# vocab_parallel_logits: [b/q, s, v/q]
|
||||
# target: [b/q, s]
|
||||
logits_max = torch.max(logits, dim=-1)[0]
|
||||
torch.distributed.all_reduce(logits_max,
|
||||
op=torch.distributed.ReduceOp.MAX,
|
||||
group=gpc.get_group(ParallelMode.PARALLEL_2D_ROW))
|
||||
# Subtract the maximum value.
|
||||
# vocab_parallel_logits.sub_(logits_max.unsqueeze(dim=-1))
|
||||
logits = logits - logits_max.unsqueeze(dim=-1)
|
||||
|
||||
vocab_size = logits.size(-1)
|
||||
rank = gpc.get_local_rank(ParallelMode.PARALLEL_2D_ROW)
|
||||
vocab_start = rank * (vocab_size)
|
||||
vocab_end = (rank + 1) * (vocab_size) - 1
|
||||
|
||||
target_mask = (targets < vocab_start) | (targets > vocab_end)
|
||||
|
||||
masked_target = targets.clone() - vocab_start
|
||||
masked_target[target_mask] = 0
|
||||
arange_1d = torch.arange(
|
||||
start=0,
|
||||
end=logits.size()[0],
|
||||
)
|
||||
predicted_logits = logits[arange_1d, masked_target]
|
||||
predicted_logits[target_mask] = 0.
|
||||
dist.all_reduce(predicted_logits, group=gpc.get_group(ParallelMode.PARALLEL_2D_ROW))
|
||||
|
||||
exp_logits = torch.exp(logits)
|
||||
sum_exp_logits = exp_logits.sum(dim=1)
|
||||
dist.all_reduce(sum_exp_logits, group=gpc.get_group(ParallelMode.PARALLEL_2D_ROW))
|
||||
|
||||
loss = torch.log(sum_exp_logits) - predicted_logits
|
||||
|
||||
exp_logits.div_(sum_exp_logits.unsqueeze(dim=-1))
|
||||
ctx.save_for_backward(exp_logits, target_mask, masked_target)
|
||||
|
||||
return loss
|
||||
|
||||
@staticmethod
|
||||
@custom_bwd
|
||||
def backward(ctx, output_grad):
|
||||
# Retrieve tensors from the forward path.
|
||||
softmax, target_mask, masked_target = ctx.saved_tensors
|
||||
|
||||
# All the inputs have softmax as their gradient.
|
||||
grad_input = softmax
|
||||
|
||||
# For simplicity, work with the 2D gradient.
|
||||
partition_vocab_size = softmax.size()[-1]
|
||||
grad_2d = grad_input.view(-1, partition_vocab_size)
|
||||
|
||||
# Add the gradient from matching classes.
|
||||
arange_1d = torch.arange(start=0, end=grad_2d.size()[0], device=get_current_device())
|
||||
grad_2d[arange_1d, masked_target] -= (1.0 - target_mask.view(-1).float())
|
||||
|
||||
# Finally elementwise multiplication with the output gradients.
|
||||
grad_input.mul_(output_grad.unsqueeze(dim=-1))
|
||||
|
||||
return grad_input, None
|
||||
|
||||
|
||||
@LOSSES.register_module
|
||||
class VocabParallelCrossEntropyLoss2D(_Loss):
|
||||
"""Vocab parallel cross entropy loss for 2D parallelism.
|
||||
|
||||
Args:
|
||||
reduction (bool, optional): whether to average the loss, defaults to True.
|
||||
"""
|
||||
|
||||
def __init__(self, reduction=True):
|
||||
super().__init__()
|
||||
self.reduction_mean = reduction
|
||||
|
||||
def forward(self, logits, targets):
|
||||
"""Calculate loss between logits and targets.
|
||||
|
||||
Args:
|
||||
logits (:class:`torch.tensor`): Predicted unnormalized scores (often referred to as logits).
|
||||
targets (:class:`torch.tensor`): Ground truth class indices or class probabilities.
|
||||
"""
|
||||
targets = split_batch_2d(targets)
|
||||
loss = _VocabParallelCrossEntropy2D.apply(
|
||||
logits,
|
||||
targets,
|
||||
)
|
||||
if self.reduction_mean:
|
||||
loss = loss.mean()
|
||||
loss = reduce_by_batch_2d(loss, True)
|
||||
return loss
|
||||
@@ -1,150 +0,0 @@
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
from torch.cuda.amp import custom_bwd, custom_fwd
|
||||
from torch.nn.functional import cross_entropy
|
||||
from torch.nn.modules.loss import _Loss
|
||||
|
||||
from colossalai.context import ParallelMode
|
||||
from colossalai.core import global_context as gpc
|
||||
from colossalai.legacy.registry import LOSSES
|
||||
from colossalai.nn.layer.parallel_2p5d import reduce_by_batch_2p5d, split_batch_2p5d
|
||||
from colossalai.nn.layer.parallel_2p5d._utils import assert_tesseract_initialization
|
||||
from colossalai.utils import get_current_device
|
||||
|
||||
|
||||
@LOSSES.register_module
|
||||
class CrossEntropyLoss2p5D(_Loss):
|
||||
r"""Cross entropy loss for 2.5D parallelism
|
||||
|
||||
Args:
|
||||
reduction (bool, optional): whether to average the loss, defaults to True.
|
||||
|
||||
The ``args`` and ``kwargs`` should include parameters below:
|
||||
::
|
||||
|
||||
weight (Tensor, optional)
|
||||
size_average (bool, optional)
|
||||
ignore_index (int, optional)
|
||||
reduce (bool, optional)
|
||||
label_smoothing (float, optional)
|
||||
|
||||
More details about ``args``, ``kwargs`` and ``torch.nn.functional.cross_entropy`` could be found in
|
||||
`Cross_entropy <https://pytorch.org/docs/stable/generated/torch.nn.functional.cross_entropy.html#torch.nn.functional.cross_entropy>`_.
|
||||
"""
|
||||
|
||||
def __init__(self, reduction=True, *args, **kwargs):
|
||||
super().__init__()
|
||||
assert_tesseract_initialization()
|
||||
self.reduction_mean = reduction
|
||||
self.loss_args = args
|
||||
self.loss_kwargs = kwargs
|
||||
|
||||
def forward(self, logits, targets):
|
||||
"""Calculate loss between logits and targets.
|
||||
|
||||
Args:
|
||||
logits (:class:`torch.tensor`): Predicted unnormalized scores (often referred to as logits).
|
||||
targets (:class:`torch.tensor`): Ground truth class indices or class probabilities.
|
||||
"""
|
||||
targets = split_batch_2p5d(targets)
|
||||
loss = cross_entropy(logits, targets, reduction='none', *self.loss_args, **self.loss_kwargs)
|
||||
if self.reduction_mean:
|
||||
loss = loss.mean()
|
||||
loss = reduce_by_batch_2p5d(loss, True)
|
||||
return loss
|
||||
|
||||
|
||||
class _VocabParallelCrossEntropy2p5D(torch.autograd.Function):
|
||||
### Modified based on megatron.mpu.cross_entropy ###
|
||||
|
||||
@staticmethod
|
||||
@custom_fwd(cast_inputs=torch.float32)
|
||||
def forward(ctx, logits, targets):
|
||||
# logits: [b/dq, h/q]
|
||||
# loss: [b/dq]
|
||||
# targets: [b/dq, h/q]
|
||||
logits_max = torch.max(logits, dim=-1)[0]
|
||||
torch.distributed.all_reduce(logits_max,
|
||||
op=torch.distributed.ReduceOp.MAX,
|
||||
group=gpc.get_group(ParallelMode.PARALLEL_2P5D_ROW))
|
||||
# Subtract the maximum value.
|
||||
logits = logits - logits_max.unsqueeze(dim=-1)
|
||||
|
||||
vocab_size = logits.size(-1)
|
||||
rank = gpc.get_local_rank(ParallelMode.PARALLEL_2P5D_ROW)
|
||||
vocab_start = rank * (vocab_size)
|
||||
vocab_end = (rank + 1) * (vocab_size) - 1
|
||||
|
||||
target_mask = (targets < vocab_start) | (targets > vocab_end)
|
||||
|
||||
masked_target = targets.clone() - vocab_start
|
||||
masked_target[target_mask] = 0
|
||||
arange_1d = torch.arange(
|
||||
start=0,
|
||||
end=logits.size()[0],
|
||||
)
|
||||
predicted_logits = logits[arange_1d, masked_target]
|
||||
predicted_logits[target_mask] = 0.
|
||||
dist.all_reduce(predicted_logits, group=gpc.get_group(ParallelMode.PARALLEL_2P5D_ROW))
|
||||
|
||||
exp_logits = torch.exp(logits)
|
||||
sum_exp_logits = exp_logits.sum(dim=1)
|
||||
dist.all_reduce(sum_exp_logits, group=gpc.get_group(ParallelMode.PARALLEL_2P5D_ROW))
|
||||
|
||||
loss = torch.log(sum_exp_logits) - predicted_logits
|
||||
|
||||
exp_logits.div_(sum_exp_logits.unsqueeze(dim=-1))
|
||||
ctx.save_for_backward(exp_logits, target_mask, masked_target)
|
||||
|
||||
return loss
|
||||
|
||||
@staticmethod
|
||||
@custom_bwd
|
||||
def backward(ctx, output_grad):
|
||||
# Retrieve tensors from the forward path.
|
||||
softmax, target_mask, masked_target = ctx.saved_tensors
|
||||
|
||||
# All the inputs have softmax as their gradient.
|
||||
grad_input = softmax
|
||||
|
||||
# For simplicity, work with the 2D gradient.
|
||||
partition_vocab_size = softmax.size()[-1]
|
||||
grad_2d = grad_input.view(-1, partition_vocab_size)
|
||||
|
||||
# Add the gradient from matching classes.
|
||||
arange_1d = torch.arange(start=0, end=grad_2d.size()[0], device=get_current_device())
|
||||
grad_2d[arange_1d, masked_target] -= (1.0 - target_mask.view(-1).float())
|
||||
|
||||
# Finally elementwise multiplication with the output gradients.
|
||||
grad_input.mul_(output_grad.unsqueeze(dim=-1))
|
||||
|
||||
return grad_input, None
|
||||
|
||||
|
||||
@LOSSES.register_module
|
||||
class VocabParallelCrossEntropyLoss2p5D(_Loss):
|
||||
"""
|
||||
Vocab parallel cross entropy loss for 2.5D parallelism
|
||||
|
||||
Args:
|
||||
reduction (bool, optional): whether to average the loss, defaults to True.
|
||||
"""
|
||||
|
||||
def __init__(self, reduction=True):
|
||||
super().__init__()
|
||||
self.reduction_mean = reduction
|
||||
|
||||
def forward(self, logits, targets):
|
||||
"""Calculate loss between logits and targets.
|
||||
|
||||
Args:
|
||||
logits (:class:`torch.tensor`): Predicted unnormalized scores (often referred to as logits).
|
||||
targets (:class:`torch.tensor`): Ground truth class indices or class probabilities.
|
||||
"""
|
||||
targets = split_batch_2p5d(targets)
|
||||
loss = _VocabParallelCrossEntropy2p5D.apply(logits, targets)
|
||||
if self.reduction_mean:
|
||||
loss = loss.mean()
|
||||
loss = reduce_by_batch_2p5d(loss, True)
|
||||
|
||||
return loss
|
||||
@@ -1,148 +0,0 @@
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
from torch.cuda.amp import custom_bwd, custom_fwd
|
||||
from torch.nn.functional import cross_entropy
|
||||
from torch.nn.modules.loss import _Loss
|
||||
|
||||
from colossalai.constants import INPUT_GROUP_3D, OUTPUT_GROUP_3D, WEIGHT_GROUP_3D
|
||||
from colossalai.core import global_context as gpc
|
||||
from colossalai.legacy.registry import LOSSES
|
||||
from colossalai.nn.layer.parallel_3d import reduce_by_batch_3d, split_tensor_3d
|
||||
from colossalai.nn.layer.parallel_3d._utils import get_parallel_mode_from_env
|
||||
from colossalai.utils import get_current_device
|
||||
|
||||
|
||||
@LOSSES.register_module
|
||||
class CrossEntropyLoss3D(_Loss):
|
||||
r"""Cross entropy loss for 3D parallelism.
|
||||
|
||||
Args:
|
||||
reduction (bool, optional): whether to average the loss, defaults to True.
|
||||
|
||||
The ``args`` and ``kwargs`` should include parameters below:
|
||||
::
|
||||
|
||||
weight (Tensor, optional)
|
||||
size_average (bool, optional)
|
||||
ignore_index (int, optional)
|
||||
reduce (bool, optional)
|
||||
label_smoothing (float, optional)
|
||||
|
||||
More details about ``args``, ``kwargs`` and ``torch.nn.functional.cross_entropy`` could be found in
|
||||
`Cross_entropy <https://pytorch.org/docs/stable/generated/torch.nn.functional.cross_entropy.html#torch.nn.functional.cross_entropy>`_.
|
||||
"""
|
||||
|
||||
def __init__(self, reduction=True, *args, **kwargs):
|
||||
super().__init__()
|
||||
self.input_parallel_mode = get_parallel_mode_from_env(INPUT_GROUP_3D)
|
||||
self.weight_parallel_mode = get_parallel_mode_from_env(WEIGHT_GROUP_3D)
|
||||
self.reduction_mean = reduction
|
||||
self.loss_args = args
|
||||
self.loss_kwargs = kwargs
|
||||
|
||||
def forward(self, logits, targets):
|
||||
"""Calculate loss between logits and targets.
|
||||
|
||||
Args:
|
||||
logits (:class:`torch.tensor`): Predicted unnormalized scores (often referred to as logits).
|
||||
targets (:class:`torch.tensor`): Ground truth class indices or class probabilities.
|
||||
"""
|
||||
targets = split_tensor_3d(targets, 0, self.weight_parallel_mode)
|
||||
targets = split_tensor_3d(targets, 0, self.input_parallel_mode)
|
||||
loss = cross_entropy(logits, targets, reduction='none', *self.loss_args, **self.loss_kwargs)
|
||||
if self.reduction_mean:
|
||||
loss = loss.mean()
|
||||
loss = reduce_by_batch_3d(loss, self.input_parallel_mode, self.weight_parallel_mode, True)
|
||||
return loss
|
||||
|
||||
|
||||
class _VocabParallelCrossEntropy3D(torch.autograd.Function):
|
||||
# Adapted from megatron.mpu.cross_entropy
|
||||
# loss[i] = -logits[i][targets] + log(sum(exp(logits[i])))
|
||||
|
||||
@staticmethod
|
||||
@custom_fwd(cast_inputs=torch.float32)
|
||||
def forward(ctx, logits, targets, output_parallel_mode):
|
||||
# logits: [b/q^2, c/q]
|
||||
# labels: [b/q^2]
|
||||
# loss: [b/q^2]
|
||||
logits_max = torch.max(logits, dim=-1)[0]
|
||||
dist.all_reduce(logits_max, op=torch.distributed.ReduceOp.MAX, group=gpc.get_group(output_parallel_mode))
|
||||
# Subtract the maximum value.
|
||||
logits = logits - logits_max.unsqueeze(dim=-1)
|
||||
|
||||
vocab_size_per_partition = logits.size()[-1]
|
||||
rank = gpc.get_local_rank(output_parallel_mode)
|
||||
vocab_start = rank * vocab_size_per_partition
|
||||
vocab_end = (rank + 1) * vocab_size_per_partition - 1
|
||||
|
||||
# loss[i] = 0 if targets[i] < vocab_start or targets[i] > vocab_end
|
||||
target_mask = (targets < vocab_start) | (targets > vocab_end)
|
||||
masked_target = targets.clone() - vocab_start
|
||||
masked_target[target_mask] = 0
|
||||
arange_1d = torch.arange(start=0, end=logits.size()[0], device=get_current_device())
|
||||
predicted_logits = logits[arange_1d, masked_target]
|
||||
predicted_logits = predicted_logits.clone().contiguous().view_as(targets)
|
||||
predicted_logits[target_mask] = 0.
|
||||
dist.all_reduce(predicted_logits, group=gpc.get_group(output_parallel_mode))
|
||||
|
||||
# Loss = log(sum(exp(logits))) - predicted-logit.
|
||||
exp_logits = torch.exp(logits)
|
||||
sum_exp_logits = exp_logits.sum(dim=-1)
|
||||
dist.all_reduce(sum_exp_logits, group=gpc.get_group(output_parallel_mode))
|
||||
loss = torch.log(sum_exp_logits) - predicted_logits
|
||||
|
||||
exp_logits.div_(sum_exp_logits.unsqueeze(dim=-1))
|
||||
ctx.save_for_backward(exp_logits, target_mask, masked_target)
|
||||
|
||||
return loss
|
||||
|
||||
@staticmethod
|
||||
@custom_bwd
|
||||
def backward(ctx, output_grad):
|
||||
# Retrieve tensors from the forward path.
|
||||
softmax, target_mask, masked_target = ctx.saved_tensors
|
||||
|
||||
# All the inputs have softmax as their gradient.
|
||||
input_grad = softmax
|
||||
# For simplicity, work with the 2D gradient.
|
||||
partition_vocab_size = softmax.size()[-1]
|
||||
grad_2d = input_grad.view(-1, partition_vocab_size)
|
||||
|
||||
# Add the gradient from matching classes.
|
||||
arange_1d = torch.arange(start=0, end=grad_2d.size()[0], device=get_current_device())
|
||||
grad_2d[arange_1d, masked_target] -= (1.0 - target_mask.view(-1).float())
|
||||
input_grad.mul_(output_grad.unsqueeze(dim=-1))
|
||||
|
||||
return input_grad, None, None, None
|
||||
|
||||
|
||||
@LOSSES.register_module
|
||||
class VocabParallelCrossEntropyLoss3D(_Loss):
|
||||
"""Vocab parallel cross entropy loss for 2D parallelism.
|
||||
|
||||
Args:
|
||||
reduction (bool, optional): whether to average the loss, defaults to True.
|
||||
"""
|
||||
|
||||
def __init__(self, reduction=True):
|
||||
super().__init__()
|
||||
self.input_parallel_mode = get_parallel_mode_from_env(INPUT_GROUP_3D)
|
||||
self.weight_parallel_mode = get_parallel_mode_from_env(WEIGHT_GROUP_3D)
|
||||
self.output_parallel_mode = get_parallel_mode_from_env(OUTPUT_GROUP_3D)
|
||||
self.reduction_mean = reduction
|
||||
|
||||
def forward(self, logits, targets):
|
||||
"""Calculate loss between logits and targets.
|
||||
|
||||
Args:
|
||||
logits (:class:`torch.tensor`): Predicted unnormalized scores (often referred to as logits).
|
||||
targets (:class:`torch.tensor`): Ground truth class indices or class probabilities.
|
||||
"""
|
||||
targets = split_tensor_3d(targets, 0, self.weight_parallel_mode)
|
||||
targets = split_tensor_3d(targets, 0, self.input_parallel_mode)
|
||||
loss = _VocabParallelCrossEntropy3D.apply(logits, targets, self.output_parallel_mode)
|
||||
if self.reduction_mean:
|
||||
loss = loss.mean()
|
||||
loss = reduce_by_batch_3d(loss, self.input_parallel_mode, self.weight_parallel_mode, True)
|
||||
return loss
|
||||
Reference in New Issue
Block a user