mirror of
https://github.com/hpcaitech/ColossalAI.git
synced 2025-05-22 23:25:21 +00:00
09c023bee2
* [fx] modify the calculation of node_size in MetaInfoProp for activation checkpointing usages * [fx] modify the calculation of node_size in MetaInfoProp for activation checkpointing usages * [fx] modify the calculation of node_size in MetaInfoProp for activation checkpointing usages * [fx] merge development into main (#1) * [fx] activation checkpointing using Chen strategies. * [fx] add test for ckpt_solver_chen * [fx] add vanilla activation checkpoint search with test on resnet and densenet * [fx] add a namespace code for solver_chen. * [fx] fix the false interpretation of algorithm 3 in https://arxiv.org/abs/1604.06174. * [fx] fix lowercase naming conventions. * [fx] simplify test for ckpt. * [fx] add rules to linearize computation graphs for searching. (#2) * [fx] modify the calculation of node_size in MetaInfoProp for activation checkpointing usages * [fx] modify the calculation of node_size in MetaInfoProp for activation checkpointing usages * [fx] modify the calculation of node_size in MetaInfoProp for activation checkpointing usages * [fx] merge development into main (#1) * [fx] activation checkpointing using Chen strategies. * [fx] add test for ckpt_solver_chen * [fx] add vanilla activation checkpoint search with test on resnet and densenet * [fx] add a namespace code for solver_chen. * [fx] fix the false interpretation of algorithm 3 in https://arxiv.org/abs/1604.06174. * [fx] fix lowercase naming conventions. * [fx] simplify test for ckpt. * [fx] fix test and algorithm bugs in activation checkpointing. * [fx] polish ckpt_test. * [fx] add rules to linearize computation graphs for searching. * [fx] remove chen_sqrt for sake of simplicity * [fx] remove chen_sqrt for sake of simplicity * [fx] remove chen_sqrt for sake of simplicity * [fx] remove chen_sqrt for sake of simplicity * [fx] fix inconsistencies. * [fx] fix MetaInfoProp. * [fx] fix MetaInfoProp. * [fx] consider MetaInfoProp for inplace operands. * [fx] consider MetaInfoProp for inplace operands. * [fx] consider MetaInfoProp for inplace operands. * [fx] consider MetaInfoProp for inplace operands. * [fx] consider MetaInfoProp for inplace operands. * [fx] add profiler for fx nodes. * [fx] add profiler for fx nodes. * [fx] add profiler for fx nodes. * [fx] add profiler for fx nodes. * [fx] add profiler for fx nodes. * [fx] add profiler for fx nodes. * [fx] add profiler for fx nodes. * [fx] fix error in tests. * [fx] unfix bug. * [fx] unfix bug. * [fx] patch more modules and functions. * [fx] change name of utils.py to profiler.py * [fx] add profiler for rnn. * [fx] add profiler for rnn. * [fx] polish and add more patch for profiler. * [fx] polish and add more patch for profiler.
34 lines
1.3 KiB
Python
34 lines
1.3 KiB
Python
from typing import Tuple
|
|
import torch
|
|
from ..registry import meta_profiler_function
|
|
|
|
# TODO: different activation has different FLOPs count, currently unused.
|
|
_multiplier = {
|
|
torch.nn.functional.relu: 1,
|
|
torch.nn.functional.prelu: 4,
|
|
torch.nn.functional.sigmoid: 4,
|
|
torch.nn.functional.tanh: 5,
|
|
torch.nn.functional.leaky_relu: 3,
|
|
torch.nn.functional.elu: 4,
|
|
torch.nn.functional.relu6: 2,
|
|
torch.nn.functional.gelu: 9,
|
|
torch.nn.functional.hardswish: 5,
|
|
torch.nn.functional.hardsigmoid: 4,
|
|
}
|
|
|
|
|
|
@meta_profiler_function.register(torch.nn.functional.leaky_relu)
|
|
@meta_profiler_function.register(torch.nn.functional.elu)
|
|
@meta_profiler_function.register(torch.nn.functional.gelu)
|
|
@meta_profiler_function.register(torch.nn.functional.relu6)
|
|
@meta_profiler_function.register(torch.nn.functional.prelu)
|
|
@meta_profiler_function.register(torch.nn.functional.relu)
|
|
@meta_profiler_function.register(torch.nn.functional.sigmoid)
|
|
@meta_profiler_function.register(torch.nn.functional.tanh)
|
|
@meta_profiler_function.register(torch.nn.functional.hardswish)
|
|
@meta_profiler_function.register(torch.nn.functional.hardsigmoid)
|
|
def torch_nn_func_non_linear_act(input: torch.Tensor, inplace: bool = False) -> Tuple[int, int]:
|
|
flops = input.numel()
|
|
macs = 0
|
|
return flops, macs
|