mirror of
https://github.com/hpcaitech/ColossalAI.git
synced 2026-01-29 21:49:54 +00:00
[doc] Fix typo under colossalai and doc(#3618)
* Fixed several spelling errors under colossalai * Fix the spelling error in colossalai and docs directory * Cautious Changed the spelling error under the example folder * Update runtime_preparation_pass.py revert autograft to autograd * Update search_chunk.py utile to until * Update check_installation.py change misteach to mismatch in line 91 * Update 1D_tensor_parallel.md revert to perceptron * Update 2D_tensor_parallel.md revert to perceptron in line 73 * Update 2p5D_tensor_parallel.md revert to perceptron in line 71 * Update 3D_tensor_parallel.md revert to perceptron in line 80 * Update README.md revert to resnet in line 42 * Update reorder_graph.py revert to indice in line 7 * Update p2p.py revert to megatron in line 94 * Update initialize.py revert to torchrun in line 198 * Update routers.py change to detailed in line 63 * Update routers.py change to detailed in line 146 * Update README.md revert random number in line 402
This commit is contained in:
@@ -74,7 +74,7 @@ class ColoInitContext(InsertPostInitMethodToModuleSubClasses):
|
||||
"""
|
||||
Args:
|
||||
device (torch.device): the device where parameters initialized are resident. Defaults to torch.device('cpu').
|
||||
dtype (torch.dtype): the dtype of parameters initialized. Defults to torch.float.
|
||||
dtype (torch.dtype): the dtype of parameters initialized. Defaults to torch.float.
|
||||
default_pg (ProcessGroup): the default process group for all initialized parameters.
|
||||
default_dist_spec: the default distributed specifications.
|
||||
"""
|
||||
@@ -164,7 +164,7 @@ def post_process_colo_init_ctx(model: torch.nn.Module,
|
||||
model (torch.nn.module): the model
|
||||
device (torch.device, optional): device type of the model params. Defaults to torch.device('cpu').
|
||||
dtype (torch.dtype, optional): dtype of the model params. Defaults to torch.float.
|
||||
default_pg (Optional[ProcessGroup], optional): default process group. Defaults to None. Inidicates a DP-only process group.
|
||||
default_pg (Optional[ProcessGroup], optional): default process group. Defaults to None. Indicates a DP-only process group.
|
||||
default_dist_spec (Any, optional): default dist spec of params. Defaults to None.
|
||||
|
||||
Raises:
|
||||
|
||||
@@ -42,7 +42,7 @@ class ZeroDDP(ColoDDP):
|
||||
|
||||
Args:
|
||||
module (torch.nn.Module): Module to apply ZeRO-DP.
|
||||
gemini_manager (GeminiManager): Manages the chunk manager and heterogeneous momery space.
|
||||
gemini_manager (GeminiManager): Manages the chunk manager and heterogeneous memory space.
|
||||
For more details, see the API reference of ``GeminiManager``.
|
||||
pin_memory (bool): Chunks on CPU Memory use pin-memory.
|
||||
force_outputs_fp32 (bool): If set to True, outputs will be fp32. Otherwise, outputs will be fp16.
|
||||
@@ -684,7 +684,7 @@ class GeminiDDP(ZeroDDP):
|
||||
memstats: Optional[MemStats] = None,
|
||||
verbose: bool = False) -> None:
|
||||
"""
|
||||
A torch.Module warpper using ZeRO-DP and Genimi.
|
||||
A torch.Module wrapper using ZeRO-DP and Gemini.
|
||||
ZeRO is for parallel. Gemini is for memory management.
|
||||
WARNING: The class will modify the module inline!
|
||||
|
||||
@@ -706,7 +706,7 @@ class GeminiDDP(ZeroDDP):
|
||||
Users can provide this argument to speed up searching.
|
||||
If users do not know this argument before training, it is ok. We will use a default value 1024.
|
||||
min_chunk_size_mb (float, optional): the minimum chunk size in MegaByte.
|
||||
If the aggregate size of parameters is still samller than the minimum chunk size,
|
||||
If the aggregate size of parameters is still smaller than the minimum chunk size,
|
||||
all parameters will be compacted into one small chunk.
|
||||
memstats (MemStats, optional) the memory statistics collector by a runtime memory tracer.
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user