github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-22 18:09:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

[checkpointio] disable buffering

Browse Source
This commit is contained in:
ver217
2024-11-21 14:33:26 +08:00
parent cf519dac6a
commit 8fddbab04c
4 changed files with 11 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
colossalai/checkpoint_io/utils.py
View File

@@ -311,7 +311,7 @@ def async_save_state_dict_shards(
index_file.append_weight_map(key, shard_file)
checkpoint_file_path = os.path.join(checkpoint, shard_file)
writer = AsyncFileWriter(open(checkpoint_file_path, "wb"), n_write_entries, backend="pthread")
writer = AsyncFileWriter(open(checkpoint_file_path, "wb", buffering=0), n_write_entries, backend="pthread")
writers.append(writer)
if pinned_state_dict is not None: