move logging to producer

2025-09-06 11:32:10 +00:00 · 2025-05-14 18:10:57 +08:00
parent 47a7dc7142
commit 50070c1e84
7 changed files with 92 additions and 70 deletions
--- a/applications/ColossalChat/coati/distributed/utils.py
+++ b/applications/ColossalChat/coati/distributed/utils.py
@@ -135,7 +135,7 @@ def masked_sum(tensor: torch.Tensor, mask: torch.Tensor, dim: int = 1) -> torch.
    return tensor.sum(dim=dim)


-def safe_write_jsonl(file_path, data):
+def safe_append_to_jsonl_file(file_path, data):
    with FileLock(file_path + ".lock"):
        # Ensure file exists
        os.makedirs(os.path.dirname(file_path), exist_ok=True)