From 382307a62ce361bc2bae903d0d352787033d4ee0 Mon Sep 17 00:00:00 2001 From: Tong Li Date: Thu, 22 May 2025 11:52:41 +0800 Subject: [PATCH] fix default eval setting (#6321) Co-authored-by: Tong Li --- .../coati/distributed/producer.py | 2 +- applications/ColossalChat/rl_example.py | 23 +++++++++++++++---- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/applications/ColossalChat/coati/distributed/producer.py b/applications/ColossalChat/coati/distributed/producer.py index 916ae3326..f5bdc6835 100644 --- a/applications/ColossalChat/coati/distributed/producer.py +++ b/applications/ColossalChat/coati/distributed/producer.py @@ -151,7 +151,7 @@ class BaseProducer: raise ValueError(f"Unknown evaluation function type {evaluation_function_type}") self.response_format_tags = response_format_tags else: - raise ValueError("eval_dataset_config is not defined") + print("No eval dataset provided, skip eval") self.device = get_current_device() # init backend diff --git a/applications/ColossalChat/rl_example.py b/applications/ColossalChat/rl_example.py index c08fcb2ba..eed0af362 100644 --- a/applications/ColossalChat/rl_example.py +++ b/applications/ColossalChat/rl_example.py @@ -10,7 +10,16 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("-m", "--model", type=str, default="Qwen/Qwen2.5-7B") parser.add_argument("-d", "--dataset", type=str, default="data.jsonl") - parser.add_argument("-p", "--project", type=str, default="GRPO-V3", help="Project name.") + parser.add_argument( + "-ed", + "--eval-dataset", + type=str, + default=None, + help="Evaluation dataset for each task, please use json format to specify the dataset for each task. \ + For example: {'task1':'data_eval_task1.jsonl', 'task2':'data_eval_task2.jsonl'}, the jsonl file should be in the same format as the training dataset. \ + The key is the task name, and the value is the path to the jsonl file", + ) + parser.add_argument("-p", "--project", type=str, default="GRPO", help="Project name.") parser.add_argument("-e", "--num-episodes", type=int, default=1, help="Number of episodes to train.") # Distributed training parameters @@ -301,10 +310,14 @@ if __name__ == "__main__": project_name=args.project, save_interval=args.save_interval, save_dir=os.path.join(args.save_dir, args.project.replace(" ", "_")), - eval_dataset_config={ - k: {"path": v, "max_length": args.max_prompt_tokens, "system_prompt": args.system_prompt} - for k, v in json.loads(args.eval_dataset).items() - }, + eval_dataset_config=( + { + k: {"path": v, "max_length": args.max_prompt_tokens, "system_prompt": args.system_prompt} + for k, v in json.loads(args.eval_dataset).items() + } + if args.eval_dataset + else None + ), eval_interval=args.eval_interval, eval_save_dir=os.path.join(args.eval_save_dir, args.project.replace(" ", "_")), eval_generation_config=eval_generation_config,