Skip to content

Commit b242f32

Browse files
committed
make timeout optionl
Signed-off-by: Wei Du <wedu@nvidia.com>
1 parent 1c3cef9 commit b242f32

1 file changed

Lines changed: 5 additions & 1 deletion

File tree

nemo_rl/algorithms/grpo.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -706,7 +706,11 @@ def grpo_train(
706706

707707
should_save_by_step = (is_last_step or (step + 1) % master_config["checkpointing"]["save_period"] == 0)
708708
# +1 because step is 0-indexed
709-
should_save_by_timeout = timeout.check_save()
709+
# Check if timeout-based checkpointing is enabled in config.
710+
# If so, use TimeoutChecker to determine whether we should save due to timeout.Otherwise, default to False (no timeout-based saving).
711+
if 'timeout' in master_config: should_save_by_timeout = timeout.check_save()
712+
else: should_save_by_timeout = False
713+
710714

711715
if master_config["checkpointing"]["enabled"] and (should_save_by_step or should_save_by_timeout):
712716
policy.prepare_for_training()

0 commit comments

Comments
 (0)