diff --git a/configs/deepspeed/ds_config_gptj.json b/configs/deepspeed/ds_config_gptj.json
index 6f9b2961..e6ec172d 100644
--- a/configs/deepspeed/ds_config_gptj.json
+++ b/configs/deepspeed/ds_config_gptj.json
@@ -35,14 +35,5 @@
 		  ],
 		  "eps": 1e-08
 		}
-	  },
-	  "scheduler": {
-		"type": "WarmupLR",
-		"params": {
-		  "warmup_min_lr": 0,
-		  "warmup_max_lr": "auto",
-		  "warmup_num_steps": "auto",
-		  "warmup_type": "linear"
-		}
 	  }
 }
\ No newline at end of file