diff --git a/.gitmodules b/.gitmodules
index 544a1371..371af62e 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,6 +1,3 @@
-[submodule "transformers"]
-	path = transformers
-	url = https://github.com/huggingface/transformers.git
 [submodule "peft"]
 	path = peft
 	url = https://github.com/huggingface/peft.git
diff --git a/requirements.txt b/requirements.txt
index f43f8b7e..b38ab36c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,7 +2,7 @@ accelerate
 datasets
 torchmetrics
 evaluate
-transformers
+transformers>=4.28.0
 wandb
 pip
 peft
diff --git a/transformers b/transformers
deleted file mode 160000
index cae78c46..00000000
--- a/transformers
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit cae78c46d658a8e496a815c2ee49b9b178fb9c9a