diff --git a/pilot/model/loader.py b/pilot/model/loader.py
index a3d443da8..9fe6207c1 100644
--- a/pilot/model/loader.py
+++ b/pilot/model/loader.py
@@ -113,7 +113,11 @@ class ModelLoader(metaclass=Singleton):
             or self.device == "mps"
             and tokenizer
         ):
-            model.to(self.device)
+            # 4-bit not support this
+            try:
+                model.to(self.device)
+            except ValueError:
+                pass
 
         if debug:
             print(model)