[inference] refactored config (#5376)

2025-09-09 21:09:18 +00:00 · 2024-02-08 14:04:14 +08:00
parent 1f8c7e7046
commit 9afa52061f
2 changed files with 32 additions and 22 deletions
--- a/colossalai/inference/core/engine.py
+++ b/colossalai/inference/core/engine.py
@@ -130,7 +130,6 @@ class InferenceEngine:
            enable_flash_attention=False,
            enable_jit_fused=False,
            enable_sequence_parallelism=False,
-            extra_kwargs={"quant": self.inference_config.quant_mode},
        )
        shardformer = ShardFormer(shard_config=shardconfig)
        shard_model, _ = shardformer.optimize(model, model_policy)