[Inference/Feat] Add kvcache quantization support for FlashDecoding (#5656)

2025-09-11 22:10:37 +00:00 · 2024-04-26 19:40:37 +08:00
parent 5be590b99e
commit 8ccb6714e7
5 changed files with 482 additions and 174 deletions
--- a/extensions/csrc/kernel/cuda/context_kv_cache_memcpy_kernel.cu
+++ b/extensions/csrc/kernel/cuda/context_kv_cache_memcpy_kernel.cu
@@ -174,13 +174,13 @@ void context_kv_cache_memcpy(
        key.scalar_type(),
        "context_kv_cache_memcpy",
        apply_context_kv_cache_memcpy<scalar_t>(
-            key,
-            value,
-            key_cache,
-            value_cache,
-            sequence_lengths,
-            cu_seqlens,
-            block_tables,
-            max_seq_len_in_batch
-        );)
+    key,
+    value,
+    key_cache,
+    value_cache,
+    sequence_lengths,
+    cu_seqlens,
+    block_tables,
+    max_seq_len_in_batch
+    );)
 }