From 1e1060f9988fa188fd243c093610889594ba18fd Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Fri, 11 Sep 2026 11:16:57 -0400 Subject: [PATCH] [Kimi Perf] Group fp8 mla cahche insertion, 4~6x kernel level performance improvement for small batch (#55356) Signed-off-by: yewentao256 --- csrc/libtorch_stable/cache_kernels.cu | 117 ++++++++++++++++++----- csrc/libtorch_stable/ops.h | 12 +-- csrc/libtorch_stable/torch_bindings.cpp | 8 +- tests/kernels/attention/test_cache.py | 88 +++++++++++++++++ vllm/_custom_ops.py | 4 + vllm/models/kimi_k3/nvidia/dspark_mla.py | 26 ++++- 6 files changed, 216 insertions(+), 39 deletions(-) diff --git a/csrc/libtorch_stable/cache_kernels.cu b/csrc/libtorch_stable/cache_kernels.cu index 79aa63a5b8..f42d61a4f4 100644 --- a/csrc/libtorch_stable/cache_kernels.cu +++ b/csrc/libtorch_stable/cache_kernels.cu @@ -456,14 +456,14 @@ __global__ void concat_and_cache_mla_kernel( // Grouped variant of concat_and_cache_mla: inserts the context K/V for every // draft layer in a single launch. Grid is (num_tokens, num_layers); each layer // reads its own cache base pointer from kv_cache_ptrs (same pointer-array -// pattern as copy_blocks_kernel). bf16 only, so it is a raw 16-bit copy with no -// scaling or quantization; scalar_t is uint16_t for portability. -template +// pattern as copy_blocks_kernel). +template __global__ void concat_and_cache_mla_grouped_kernel( const scalar_t* __restrict__ kv_c, // [num_layers, num_tokens, // kv_lora_rank] const scalar_t* __restrict__ k_pe, // [num_layers, num_tokens, pe_dim] const int64_t* __restrict__ kv_cache_ptrs, // [num_layers] + const float* __restrict__ kv_scales, // [num_layers] or nullptr const int64_t* __restrict__ slot_mapping, // [num_layers, num_tokens] const int64_t kv_c_layer_stride, const int64_t kv_c_token_stride, const int64_t k_pe_layer_stride, const int64_t k_pe_token_stride, @@ -481,12 +481,16 @@ __global__ void concat_and_cache_mla_grouped_kernel( const int64_t block_idx = slot_idx / block_size; const int64_t block_offset = slot_idx % block_size; - scalar_t* __restrict__ kv_cache = - reinterpret_cast(kv_cache_ptrs[layer_idx]); + cache_t* __restrict__ kv_cache = + reinterpret_cast(kv_cache_ptrs[layer_idx]); const scalar_t* __restrict__ kv_c_layer = kv_c + layer_idx * kv_c_layer_stride; const scalar_t* __restrict__ k_pe_layer = k_pe + layer_idx * k_pe_layer_stride; + float scale = 0.0f; + if constexpr (kv_dt != Fp8KVCacheDataType::kAuto) { + scale = kv_scales[layer_idx]; + } auto copy = [&](const scalar_t* __restrict__ src, int64_t src_token_stride, int size, int offset) { @@ -494,7 +498,12 @@ __global__ void concat_and_cache_mla_grouped_kernel( const int64_t src_idx = token_idx * src_token_stride + i; const int64_t dst_idx = block_idx * block_stride + block_offset * entry_stride + i + offset; - kv_cache[dst_idx] = src[src_idx]; + if constexpr (kv_dt == Fp8KVCacheDataType::kAuto) { + kv_cache[dst_idx] = src[src_idx]; + } else { + kv_cache[dst_idx] = + fp8::scaled_convert(src[src_idx], scale); + } } }; @@ -1001,20 +1010,55 @@ void concat_and_cache_mla_grouped( torch::stable::Tensor& k_pe, // [num_layers, num_tokens, pe_dim] torch::stable::Tensor& kv_cache_ptrs, // [num_layers] int64, on device torch::stable::Tensor& slot_mapping, // [num_layers, num_tokens] int64 - int64_t block_size, int64_t block_stride, int64_t entry_stride) { - int num_layers = kv_c.size(0); - int num_tokens = kv_c.size(1); - int kv_lora_rank = kv_c.size(2); - int pe_dim = k_pe.size(2); + int64_t block_size, int64_t block_stride, int64_t entry_stride, + std::optional kv_scales, // [num_layers] or None + const std::string& kv_cache_dtype) { + const bool use_fp8 = kv_cache_dtype == "fp8" || + kv_cache_dtype == "fp8_e4m3" || + kv_cache_dtype == "fp8_e5m2"; +#ifdef USE_ROCM + STD_TORCH_CHECK(kv_cache_dtype != "fp8_e5m2", + "concat_and_cache_mla_grouped does not support fp8_e5m2 " + "KV cache on ROCm"); +#endif + STD_TORCH_CHECK( + use_fp8 || kv_cache_dtype == "auto" || kv_cache_dtype == "bfloat16", + "concat_and_cache_mla_grouped only supports BF16 and plain " + "FP8 KV cache; got ", + kv_cache_dtype); STD_TORCH_CHECK( kv_c.scalar_type() == torch::headeronly::ScalarType::BFloat16 && k_pe.scalar_type() == torch::headeronly::ScalarType::BFloat16, - "concat_and_cache_mla_grouped only supports a bf16 KV cache; got kv_c=", + "concat_and_cache_mla_grouped requires BF16 inputs; got kv_c=", kv_c.scalar_type(), ", k_pe=", k_pe.scalar_type()); STD_TORCH_CHECK( - kv_cache_ptrs.scalar_type() == torch::headeronly::ScalarType::Long, - "kv_cache_ptrs must be int64"); + kv_cache_ptrs.scalar_type() == torch::headeronly::ScalarType::Long && + slot_mapping.scalar_type() == torch::headeronly::ScalarType::Long, + "cache pointers and slot mapping must be int64"); + + const int num_layers = kv_c.size(0); + const int num_tokens = kv_c.size(1); + const int kv_lora_rank = kv_c.size(2); + const int pe_dim = k_pe.size(2); + const float* kv_scales_ptr = nullptr; + if (use_fp8) { + STD_TORCH_CHECK(kv_scales.has_value(), + "FP8 grouped cache insert requires kv_scales"); + STD_TORCH_CHECK( + kv_scales->scalar_type() == torch::headeronly::ScalarType::Float, + "kv_scales must be float32"); + STD_TORCH_CHECK(kv_scales->numel() == num_layers, + "kv_scales must contain one scale per layer"); + STD_TORCH_CHECK(kv_scales->is_cuda() && kv_scales->get_device_index() == + kv_c.get_device_index(), + "kv_scales must be on the same CUDA device as kv_c"); + STD_TORCH_CHECK(kv_scales->is_contiguous(), "kv_scales must be contiguous"); + kv_scales_ptr = kv_scales->const_data_ptr(); + } else { + STD_TORCH_CHECK(!kv_scales.has_value(), + "BF16 grouped cache insert does not use kv_scales"); + } if (num_tokens == 0 || num_layers == 0) { return; @@ -1030,17 +1074,40 @@ void concat_and_cache_mla_grouped( kv_c.get_device_index()); const cudaStream_t stream = get_current_cuda_stream(); - dim3 grid(num_tokens, num_layers); - dim3 block(std::min(kv_lora_rank, 512)); - vllm::concat_and_cache_mla_grouped_kernel - <<>>( - reinterpret_cast(kv_c.data_ptr()), - reinterpret_cast(k_pe.data_ptr()), - kv_cache_ptrs.const_data_ptr(), - slot_mapping.const_data_ptr(), kv_c_layer_stride, - kv_c_token_stride, k_pe_layer_stride, k_pe_token_stride, - slot_layer_stride, block_stride, entry_stride, kv_lora_rank, pe_dim, - block_size); + const dim3 grid(num_tokens, num_layers); + const dim3 block(std::min(kv_lora_rank, 512)); + + if (!use_fp8) { + vllm::concat_and_cache_mla_grouped_kernel + <<>>( + reinterpret_cast(kv_c.data_ptr()), + reinterpret_cast(k_pe.data_ptr()), + kv_cache_ptrs.const_data_ptr(), nullptr, + slot_mapping.const_data_ptr(), kv_c_layer_stride, + kv_c_token_stride, k_pe_layer_stride, k_pe_token_stride, + slot_layer_stride, block_stride, entry_stride, kv_lora_rank, pe_dim, + block_size); + return; + } + +#define LAUNCH_GROUPED_FP8(KV_DTYPE) \ + vllm::concat_and_cache_mla_grouped_kernel<__nv_bfloat16, uint8_t, KV_DTYPE> \ + <<>>( \ + reinterpret_cast(kv_c.data_ptr()), \ + reinterpret_cast(k_pe.data_ptr()), \ + kv_cache_ptrs.const_data_ptr(), kv_scales_ptr, \ + slot_mapping.const_data_ptr(), kv_c_layer_stride, \ + kv_c_token_stride, k_pe_layer_stride, k_pe_token_stride, \ + slot_layer_stride, block_stride, entry_stride, kv_lora_rank, pe_dim, \ + block_size) + + if (kv_cache_dtype == "fp8_e5m2") { + LAUNCH_GROUPED_FP8(vllm::Fp8KVCacheDataType::kFp8E5M2); + } else { + LAUNCH_GROUPED_FP8(vllm::Fp8KVCacheDataType::kFp8E4M3); + } +#undef LAUNCH_GROUPED_FP8 } namespace vllm { diff --git a/csrc/libtorch_stable/ops.h b/csrc/libtorch_stable/ops.h index a8beef5cda..13429a54e6 100644 --- a/csrc/libtorch_stable/ops.h +++ b/csrc/libtorch_stable/ops.h @@ -654,12 +654,12 @@ void concat_and_cache_mla(torch::stable::Tensor& kv_c, const std::string& kv_cache_dtype, torch::stable::Tensor& scale); -void concat_and_cache_mla_grouped(torch::stable::Tensor& kv_c, - torch::stable::Tensor& k_pe, - torch::stable::Tensor& kv_cache_ptrs, - torch::stable::Tensor& slot_mapping, - int64_t block_size, int64_t block_stride, - int64_t entry_stride); +void concat_and_cache_mla_grouped( + torch::stable::Tensor& kv_c, torch::stable::Tensor& k_pe, + torch::stable::Tensor& kv_cache_ptrs, torch::stable::Tensor& slot_mapping, + int64_t block_size, int64_t block_stride, int64_t entry_stride, + std::optional kv_scales, + const std::string& kv_cache_dtype); // NOTE: k_pe and kv_c order is flipped compared to concat_and_cache_mla void concat_and_cache_mla_rope_fused( diff --git a/csrc/libtorch_stable/torch_bindings.cpp b/csrc/libtorch_stable/torch_bindings.cpp index 3781eeaebf..336c495113 100644 --- a/csrc/libtorch_stable/torch_bindings.cpp +++ b/csrc/libtorch_stable/torch_bindings.cpp @@ -976,14 +976,16 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C_cache_ops, ops) { " str kv_cache_dtype," " Tensor scale) -> ()"); - // Grouped concat_and_cache_mla across all layers (bf16 only). Each - // layer's cache base pointer is read from kv_cache_ptrs. + // Grouped concat_and_cache_mla across all layers. Each layer's cache base + // pointer and optional plain-FP8 scale are read from device tensors. ops.def( "concat_and_cache_mla_grouped(Tensor kv_c, Tensor k_pe," " Tensor kv_cache_ptrs," " Tensor slot_mapping," " int block_size, int block_stride," - " int entry_stride) -> ()"); + " int entry_stride," + " Tensor? kv_scales=None," + " str kv_cache_dtype='auto') -> ()"); // Rotate Q and K, then write to kv cache for MLA ops.def( diff --git a/tests/kernels/attention/test_cache.py b/tests/kernels/attention/test_cache.py index 6fabcaa789..0d5066d2f1 100644 --- a/tests/kernels/attention/test_cache.py +++ b/tests/kernels/attention/test_cache.py @@ -805,6 +805,94 @@ def test_concat_and_cache_mla( torch.testing.assert_close(kv_cache, ref_kv_cache) +@pytest.mark.parametrize("device", CUDA_DEVICES) +@pytest.mark.parametrize("shared_slot_mapping", [False, True]) +@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8", "fp8_e5m2"]) +@torch.inference_mode() +def test_concat_and_cache_mla_grouped( + device: str, + shared_slot_mapping: bool, + kv_cache_dtype: str, +) -> None: + if kv_cache_dtype == "fp8_e5m2" and current_platform.is_rocm(): + pytest.skip("fp8_e5m2 KV cache is not supported on ROCm/HIP") + + set_random_seed(0) + torch.set_default_device(device) + torch.accelerator.set_device_index(device) + + num_layers = 5 + num_tokens = 42 + num_blocks = 8 + block_size = 16 + kv_lora_rank = 512 + qk_rope_head_dim = 64 + entry_size = kv_lora_rank + qk_rope_head_dim + total_slots = num_blocks * block_size + + kv_c = torch.randn(num_layers, num_tokens, kv_lora_rank, dtype=torch.bfloat16) + k_pe = torch.randn(num_layers, num_tokens, qk_rope_head_dim, dtype=torch.bfloat16) + kv_caches = torch.zeros( + num_layers, + num_blocks, + block_size, + entry_size, + dtype=torch.bfloat16 if kv_cache_dtype == "auto" else torch.uint8, + ) + reference = torch.zeros_like(kv_caches) + scales = torch.linspace(0.05, 0.25, num_layers, dtype=torch.float32) + + slot_mapping = torch.stack( + [torch.randperm(total_slots)[:num_tokens] for _ in range(num_layers)] + ) + slot_mapping[:, -1] = -1 + if shared_slot_mapping: + slot_mapping = slot_mapping[:1].expand(num_layers, -1) + + for layer_idx in range(num_layers): + ops.concat_and_cache_mla( + kv_c[layer_idx], + k_pe[layer_idx], + reference[layer_idx], + slot_mapping[layer_idx], + kv_cache_dtype, + scales[layer_idx], + ) + + cache_ptrs = torch.tensor( + [kv_caches[layer_idx].data_ptr() for layer_idx in range(num_layers)], + dtype=torch.int64, + ) + ref_cache = kv_caches[0] + + def run_grouped(kv_scales: torch.Tensor | None) -> None: + ops.concat_and_cache_mla_grouped( + kv_c, + k_pe, + cache_ptrs, + slot_mapping, + ref_cache.size(1), + ref_cache.stride(0), + ref_cache.stride(1), + kv_scales, + kv_cache_dtype, + ) + + run_grouped(None if kv_cache_dtype == "auto" else scales) + + torch.testing.assert_close(kv_caches, reference, rtol=0, atol=0) + + if kv_cache_dtype == "fp8" and not shared_slot_mapping: + noncontiguous_scales = torch.ones(num_layers, 2)[:, 0] + assert not noncontiguous_scales.is_contiguous() + for invalid_scales, error in ( + (scales.cpu(), "same CUDA device"), + (noncontiguous_scales, "must be contiguous"), + ): + with pytest.raises(RuntimeError, match=error): + run_grouped(invalid_scales) + + @pytest.mark.parametrize("kv_lora_rank", KV_LORA_RANKS) @pytest.mark.parametrize("qk_rope_head_dim", QK_ROPE_HEAD_DIMS) @pytest.mark.parametrize("num_tokens", NUM_TOKENS_MLA) diff --git a/vllm/_custom_ops.py b/vllm/_custom_ops.py index db20adb994..c5fbb76d98 100644 --- a/vllm/_custom_ops.py +++ b/vllm/_custom_ops.py @@ -2828,6 +2828,8 @@ def concat_and_cache_mla_grouped( block_size: int, block_stride: int, entry_stride: int, + kv_scales: torch.Tensor | None = None, + kv_cache_dtype: str = "auto", ) -> None: torch.ops._C_cache_ops.concat_and_cache_mla_grouped( kv_c, @@ -2837,6 +2839,8 @@ def concat_and_cache_mla_grouped( block_size, block_stride, entry_stride, + kv_scales, + kv_cache_dtype, ) diff --git a/vllm/models/kimi_k3/nvidia/dspark_mla.py b/vllm/models/kimi_k3/nvidia/dspark_mla.py index 97febda0c8..87f9e94d9b 100644 --- a/vllm/models/kimi_k3/nvidia/dspark_mla.py +++ b/vllm/models/kimi_k3/nvidia/dspark_mla.py @@ -28,6 +28,10 @@ from vllm.models.kimi_k3.nvidia.model import KimiMLP from vllm.utils.torch_utils import is_quantized_kv_cache from vllm.v1.worker.workspace import current_workspace_manager +_GROUPED_KV_CACHE_DTYPES = frozenset( + {"auto", "bfloat16", "fp8", "fp8_e4m3", "fp8_e5m2"} +) + def _duplicate_context_kv_weights( weights: Iterable[tuple[str, torch.Tensor]], num_layers: int @@ -233,6 +237,13 @@ class K3DSparkModel(nn.Module): self._context_kv_lora_rank = attn0.kv_lora_rank self._context_rope_dim = attn0.qk_rope_head_dim self._context_rms_norm_eps = attn0.kv_a_layernorm.variance_epsilon + self._context_kv_scales: torch.Tensor | None = None + if attn0.kv_cache_dtype in _GROUPED_KV_CACHE_DTYPES and is_quantized_kv_cache( + attn0.kv_cache_dtype + ): + self._context_kv_scales = torch.stack( + [attn._k_scale.reshape(()) for attn in attentions] + ) def _precompute_fused_context_kv( self, @@ -286,16 +297,18 @@ class K3DSparkModel(nn.Module): return cache_layers = [layer.self_attn for layer in self.layers] + cache_dtype = cache_layers[0].kv_cache_dtype if ( - not is_quantized_kv_cache(cache_layers[0].kv_cache_dtype) + cache_dtype in _GROUPED_KV_CACHE_DTYPES + and all(cl.kv_cache_dtype == cache_dtype for cl in cache_layers) and self._has_uniform_block_layout(cache_layers) and ( isinstance(context_slot_mapping, torch.Tensor) or all(s is not None for s in context_slot_mapping) ) ): - # Grouped context KV insert only supports unquantized (bf16) KV cache - # and assumes that all layers share the same block layout. + # Grouped context KV insert assumes that all layers share the same + # cache dtype and block layout. if isinstance(context_slot_mapping, (list, tuple)): per_layer_slot_mappings = [ @@ -321,6 +334,8 @@ class K3DSparkModel(nn.Module): ref_cache.size(1), ref_cache.stride(0), ref_cache.stride(1), + self._context_kv_scales, + cache_dtype, ) return @@ -483,6 +498,7 @@ class K3DSparkForCausalLM(nn.Module): loader = AutoWeightsLoader(self) # read: 1. all weights. 2. context kv weights weights = _duplicate_context_kv_weights(weights, len(self.model.layers)) - loaded_weights = loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) + + def process_weights_after_loading(self) -> None: self.model._build_fused_context_kv_metadata() - return loaded_weights