[Kimi Perf] Group fp8 mla cahche insertion, 4~6x kernel level performance improvement for small batch (#55356)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-09-11 08:16:57 -07:00
committed by GitHub
parent 22258a26bc
commit 1e1060f998
6 changed files with 216 additions and 39 deletions
+92 -25
View File
@@ -456,14 +456,14 @@ __global__ void concat_and_cache_mla_kernel(
// Grouped variant of concat_and_cache_mla: inserts the context K/V for every
// draft layer in a single launch. Grid is (num_tokens, num_layers); each layer
// reads its own cache base pointer from kv_cache_ptrs (same pointer-array
// pattern as copy_blocks_kernel). bf16 only, so it is a raw 16-bit copy with no
// scaling or quantization; scalar_t is uint16_t for portability.
template <typename scalar_t>
// pattern as copy_blocks_kernel).
template <typename scalar_t, typename cache_t, Fp8KVCacheDataType kv_dt>
__global__ void concat_and_cache_mla_grouped_kernel(
const scalar_t* __restrict__ kv_c, // [num_layers, num_tokens,
// kv_lora_rank]
const scalar_t* __restrict__ k_pe, // [num_layers, num_tokens, pe_dim]
const int64_t* __restrict__ kv_cache_ptrs, // [num_layers]
const float* __restrict__ kv_scales, // [num_layers] or nullptr
const int64_t* __restrict__ slot_mapping, // [num_layers, num_tokens]
const int64_t kv_c_layer_stride, const int64_t kv_c_token_stride,
const int64_t k_pe_layer_stride, const int64_t k_pe_token_stride,
@@ -481,12 +481,16 @@ __global__ void concat_and_cache_mla_grouped_kernel(
const int64_t block_idx = slot_idx / block_size;
const int64_t block_offset = slot_idx % block_size;
scalar_t* __restrict__ kv_cache =
reinterpret_cast<scalar_t*>(kv_cache_ptrs[layer_idx]);
cache_t* __restrict__ kv_cache =
reinterpret_cast<cache_t*>(kv_cache_ptrs[layer_idx]);
const scalar_t* __restrict__ kv_c_layer =
kv_c + layer_idx * kv_c_layer_stride;
const scalar_t* __restrict__ k_pe_layer =
k_pe + layer_idx * k_pe_layer_stride;
float scale = 0.0f;
if constexpr (kv_dt != Fp8KVCacheDataType::kAuto) {
scale = kv_scales[layer_idx];
}
auto copy = [&](const scalar_t* __restrict__ src, int64_t src_token_stride,
int size, int offset) {
@@ -494,7 +498,12 @@ __global__ void concat_and_cache_mla_grouped_kernel(
const int64_t src_idx = token_idx * src_token_stride + i;
const int64_t dst_idx =
block_idx * block_stride + block_offset * entry_stride + i + offset;
kv_cache[dst_idx] = src[src_idx];
if constexpr (kv_dt == Fp8KVCacheDataType::kAuto) {
kv_cache[dst_idx] = src[src_idx];
} else {
kv_cache[dst_idx] =
fp8::scaled_convert<cache_t, scalar_t, kv_dt>(src[src_idx], scale);
}
}
};
@@ -1001,20 +1010,55 @@ void concat_and_cache_mla_grouped(
torch::stable::Tensor& k_pe, // [num_layers, num_tokens, pe_dim]
torch::stable::Tensor& kv_cache_ptrs, // [num_layers] int64, on device
torch::stable::Tensor& slot_mapping, // [num_layers, num_tokens] int64
int64_t block_size, int64_t block_stride, int64_t entry_stride) {
int num_layers = kv_c.size(0);
int num_tokens = kv_c.size(1);
int kv_lora_rank = kv_c.size(2);
int pe_dim = k_pe.size(2);
int64_t block_size, int64_t block_stride, int64_t entry_stride,
std::optional<torch::stable::Tensor> kv_scales, // [num_layers] or None
const std::string& kv_cache_dtype) {
const bool use_fp8 = kv_cache_dtype == "fp8" ||
kv_cache_dtype == "fp8_e4m3" ||
kv_cache_dtype == "fp8_e5m2";
#ifdef USE_ROCM
STD_TORCH_CHECK(kv_cache_dtype != "fp8_e5m2",
"concat_and_cache_mla_grouped does not support fp8_e5m2 "
"KV cache on ROCm");
#endif
STD_TORCH_CHECK(
use_fp8 || kv_cache_dtype == "auto" || kv_cache_dtype == "bfloat16",
"concat_and_cache_mla_grouped only supports BF16 and plain "
"FP8 KV cache; got ",
kv_cache_dtype);
STD_TORCH_CHECK(
kv_c.scalar_type() == torch::headeronly::ScalarType::BFloat16 &&
k_pe.scalar_type() == torch::headeronly::ScalarType::BFloat16,
"concat_and_cache_mla_grouped only supports a bf16 KV cache; got kv_c=",
"concat_and_cache_mla_grouped requires BF16 inputs; got kv_c=",
kv_c.scalar_type(), ", k_pe=", k_pe.scalar_type());
STD_TORCH_CHECK(
kv_cache_ptrs.scalar_type() == torch::headeronly::ScalarType::Long,
"kv_cache_ptrs must be int64");
kv_cache_ptrs.scalar_type() == torch::headeronly::ScalarType::Long &&
slot_mapping.scalar_type() == torch::headeronly::ScalarType::Long,
"cache pointers and slot mapping must be int64");
const int num_layers = kv_c.size(0);
const int num_tokens = kv_c.size(1);
const int kv_lora_rank = kv_c.size(2);
const int pe_dim = k_pe.size(2);
const float* kv_scales_ptr = nullptr;
if (use_fp8) {
STD_TORCH_CHECK(kv_scales.has_value(),
"FP8 grouped cache insert requires kv_scales");
STD_TORCH_CHECK(
kv_scales->scalar_type() == torch::headeronly::ScalarType::Float,
"kv_scales must be float32");
STD_TORCH_CHECK(kv_scales->numel() == num_layers,
"kv_scales must contain one scale per layer");
STD_TORCH_CHECK(kv_scales->is_cuda() && kv_scales->get_device_index() ==
kv_c.get_device_index(),
"kv_scales must be on the same CUDA device as kv_c");
STD_TORCH_CHECK(kv_scales->is_contiguous(), "kv_scales must be contiguous");
kv_scales_ptr = kv_scales->const_data_ptr<float>();
} else {
STD_TORCH_CHECK(!kv_scales.has_value(),
"BF16 grouped cache insert does not use kv_scales");
}
if (num_tokens == 0 || num_layers == 0) {
return;
@@ -1030,17 +1074,40 @@ void concat_and_cache_mla_grouped(
kv_c.get_device_index());
const cudaStream_t stream = get_current_cuda_stream();
dim3 grid(num_tokens, num_layers);
dim3 block(std::min(kv_lora_rank, 512));
vllm::concat_and_cache_mla_grouped_kernel<uint16_t>
<<<grid, block, 0, stream>>>(
reinterpret_cast<const uint16_t*>(kv_c.data_ptr()),
reinterpret_cast<const uint16_t*>(k_pe.data_ptr()),
kv_cache_ptrs.const_data_ptr<int64_t>(),
slot_mapping.const_data_ptr<int64_t>(), kv_c_layer_stride,
kv_c_token_stride, k_pe_layer_stride, k_pe_token_stride,
slot_layer_stride, block_stride, entry_stride, kv_lora_rank, pe_dim,
block_size);
const dim3 grid(num_tokens, num_layers);
const dim3 block(std::min(kv_lora_rank, 512));
if (!use_fp8) {
vllm::concat_and_cache_mla_grouped_kernel<uint16_t, uint16_t,
vllm::Fp8KVCacheDataType::kAuto>
<<<grid, block, 0, stream>>>(
reinterpret_cast<const uint16_t*>(kv_c.data_ptr()),
reinterpret_cast<const uint16_t*>(k_pe.data_ptr()),
kv_cache_ptrs.const_data_ptr<int64_t>(), nullptr,
slot_mapping.const_data_ptr<int64_t>(), kv_c_layer_stride,
kv_c_token_stride, k_pe_layer_stride, k_pe_token_stride,
slot_layer_stride, block_stride, entry_stride, kv_lora_rank, pe_dim,
block_size);
return;
}
#define LAUNCH_GROUPED_FP8(KV_DTYPE) \
vllm::concat_and_cache_mla_grouped_kernel<__nv_bfloat16, uint8_t, KV_DTYPE> \
<<<grid, block, 0, stream>>>( \
reinterpret_cast<const __nv_bfloat16*>(kv_c.data_ptr()), \
reinterpret_cast<const __nv_bfloat16*>(k_pe.data_ptr()), \
kv_cache_ptrs.const_data_ptr<int64_t>(), kv_scales_ptr, \
slot_mapping.const_data_ptr<int64_t>(), kv_c_layer_stride, \
kv_c_token_stride, k_pe_layer_stride, k_pe_token_stride, \
slot_layer_stride, block_stride, entry_stride, kv_lora_rank, pe_dim, \
block_size)
if (kv_cache_dtype == "fp8_e5m2") {
LAUNCH_GROUPED_FP8(vllm::Fp8KVCacheDataType::kFp8E5M2);
} else {
LAUNCH_GROUPED_FP8(vllm::Fp8KVCacheDataType::kFp8E4M3);
}
#undef LAUNCH_GROUPED_FP8
}
namespace vllm {
+6 -6
View File
@@ -654,12 +654,12 @@ void concat_and_cache_mla(torch::stable::Tensor& kv_c,
const std::string& kv_cache_dtype,
torch::stable::Tensor& scale);
void concat_and_cache_mla_grouped(torch::stable::Tensor& kv_c,
torch::stable::Tensor& k_pe,
torch::stable::Tensor& kv_cache_ptrs,
torch::stable::Tensor& slot_mapping,
int64_t block_size, int64_t block_stride,
int64_t entry_stride);
void concat_and_cache_mla_grouped(
torch::stable::Tensor& kv_c, torch::stable::Tensor& k_pe,
torch::stable::Tensor& kv_cache_ptrs, torch::stable::Tensor& slot_mapping,
int64_t block_size, int64_t block_stride, int64_t entry_stride,
std::optional<torch::stable::Tensor> kv_scales,
const std::string& kv_cache_dtype);
// NOTE: k_pe and kv_c order is flipped compared to concat_and_cache_mla
void concat_and_cache_mla_rope_fused(
+5 -3
View File
@@ -976,14 +976,16 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C_cache_ops, ops) {
" str kv_cache_dtype,"
" Tensor scale) -> ()");
// Grouped concat_and_cache_mla across all layers (bf16 only). Each
// layer's cache base pointer is read from kv_cache_ptrs.
// Grouped concat_and_cache_mla across all layers. Each layer's cache base
// pointer and optional plain-FP8 scale are read from device tensors.
ops.def(
"concat_and_cache_mla_grouped(Tensor kv_c, Tensor k_pe,"
" Tensor kv_cache_ptrs,"
" Tensor slot_mapping,"
" int block_size, int block_stride,"
" int entry_stride) -> ()");
" int entry_stride,"
" Tensor? kv_scales=None,"
" str kv_cache_dtype='auto') -> ()");
// Rotate Q and K, then write to kv cache for MLA
ops.def(
+88
View File
@@ -805,6 +805,94 @@ def test_concat_and_cache_mla(
torch.testing.assert_close(kv_cache, ref_kv_cache)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("shared_slot_mapping", [False, True])
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8", "fp8_e5m2"])
@torch.inference_mode()
def test_concat_and_cache_mla_grouped(
device: str,
shared_slot_mapping: bool,
kv_cache_dtype: str,
) -> None:
if kv_cache_dtype == "fp8_e5m2" and current_platform.is_rocm():
pytest.skip("fp8_e5m2 KV cache is not supported on ROCm/HIP")
set_random_seed(0)
torch.set_default_device(device)
torch.accelerator.set_device_index(device)
num_layers = 5
num_tokens = 42
num_blocks = 8
block_size = 16
kv_lora_rank = 512
qk_rope_head_dim = 64
entry_size = kv_lora_rank + qk_rope_head_dim
total_slots = num_blocks * block_size
kv_c = torch.randn(num_layers, num_tokens, kv_lora_rank, dtype=torch.bfloat16)
k_pe = torch.randn(num_layers, num_tokens, qk_rope_head_dim, dtype=torch.bfloat16)
kv_caches = torch.zeros(
num_layers,
num_blocks,
block_size,
entry_size,
dtype=torch.bfloat16 if kv_cache_dtype == "auto" else torch.uint8,
)
reference = torch.zeros_like(kv_caches)
scales = torch.linspace(0.05, 0.25, num_layers, dtype=torch.float32)
slot_mapping = torch.stack(
[torch.randperm(total_slots)[:num_tokens] for _ in range(num_layers)]
)
slot_mapping[:, -1] = -1
if shared_slot_mapping:
slot_mapping = slot_mapping[:1].expand(num_layers, -1)
for layer_idx in range(num_layers):
ops.concat_and_cache_mla(
kv_c[layer_idx],
k_pe[layer_idx],
reference[layer_idx],
slot_mapping[layer_idx],
kv_cache_dtype,
scales[layer_idx],
)
cache_ptrs = torch.tensor(
[kv_caches[layer_idx].data_ptr() for layer_idx in range(num_layers)],
dtype=torch.int64,
)
ref_cache = kv_caches[0]
def run_grouped(kv_scales: torch.Tensor | None) -> None:
ops.concat_and_cache_mla_grouped(
kv_c,
k_pe,
cache_ptrs,
slot_mapping,
ref_cache.size(1),
ref_cache.stride(0),
ref_cache.stride(1),
kv_scales,
kv_cache_dtype,
)
run_grouped(None if kv_cache_dtype == "auto" else scales)
torch.testing.assert_close(kv_caches, reference, rtol=0, atol=0)
if kv_cache_dtype == "fp8" and not shared_slot_mapping:
noncontiguous_scales = torch.ones(num_layers, 2)[:, 0]
assert not noncontiguous_scales.is_contiguous()
for invalid_scales, error in (
(scales.cpu(), "same CUDA device"),
(noncontiguous_scales, "must be contiguous"),
):
with pytest.raises(RuntimeError, match=error):
run_grouped(invalid_scales)
@pytest.mark.parametrize("kv_lora_rank", KV_LORA_RANKS)
@pytest.mark.parametrize("qk_rope_head_dim", QK_ROPE_HEAD_DIMS)
@pytest.mark.parametrize("num_tokens", NUM_TOKENS_MLA)
+4
View File
@@ -2828,6 +2828,8 @@ def concat_and_cache_mla_grouped(
block_size: int,
block_stride: int,
entry_stride: int,
kv_scales: torch.Tensor | None = None,
kv_cache_dtype: str = "auto",
) -> None:
torch.ops._C_cache_ops.concat_and_cache_mla_grouped(
kv_c,
@@ -2837,6 +2839,8 @@ def concat_and_cache_mla_grouped(
block_size,
block_stride,
entry_stride,
kv_scales,
kv_cache_dtype,
)
+21 -5
View File
@@ -28,6 +28,10 @@ from vllm.models.kimi_k3.nvidia.model import KimiMLP
from vllm.utils.torch_utils import is_quantized_kv_cache
from vllm.v1.worker.workspace import current_workspace_manager
_GROUPED_KV_CACHE_DTYPES = frozenset(
{"auto", "bfloat16", "fp8", "fp8_e4m3", "fp8_e5m2"}
)
def _duplicate_context_kv_weights(
weights: Iterable[tuple[str, torch.Tensor]], num_layers: int
@@ -233,6 +237,13 @@ class K3DSparkModel(nn.Module):
self._context_kv_lora_rank = attn0.kv_lora_rank
self._context_rope_dim = attn0.qk_rope_head_dim
self._context_rms_norm_eps = attn0.kv_a_layernorm.variance_epsilon
self._context_kv_scales: torch.Tensor | None = None
if attn0.kv_cache_dtype in _GROUPED_KV_CACHE_DTYPES and is_quantized_kv_cache(
attn0.kv_cache_dtype
):
self._context_kv_scales = torch.stack(
[attn._k_scale.reshape(()) for attn in attentions]
)
def _precompute_fused_context_kv(
self,
@@ -286,16 +297,18 @@ class K3DSparkModel(nn.Module):
return
cache_layers = [layer.self_attn for layer in self.layers]
cache_dtype = cache_layers[0].kv_cache_dtype
if (
not is_quantized_kv_cache(cache_layers[0].kv_cache_dtype)
cache_dtype in _GROUPED_KV_CACHE_DTYPES
and all(cl.kv_cache_dtype == cache_dtype for cl in cache_layers)
and self._has_uniform_block_layout(cache_layers)
and (
isinstance(context_slot_mapping, torch.Tensor)
or all(s is not None for s in context_slot_mapping)
)
):
# Grouped context KV insert only supports unquantized (bf16) KV cache
# and assumes that all layers share the same block layout.
# Grouped context KV insert assumes that all layers share the same
# cache dtype and block layout.
if isinstance(context_slot_mapping, (list, tuple)):
per_layer_slot_mappings = [
@@ -321,6 +334,8 @@ class K3DSparkModel(nn.Module):
ref_cache.size(1),
ref_cache.stride(0),
ref_cache.stride(1),
self._context_kv_scales,
cache_dtype,
)
return
@@ -483,6 +498,7 @@ class K3DSparkForCausalLM(nn.Module):
loader = AutoWeightsLoader(self)
# read: 1. all weights. 2. context kv weights
weights = _duplicate_context_kv_weights(weights, len(self.model.layers))
loaded_weights = loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def process_weights_after_loading(self) -> None:
self.model._build_fused_context_kv_metadata()
return loaded_weights