[Kimi Perf] Group fp8 mla cahche insertion, 4~6x kernel level performance improvement for small batch (#55356)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -456,14 +456,14 @@ __global__ void concat_and_cache_mla_kernel(
|
||||
// Grouped variant of concat_and_cache_mla: inserts the context K/V for every
|
||||
// draft layer in a single launch. Grid is (num_tokens, num_layers); each layer
|
||||
// reads its own cache base pointer from kv_cache_ptrs (same pointer-array
|
||||
// pattern as copy_blocks_kernel). bf16 only, so it is a raw 16-bit copy with no
|
||||
// scaling or quantization; scalar_t is uint16_t for portability.
|
||||
template <typename scalar_t>
|
||||
// pattern as copy_blocks_kernel).
|
||||
template <typename scalar_t, typename cache_t, Fp8KVCacheDataType kv_dt>
|
||||
__global__ void concat_and_cache_mla_grouped_kernel(
|
||||
const scalar_t* __restrict__ kv_c, // [num_layers, num_tokens,
|
||||
// kv_lora_rank]
|
||||
const scalar_t* __restrict__ k_pe, // [num_layers, num_tokens, pe_dim]
|
||||
const int64_t* __restrict__ kv_cache_ptrs, // [num_layers]
|
||||
const float* __restrict__ kv_scales, // [num_layers] or nullptr
|
||||
const int64_t* __restrict__ slot_mapping, // [num_layers, num_tokens]
|
||||
const int64_t kv_c_layer_stride, const int64_t kv_c_token_stride,
|
||||
const int64_t k_pe_layer_stride, const int64_t k_pe_token_stride,
|
||||
@@ -481,12 +481,16 @@ __global__ void concat_and_cache_mla_grouped_kernel(
|
||||
const int64_t block_idx = slot_idx / block_size;
|
||||
const int64_t block_offset = slot_idx % block_size;
|
||||
|
||||
scalar_t* __restrict__ kv_cache =
|
||||
reinterpret_cast<scalar_t*>(kv_cache_ptrs[layer_idx]);
|
||||
cache_t* __restrict__ kv_cache =
|
||||
reinterpret_cast<cache_t*>(kv_cache_ptrs[layer_idx]);
|
||||
const scalar_t* __restrict__ kv_c_layer =
|
||||
kv_c + layer_idx * kv_c_layer_stride;
|
||||
const scalar_t* __restrict__ k_pe_layer =
|
||||
k_pe + layer_idx * k_pe_layer_stride;
|
||||
float scale = 0.0f;
|
||||
if constexpr (kv_dt != Fp8KVCacheDataType::kAuto) {
|
||||
scale = kv_scales[layer_idx];
|
||||
}
|
||||
|
||||
auto copy = [&](const scalar_t* __restrict__ src, int64_t src_token_stride,
|
||||
int size, int offset) {
|
||||
@@ -494,7 +498,12 @@ __global__ void concat_and_cache_mla_grouped_kernel(
|
||||
const int64_t src_idx = token_idx * src_token_stride + i;
|
||||
const int64_t dst_idx =
|
||||
block_idx * block_stride + block_offset * entry_stride + i + offset;
|
||||
kv_cache[dst_idx] = src[src_idx];
|
||||
if constexpr (kv_dt == Fp8KVCacheDataType::kAuto) {
|
||||
kv_cache[dst_idx] = src[src_idx];
|
||||
} else {
|
||||
kv_cache[dst_idx] =
|
||||
fp8::scaled_convert<cache_t, scalar_t, kv_dt>(src[src_idx], scale);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1001,20 +1010,55 @@ void concat_and_cache_mla_grouped(
|
||||
torch::stable::Tensor& k_pe, // [num_layers, num_tokens, pe_dim]
|
||||
torch::stable::Tensor& kv_cache_ptrs, // [num_layers] int64, on device
|
||||
torch::stable::Tensor& slot_mapping, // [num_layers, num_tokens] int64
|
||||
int64_t block_size, int64_t block_stride, int64_t entry_stride) {
|
||||
int num_layers = kv_c.size(0);
|
||||
int num_tokens = kv_c.size(1);
|
||||
int kv_lora_rank = kv_c.size(2);
|
||||
int pe_dim = k_pe.size(2);
|
||||
int64_t block_size, int64_t block_stride, int64_t entry_stride,
|
||||
std::optional<torch::stable::Tensor> kv_scales, // [num_layers] or None
|
||||
const std::string& kv_cache_dtype) {
|
||||
const bool use_fp8 = kv_cache_dtype == "fp8" ||
|
||||
kv_cache_dtype == "fp8_e4m3" ||
|
||||
kv_cache_dtype == "fp8_e5m2";
|
||||
#ifdef USE_ROCM
|
||||
STD_TORCH_CHECK(kv_cache_dtype != "fp8_e5m2",
|
||||
"concat_and_cache_mla_grouped does not support fp8_e5m2 "
|
||||
"KV cache on ROCm");
|
||||
#endif
|
||||
STD_TORCH_CHECK(
|
||||
use_fp8 || kv_cache_dtype == "auto" || kv_cache_dtype == "bfloat16",
|
||||
"concat_and_cache_mla_grouped only supports BF16 and plain "
|
||||
"FP8 KV cache; got ",
|
||||
kv_cache_dtype);
|
||||
|
||||
STD_TORCH_CHECK(
|
||||
kv_c.scalar_type() == torch::headeronly::ScalarType::BFloat16 &&
|
||||
k_pe.scalar_type() == torch::headeronly::ScalarType::BFloat16,
|
||||
"concat_and_cache_mla_grouped only supports a bf16 KV cache; got kv_c=",
|
||||
"concat_and_cache_mla_grouped requires BF16 inputs; got kv_c=",
|
||||
kv_c.scalar_type(), ", k_pe=", k_pe.scalar_type());
|
||||
STD_TORCH_CHECK(
|
||||
kv_cache_ptrs.scalar_type() == torch::headeronly::ScalarType::Long,
|
||||
"kv_cache_ptrs must be int64");
|
||||
kv_cache_ptrs.scalar_type() == torch::headeronly::ScalarType::Long &&
|
||||
slot_mapping.scalar_type() == torch::headeronly::ScalarType::Long,
|
||||
"cache pointers and slot mapping must be int64");
|
||||
|
||||
const int num_layers = kv_c.size(0);
|
||||
const int num_tokens = kv_c.size(1);
|
||||
const int kv_lora_rank = kv_c.size(2);
|
||||
const int pe_dim = k_pe.size(2);
|
||||
const float* kv_scales_ptr = nullptr;
|
||||
if (use_fp8) {
|
||||
STD_TORCH_CHECK(kv_scales.has_value(),
|
||||
"FP8 grouped cache insert requires kv_scales");
|
||||
STD_TORCH_CHECK(
|
||||
kv_scales->scalar_type() == torch::headeronly::ScalarType::Float,
|
||||
"kv_scales must be float32");
|
||||
STD_TORCH_CHECK(kv_scales->numel() == num_layers,
|
||||
"kv_scales must contain one scale per layer");
|
||||
STD_TORCH_CHECK(kv_scales->is_cuda() && kv_scales->get_device_index() ==
|
||||
kv_c.get_device_index(),
|
||||
"kv_scales must be on the same CUDA device as kv_c");
|
||||
STD_TORCH_CHECK(kv_scales->is_contiguous(), "kv_scales must be contiguous");
|
||||
kv_scales_ptr = kv_scales->const_data_ptr<float>();
|
||||
} else {
|
||||
STD_TORCH_CHECK(!kv_scales.has_value(),
|
||||
"BF16 grouped cache insert does not use kv_scales");
|
||||
}
|
||||
|
||||
if (num_tokens == 0 || num_layers == 0) {
|
||||
return;
|
||||
@@ -1030,17 +1074,40 @@ void concat_and_cache_mla_grouped(
|
||||
kv_c.get_device_index());
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
|
||||
dim3 grid(num_tokens, num_layers);
|
||||
dim3 block(std::min(kv_lora_rank, 512));
|
||||
vllm::concat_and_cache_mla_grouped_kernel<uint16_t>
|
||||
<<<grid, block, 0, stream>>>(
|
||||
reinterpret_cast<const uint16_t*>(kv_c.data_ptr()),
|
||||
reinterpret_cast<const uint16_t*>(k_pe.data_ptr()),
|
||||
kv_cache_ptrs.const_data_ptr<int64_t>(),
|
||||
slot_mapping.const_data_ptr<int64_t>(), kv_c_layer_stride,
|
||||
kv_c_token_stride, k_pe_layer_stride, k_pe_token_stride,
|
||||
slot_layer_stride, block_stride, entry_stride, kv_lora_rank, pe_dim,
|
||||
block_size);
|
||||
const dim3 grid(num_tokens, num_layers);
|
||||
const dim3 block(std::min(kv_lora_rank, 512));
|
||||
|
||||
if (!use_fp8) {
|
||||
vllm::concat_and_cache_mla_grouped_kernel<uint16_t, uint16_t,
|
||||
vllm::Fp8KVCacheDataType::kAuto>
|
||||
<<<grid, block, 0, stream>>>(
|
||||
reinterpret_cast<const uint16_t*>(kv_c.data_ptr()),
|
||||
reinterpret_cast<const uint16_t*>(k_pe.data_ptr()),
|
||||
kv_cache_ptrs.const_data_ptr<int64_t>(), nullptr,
|
||||
slot_mapping.const_data_ptr<int64_t>(), kv_c_layer_stride,
|
||||
kv_c_token_stride, k_pe_layer_stride, k_pe_token_stride,
|
||||
slot_layer_stride, block_stride, entry_stride, kv_lora_rank, pe_dim,
|
||||
block_size);
|
||||
return;
|
||||
}
|
||||
|
||||
#define LAUNCH_GROUPED_FP8(KV_DTYPE) \
|
||||
vllm::concat_and_cache_mla_grouped_kernel<__nv_bfloat16, uint8_t, KV_DTYPE> \
|
||||
<<<grid, block, 0, stream>>>( \
|
||||
reinterpret_cast<const __nv_bfloat16*>(kv_c.data_ptr()), \
|
||||
reinterpret_cast<const __nv_bfloat16*>(k_pe.data_ptr()), \
|
||||
kv_cache_ptrs.const_data_ptr<int64_t>(), kv_scales_ptr, \
|
||||
slot_mapping.const_data_ptr<int64_t>(), kv_c_layer_stride, \
|
||||
kv_c_token_stride, k_pe_layer_stride, k_pe_token_stride, \
|
||||
slot_layer_stride, block_stride, entry_stride, kv_lora_rank, pe_dim, \
|
||||
block_size)
|
||||
|
||||
if (kv_cache_dtype == "fp8_e5m2") {
|
||||
LAUNCH_GROUPED_FP8(vllm::Fp8KVCacheDataType::kFp8E5M2);
|
||||
} else {
|
||||
LAUNCH_GROUPED_FP8(vllm::Fp8KVCacheDataType::kFp8E4M3);
|
||||
}
|
||||
#undef LAUNCH_GROUPED_FP8
|
||||
}
|
||||
|
||||
namespace vllm {
|
||||
|
||||
@@ -654,12 +654,12 @@ void concat_and_cache_mla(torch::stable::Tensor& kv_c,
|
||||
const std::string& kv_cache_dtype,
|
||||
torch::stable::Tensor& scale);
|
||||
|
||||
void concat_and_cache_mla_grouped(torch::stable::Tensor& kv_c,
|
||||
torch::stable::Tensor& k_pe,
|
||||
torch::stable::Tensor& kv_cache_ptrs,
|
||||
torch::stable::Tensor& slot_mapping,
|
||||
int64_t block_size, int64_t block_stride,
|
||||
int64_t entry_stride);
|
||||
void concat_and_cache_mla_grouped(
|
||||
torch::stable::Tensor& kv_c, torch::stable::Tensor& k_pe,
|
||||
torch::stable::Tensor& kv_cache_ptrs, torch::stable::Tensor& slot_mapping,
|
||||
int64_t block_size, int64_t block_stride, int64_t entry_stride,
|
||||
std::optional<torch::stable::Tensor> kv_scales,
|
||||
const std::string& kv_cache_dtype);
|
||||
|
||||
// NOTE: k_pe and kv_c order is flipped compared to concat_and_cache_mla
|
||||
void concat_and_cache_mla_rope_fused(
|
||||
|
||||
@@ -976,14 +976,16 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C_cache_ops, ops) {
|
||||
" str kv_cache_dtype,"
|
||||
" Tensor scale) -> ()");
|
||||
|
||||
// Grouped concat_and_cache_mla across all layers (bf16 only). Each
|
||||
// layer's cache base pointer is read from kv_cache_ptrs.
|
||||
// Grouped concat_and_cache_mla across all layers. Each layer's cache base
|
||||
// pointer and optional plain-FP8 scale are read from device tensors.
|
||||
ops.def(
|
||||
"concat_and_cache_mla_grouped(Tensor kv_c, Tensor k_pe,"
|
||||
" Tensor kv_cache_ptrs,"
|
||||
" Tensor slot_mapping,"
|
||||
" int block_size, int block_stride,"
|
||||
" int entry_stride) -> ()");
|
||||
" int entry_stride,"
|
||||
" Tensor? kv_scales=None,"
|
||||
" str kv_cache_dtype='auto') -> ()");
|
||||
|
||||
// Rotate Q and K, then write to kv cache for MLA
|
||||
ops.def(
|
||||
|
||||
@@ -805,6 +805,94 @@ def test_concat_and_cache_mla(
|
||||
torch.testing.assert_close(kv_cache, ref_kv_cache)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("shared_slot_mapping", [False, True])
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8", "fp8_e5m2"])
|
||||
@torch.inference_mode()
|
||||
def test_concat_and_cache_mla_grouped(
|
||||
device: str,
|
||||
shared_slot_mapping: bool,
|
||||
kv_cache_dtype: str,
|
||||
) -> None:
|
||||
if kv_cache_dtype == "fp8_e5m2" and current_platform.is_rocm():
|
||||
pytest.skip("fp8_e5m2 KV cache is not supported on ROCm/HIP")
|
||||
|
||||
set_random_seed(0)
|
||||
torch.set_default_device(device)
|
||||
torch.accelerator.set_device_index(device)
|
||||
|
||||
num_layers = 5
|
||||
num_tokens = 42
|
||||
num_blocks = 8
|
||||
block_size = 16
|
||||
kv_lora_rank = 512
|
||||
qk_rope_head_dim = 64
|
||||
entry_size = kv_lora_rank + qk_rope_head_dim
|
||||
total_slots = num_blocks * block_size
|
||||
|
||||
kv_c = torch.randn(num_layers, num_tokens, kv_lora_rank, dtype=torch.bfloat16)
|
||||
k_pe = torch.randn(num_layers, num_tokens, qk_rope_head_dim, dtype=torch.bfloat16)
|
||||
kv_caches = torch.zeros(
|
||||
num_layers,
|
||||
num_blocks,
|
||||
block_size,
|
||||
entry_size,
|
||||
dtype=torch.bfloat16 if kv_cache_dtype == "auto" else torch.uint8,
|
||||
)
|
||||
reference = torch.zeros_like(kv_caches)
|
||||
scales = torch.linspace(0.05, 0.25, num_layers, dtype=torch.float32)
|
||||
|
||||
slot_mapping = torch.stack(
|
||||
[torch.randperm(total_slots)[:num_tokens] for _ in range(num_layers)]
|
||||
)
|
||||
slot_mapping[:, -1] = -1
|
||||
if shared_slot_mapping:
|
||||
slot_mapping = slot_mapping[:1].expand(num_layers, -1)
|
||||
|
||||
for layer_idx in range(num_layers):
|
||||
ops.concat_and_cache_mla(
|
||||
kv_c[layer_idx],
|
||||
k_pe[layer_idx],
|
||||
reference[layer_idx],
|
||||
slot_mapping[layer_idx],
|
||||
kv_cache_dtype,
|
||||
scales[layer_idx],
|
||||
)
|
||||
|
||||
cache_ptrs = torch.tensor(
|
||||
[kv_caches[layer_idx].data_ptr() for layer_idx in range(num_layers)],
|
||||
dtype=torch.int64,
|
||||
)
|
||||
ref_cache = kv_caches[0]
|
||||
|
||||
def run_grouped(kv_scales: torch.Tensor | None) -> None:
|
||||
ops.concat_and_cache_mla_grouped(
|
||||
kv_c,
|
||||
k_pe,
|
||||
cache_ptrs,
|
||||
slot_mapping,
|
||||
ref_cache.size(1),
|
||||
ref_cache.stride(0),
|
||||
ref_cache.stride(1),
|
||||
kv_scales,
|
||||
kv_cache_dtype,
|
||||
)
|
||||
|
||||
run_grouped(None if kv_cache_dtype == "auto" else scales)
|
||||
|
||||
torch.testing.assert_close(kv_caches, reference, rtol=0, atol=0)
|
||||
|
||||
if kv_cache_dtype == "fp8" and not shared_slot_mapping:
|
||||
noncontiguous_scales = torch.ones(num_layers, 2)[:, 0]
|
||||
assert not noncontiguous_scales.is_contiguous()
|
||||
for invalid_scales, error in (
|
||||
(scales.cpu(), "same CUDA device"),
|
||||
(noncontiguous_scales, "must be contiguous"),
|
||||
):
|
||||
with pytest.raises(RuntimeError, match=error):
|
||||
run_grouped(invalid_scales)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_lora_rank", KV_LORA_RANKS)
|
||||
@pytest.mark.parametrize("qk_rope_head_dim", QK_ROPE_HEAD_DIMS)
|
||||
@pytest.mark.parametrize("num_tokens", NUM_TOKENS_MLA)
|
||||
|
||||
@@ -2828,6 +2828,8 @@ def concat_and_cache_mla_grouped(
|
||||
block_size: int,
|
||||
block_stride: int,
|
||||
entry_stride: int,
|
||||
kv_scales: torch.Tensor | None = None,
|
||||
kv_cache_dtype: str = "auto",
|
||||
) -> None:
|
||||
torch.ops._C_cache_ops.concat_and_cache_mla_grouped(
|
||||
kv_c,
|
||||
@@ -2837,6 +2839,8 @@ def concat_and_cache_mla_grouped(
|
||||
block_size,
|
||||
block_stride,
|
||||
entry_stride,
|
||||
kv_scales,
|
||||
kv_cache_dtype,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -28,6 +28,10 @@ from vllm.models.kimi_k3.nvidia.model import KimiMLP
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
_GROUPED_KV_CACHE_DTYPES = frozenset(
|
||||
{"auto", "bfloat16", "fp8", "fp8_e4m3", "fp8_e5m2"}
|
||||
)
|
||||
|
||||
|
||||
def _duplicate_context_kv_weights(
|
||||
weights: Iterable[tuple[str, torch.Tensor]], num_layers: int
|
||||
@@ -233,6 +237,13 @@ class K3DSparkModel(nn.Module):
|
||||
self._context_kv_lora_rank = attn0.kv_lora_rank
|
||||
self._context_rope_dim = attn0.qk_rope_head_dim
|
||||
self._context_rms_norm_eps = attn0.kv_a_layernorm.variance_epsilon
|
||||
self._context_kv_scales: torch.Tensor | None = None
|
||||
if attn0.kv_cache_dtype in _GROUPED_KV_CACHE_DTYPES and is_quantized_kv_cache(
|
||||
attn0.kv_cache_dtype
|
||||
):
|
||||
self._context_kv_scales = torch.stack(
|
||||
[attn._k_scale.reshape(()) for attn in attentions]
|
||||
)
|
||||
|
||||
def _precompute_fused_context_kv(
|
||||
self,
|
||||
@@ -286,16 +297,18 @@ class K3DSparkModel(nn.Module):
|
||||
return
|
||||
|
||||
cache_layers = [layer.self_attn for layer in self.layers]
|
||||
cache_dtype = cache_layers[0].kv_cache_dtype
|
||||
if (
|
||||
not is_quantized_kv_cache(cache_layers[0].kv_cache_dtype)
|
||||
cache_dtype in _GROUPED_KV_CACHE_DTYPES
|
||||
and all(cl.kv_cache_dtype == cache_dtype for cl in cache_layers)
|
||||
and self._has_uniform_block_layout(cache_layers)
|
||||
and (
|
||||
isinstance(context_slot_mapping, torch.Tensor)
|
||||
or all(s is not None for s in context_slot_mapping)
|
||||
)
|
||||
):
|
||||
# Grouped context KV insert only supports unquantized (bf16) KV cache
|
||||
# and assumes that all layers share the same block layout.
|
||||
# Grouped context KV insert assumes that all layers share the same
|
||||
# cache dtype and block layout.
|
||||
|
||||
if isinstance(context_slot_mapping, (list, tuple)):
|
||||
per_layer_slot_mappings = [
|
||||
@@ -321,6 +334,8 @@ class K3DSparkModel(nn.Module):
|
||||
ref_cache.size(1),
|
||||
ref_cache.stride(0),
|
||||
ref_cache.stride(1),
|
||||
self._context_kv_scales,
|
||||
cache_dtype,
|
||||
)
|
||||
return
|
||||
|
||||
@@ -483,6 +498,7 @@ class K3DSparkForCausalLM(nn.Module):
|
||||
loader = AutoWeightsLoader(self)
|
||||
# read: 1. all weights. 2. context kv weights
|
||||
weights = _duplicate_context_kv_weights(weights, len(self.model.layers))
|
||||
loaded_weights = loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
def process_weights_after_loading(self) -> None:
|
||||
self.model._build_fused_context_kv_metadata()
|
||||
return loaded_weights
|
||||
|
||||
Reference in New Issue
Block a user