[DSv4 Perf] Skip empty c128 kernel launch, around 2x kernel performance improvement. (#48957)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-07-22 10:55:20 -04:00
committed by GitHub
parent 2dc5a72e7e
commit 37e370fe93
2 changed files with 53 additions and 2 deletions
+21
View File
@@ -13,6 +13,7 @@ These tests cover:
"""
import math
from types import SimpleNamespace
import pytest
import torch
@@ -27,6 +28,7 @@ from vllm.models.deepseek_v4.common.ops.fused_compress_quant_cache import (
_fused_kv_compress_norm_rope_insert_indexer_mxfp4_attn,
_launch_two_stage_sparse_attn_compressor,
)
from vllm.models.deepseek_v4.compressor import _get_c128_boundary
from vllm.platforms import current_platform
from .test_fused_indexer_q_rope_quant import quantize_to_mxfp4
@@ -58,6 +60,25 @@ def _ue8m0_reference(x: torch.Tensor, block_size: int, fp8_max: float):
return x_fp8, scales
@pytest.mark.parametrize(
("starts", "query_start_loc", "expected"),
[
([0], [0, 127], False),
([0], [0, 128], True),
([127], [0, 1], True),
([128], [0, 127], False),
([1, 255], [0, 1, 2], True),
(None, [0, 1], None),
],
)
def test_get_c128_boundary(starts, query_start_loc, expected):
metadata = SimpleNamespace(
_num_computed_tokens_cpu=None if starts is None else torch.tensor(starts),
query_start_loc_cpu=torch.tensor(query_start_loc),
)
assert _get_c128_boundary(metadata) is expected
# ── Test A: DeepseekV4 Attention path ──────────────────────────────────────────────
+32 -2
View File
@@ -7,7 +7,7 @@ from typing import Any, ClassVar, cast
import torch
from torch import nn
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.config import CUDAGraphMode, VllmConfig, get_current_vllm_config
from vllm.forward_context import get_forward_context
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -42,6 +42,19 @@ def _prefer_two_stage_compressor() -> bool:
return current_platform.is_rocm()
def _get_c128_boundary(metadata: CommonAttentionMetadata) -> bool | None:
starts = metadata._num_computed_tokens_cpu
if starts is None:
return None
starts_list = starts.tolist()
query_start_loc = metadata.query_start_loc_cpu.tolist()
return any(
start % 128 + query_start_loc[i + 1] - query_start_loc[i] >= 128
for i, start in enumerate(starts_list)
)
class CompressorBackend(AttentionBackend):
def __init__(self):
super().__init__()
@@ -90,6 +103,7 @@ class CompressorMetadata:
token_to_req_indices: torch.Tensor | None = None # [num_tokens]
num_decode_tokens: int | None = None
c128_boundary: bool | None = None
class CompressorMetadataBuilder(AttentionMetadataBuilder):
@@ -127,6 +141,11 @@ class CompressorMetadataBuilder(AttentionMetadataBuilder):
block_size=self.block_size,
token_to_req_indices=token_to_req_indices,
num_decode_tokens=num_decode_tokens,
c128_boundary=(
_get_c128_boundary(common_attn_metadata)
if self.block_size == 8
else None
),
)
@@ -317,7 +336,8 @@ class DeepseekCompressor(nn.Module):
)
# Get the metadata and handle dummy profiling run.
attn_metadata = get_forward_context().attn_metadata
forward_context = get_forward_context()
attn_metadata = forward_context.attn_metadata
if not isinstance(attn_metadata, dict):
return
@@ -359,6 +379,16 @@ class DeepseekCompressor(nn.Module):
pdl_kwargs=pdl_kwargs,
)
# full graph cannot branch on per-step CPU metadata after capture
if (
current_platform.is_cuda()
and self.head_dim == 512
and self.compress_ratio == 128
and forward_context.cudagraph_runtime_mode != CUDAGraphMode.FULL
and state_metadata.c128_boundary is False
):
return
# Fused: compress → RMSNorm → RoPE → FP8 quant → KV cache write.
# RoPE requirements (kernel applies forward GPT-J style rotation):
# - is_neox_style=False (interleaved pairs, NOT split-half)