[DSv4 Perf] Skip empty c128 kernel launch, around 2x kernel performance improvement. (#48957)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -13,6 +13,7 @@ These tests cover:
|
||||
"""
|
||||
|
||||
import math
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -27,6 +28,7 @@ from vllm.models.deepseek_v4.common.ops.fused_compress_quant_cache import (
|
||||
_fused_kv_compress_norm_rope_insert_indexer_mxfp4_attn,
|
||||
_launch_two_stage_sparse_attn_compressor,
|
||||
)
|
||||
from vllm.models.deepseek_v4.compressor import _get_c128_boundary
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .test_fused_indexer_q_rope_quant import quantize_to_mxfp4
|
||||
@@ -58,6 +60,25 @@ def _ue8m0_reference(x: torch.Tensor, block_size: int, fp8_max: float):
|
||||
return x_fp8, scales
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("starts", "query_start_loc", "expected"),
|
||||
[
|
||||
([0], [0, 127], False),
|
||||
([0], [0, 128], True),
|
||||
([127], [0, 1], True),
|
||||
([128], [0, 127], False),
|
||||
([1, 255], [0, 1, 2], True),
|
||||
(None, [0, 1], None),
|
||||
],
|
||||
)
|
||||
def test_get_c128_boundary(starts, query_start_loc, expected):
|
||||
metadata = SimpleNamespace(
|
||||
_num_computed_tokens_cpu=None if starts is None else torch.tensor(starts),
|
||||
query_start_loc_cpu=torch.tensor(query_start_loc),
|
||||
)
|
||||
assert _get_c128_boundary(metadata) is expected
|
||||
|
||||
|
||||
# ── Test A: DeepseekV4 Attention path ──────────────────────────────────────────────
|
||||
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ from typing import Any, ClassVar, cast
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.config import CUDAGraphMode, VllmConfig, get_current_vllm_config
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -42,6 +42,19 @@ def _prefer_two_stage_compressor() -> bool:
|
||||
return current_platform.is_rocm()
|
||||
|
||||
|
||||
def _get_c128_boundary(metadata: CommonAttentionMetadata) -> bool | None:
|
||||
starts = metadata._num_computed_tokens_cpu
|
||||
if starts is None:
|
||||
return None
|
||||
|
||||
starts_list = starts.tolist()
|
||||
query_start_loc = metadata.query_start_loc_cpu.tolist()
|
||||
return any(
|
||||
start % 128 + query_start_loc[i + 1] - query_start_loc[i] >= 128
|
||||
for i, start in enumerate(starts_list)
|
||||
)
|
||||
|
||||
|
||||
class CompressorBackend(AttentionBackend):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
@@ -90,6 +103,7 @@ class CompressorMetadata:
|
||||
|
||||
token_to_req_indices: torch.Tensor | None = None # [num_tokens]
|
||||
num_decode_tokens: int | None = None
|
||||
c128_boundary: bool | None = None
|
||||
|
||||
|
||||
class CompressorMetadataBuilder(AttentionMetadataBuilder):
|
||||
@@ -127,6 +141,11 @@ class CompressorMetadataBuilder(AttentionMetadataBuilder):
|
||||
block_size=self.block_size,
|
||||
token_to_req_indices=token_to_req_indices,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
c128_boundary=(
|
||||
_get_c128_boundary(common_attn_metadata)
|
||||
if self.block_size == 8
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -317,7 +336,8 @@ class DeepseekCompressor(nn.Module):
|
||||
)
|
||||
|
||||
# Get the metadata and handle dummy profiling run.
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
if not isinstance(attn_metadata, dict):
|
||||
return
|
||||
|
||||
@@ -359,6 +379,16 @@ class DeepseekCompressor(nn.Module):
|
||||
pdl_kwargs=pdl_kwargs,
|
||||
)
|
||||
|
||||
# full graph cannot branch on per-step CPU metadata after capture
|
||||
if (
|
||||
current_platform.is_cuda()
|
||||
and self.head_dim == 512
|
||||
and self.compress_ratio == 128
|
||||
and forward_context.cudagraph_runtime_mode != CUDAGraphMode.FULL
|
||||
and state_metadata.c128_boundary is False
|
||||
):
|
||||
return
|
||||
|
||||
# Fused: compress → RMSNorm → RoPE → FP8 quant → KV cache write.
|
||||
# RoPE requirements (kernel applies forward GPT-J style rotation):
|
||||
# - is_neox_style=False (interleaved pairs, NOT split-half)
|
||||
|
||||
Reference in New Issue
Block a user