diff --git a/tests/kernels/test_compressor_kv_cache.py b/tests/kernels/test_compressor_kv_cache.py index b8f1cb8bfa..57eb493e57 100644 --- a/tests/kernels/test_compressor_kv_cache.py +++ b/tests/kernels/test_compressor_kv_cache.py @@ -13,6 +13,7 @@ These tests cover: """ import math +from types import SimpleNamespace import pytest import torch @@ -27,6 +28,7 @@ from vllm.models.deepseek_v4.common.ops.fused_compress_quant_cache import ( _fused_kv_compress_norm_rope_insert_indexer_mxfp4_attn, _launch_two_stage_sparse_attn_compressor, ) +from vllm.models.deepseek_v4.compressor import _get_c128_boundary from vllm.platforms import current_platform from .test_fused_indexer_q_rope_quant import quantize_to_mxfp4 @@ -58,6 +60,25 @@ def _ue8m0_reference(x: torch.Tensor, block_size: int, fp8_max: float): return x_fp8, scales +@pytest.mark.parametrize( + ("starts", "query_start_loc", "expected"), + [ + ([0], [0, 127], False), + ([0], [0, 128], True), + ([127], [0, 1], True), + ([128], [0, 127], False), + ([1, 255], [0, 1, 2], True), + (None, [0, 1], None), + ], +) +def test_get_c128_boundary(starts, query_start_loc, expected): + metadata = SimpleNamespace( + _num_computed_tokens_cpu=None if starts is None else torch.tensor(starts), + query_start_loc_cpu=torch.tensor(query_start_loc), + ) + assert _get_c128_boundary(metadata) is expected + + # ── Test A: DeepseekV4 Attention path ────────────────────────────────────────────── diff --git a/vllm/models/deepseek_v4/compressor.py b/vllm/models/deepseek_v4/compressor.py index 13f327f6bc..590e748145 100644 --- a/vllm/models/deepseek_v4/compressor.py +++ b/vllm/models/deepseek_v4/compressor.py @@ -7,7 +7,7 @@ from typing import Any, ClassVar, cast import torch from torch import nn -from vllm.config import VllmConfig, get_current_vllm_config +from vllm.config import CUDAGraphMode, VllmConfig, get_current_vllm_config from vllm.forward_context import get_forward_context from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase from vllm.model_executor.layers.layernorm import RMSNorm @@ -42,6 +42,19 @@ def _prefer_two_stage_compressor() -> bool: return current_platform.is_rocm() +def _get_c128_boundary(metadata: CommonAttentionMetadata) -> bool | None: + starts = metadata._num_computed_tokens_cpu + if starts is None: + return None + + starts_list = starts.tolist() + query_start_loc = metadata.query_start_loc_cpu.tolist() + return any( + start % 128 + query_start_loc[i + 1] - query_start_loc[i] >= 128 + for i, start in enumerate(starts_list) + ) + + class CompressorBackend(AttentionBackend): def __init__(self): super().__init__() @@ -90,6 +103,7 @@ class CompressorMetadata: token_to_req_indices: torch.Tensor | None = None # [num_tokens] num_decode_tokens: int | None = None + c128_boundary: bool | None = None class CompressorMetadataBuilder(AttentionMetadataBuilder): @@ -127,6 +141,11 @@ class CompressorMetadataBuilder(AttentionMetadataBuilder): block_size=self.block_size, token_to_req_indices=token_to_req_indices, num_decode_tokens=num_decode_tokens, + c128_boundary=( + _get_c128_boundary(common_attn_metadata) + if self.block_size == 8 + else None + ), ) @@ -317,7 +336,8 @@ class DeepseekCompressor(nn.Module): ) # Get the metadata and handle dummy profiling run. - attn_metadata = get_forward_context().attn_metadata + forward_context = get_forward_context() + attn_metadata = forward_context.attn_metadata if not isinstance(attn_metadata, dict): return @@ -359,6 +379,16 @@ class DeepseekCompressor(nn.Module): pdl_kwargs=pdl_kwargs, ) + # full graph cannot branch on per-step CPU metadata after capture + if ( + current_platform.is_cuda() + and self.head_dim == 512 + and self.compress_ratio == 128 + and forward_context.cudagraph_runtime_mode != CUDAGraphMode.FULL + and state_metadata.c128_boundary is False + ): + return + # Fused: compress → RMSNorm → RoPE → FP8 quant → KV cache write. # RoPE requirements (kernel applies forward GPT-J style rotation): # - is_neox_style=False (interleaved pairs, NOT split-half)