[DSv4] Refactor compressor & Fix ROCm compatibility (#43710)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
@@ -13,15 +13,13 @@ from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import MergedColumnParallelLinear
|
||||
from vllm.models.deepseek_v4.common.ops.fused_compress_quant_cache import (
|
||||
_compress_kv_sparse_attn_cutedsl,
|
||||
_fused_kv_compress_norm_rope_insert_indexer_attn,
|
||||
_fused_kv_compress_norm_rope_insert_indexer_mxfp4_attn,
|
||||
_fused_kv_compress_norm_rope_insert_sparse_attn_cutedsl,
|
||||
_norm_rope_insert_sparse_attn_cutedsl,
|
||||
compress_norm_rope_store_triton,
|
||||
)
|
||||
from vllm.models.deepseek_v4.common.ops.fused_indexer_q import MXFP4_BLOCK_SIZE
|
||||
from vllm.models.deepseek_v4.common.ops.save_partial_states import (
|
||||
save_partial_states,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionCGSupport,
|
||||
@@ -173,6 +171,16 @@ class CompressorStateCache(torch.nn.Module, AttentionLayerBase):
|
||||
|
||||
|
||||
class DeepseekCompressor(nn.Module):
|
||||
"""DeepSeek V4 KV/score compressor.
|
||||
|
||||
Owns the linear / norm / state-cache / ape state and the shared forward
|
||||
prologue (kv/score split, save_partial_states launch). The
|
||||
compress → norm → RoPE → store step is dispatched to a triton kernel
|
||||
(``compress_norm_rope_store_triton``) by default, except for the NVIDIA
|
||||
head_dim=128 indexer path which uses the cutedsl kernel
|
||||
(``compress_norm_rope_store_cutedsl``) for better performance.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
@@ -242,32 +250,18 @@ class DeepseekCompressor(nn.Module):
|
||||
assert not use_fp4_cache, (
|
||||
"MXFP4 cache is only supported for indexer (head=128)"
|
||||
)
|
||||
self._use_cutedsl_sparse_compressor = True
|
||||
self._use_cutedsl_fused_sparse_compressor = self.compress_ratio == 4
|
||||
self._compress_kernel = _compress_kv_sparse_attn_cutedsl
|
||||
self._norm_rope_store_kernel = _norm_rope_insert_sparse_attn_cutedsl
|
||||
self._fused_sparse_kernel = (
|
||||
_fused_kv_compress_norm_rope_insert_sparse_attn_cutedsl
|
||||
)
|
||||
self._quant_block = 64
|
||||
self._token_stride = self.nope_head_dim + self.rope_head_dim * 2
|
||||
self._scale_dim = self.nope_head_dim // 64 + 1 # 7 real + 1 pad
|
||||
self._num_warps = 4
|
||||
elif self.head_dim == 128:
|
||||
self._use_cutedsl_sparse_compressor = False
|
||||
if use_fp4_cache:
|
||||
self._fused_kernel = (
|
||||
_fused_kv_compress_norm_rope_insert_indexer_mxfp4_attn
|
||||
)
|
||||
self._quant_block = MXFP4_BLOCK_SIZE
|
||||
self._token_stride = self.head_dim // 2
|
||||
self._scale_dim = self.head_dim // MXFP4_BLOCK_SIZE
|
||||
else:
|
||||
self._fused_kernel = _fused_kv_compress_norm_rope_insert_indexer_attn
|
||||
self._quant_block = 128
|
||||
self._token_stride = self.head_dim
|
||||
self._scale_dim = 4 # single float32 scale
|
||||
self._num_warps = 1
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unsupported head_dim for fused quant+cache: {self.head_dim}"
|
||||
@@ -312,29 +306,22 @@ class DeepseekCompressor(nn.Module):
|
||||
)
|
||||
|
||||
# Store the KV and score (with fused APE addition) in the state.
|
||||
# NOTE: PDL is disabled — both this kernel and _fused_kernel below
|
||||
# depend on preceding kernel outputs (kv/score from the cublas GEMM;
|
||||
# state_cache from this kernel) but neither emits/waits on PDL grid
|
||||
# dependency primitives, so launch_pdl=True caused a read-after-write
|
||||
# race and non-deterministic output.
|
||||
_save_partial_states_kernel[(num_actual,)](
|
||||
kv,
|
||||
kv.stride(0),
|
||||
score,
|
||||
score.stride(0),
|
||||
self.ape,
|
||||
self.ape.stride(0),
|
||||
positions,
|
||||
state_cache,
|
||||
state_cache.stride(0),
|
||||
state_cache.stride(1),
|
||||
slot_mapping,
|
||||
block_size,
|
||||
HEAD_SIZE=kv.shape[-1],
|
||||
TRITON_BLOCK_SIZE=triton.next_power_of_2(kv.shape[-1]),
|
||||
STATE_WIDTH=state_width,
|
||||
COMPRESS_RATIO=self.compress_ratio,
|
||||
**pdl_kwargs,
|
||||
# NOTE: PDL is disabled — both this kernel and the compress kernels
|
||||
# below depend on preceding kernel outputs (kv/score from the cublas
|
||||
# GEMM; state_cache from this kernel) but neither emits/waits on PDL
|
||||
# grid dependency primitives, so launch_pdl=True caused a
|
||||
# read-after-write race and non-deterministic output.
|
||||
save_partial_states(
|
||||
kv=kv,
|
||||
score=score,
|
||||
ape=self.ape,
|
||||
positions=positions,
|
||||
state_cache=state_cache,
|
||||
slot_mapping=slot_mapping,
|
||||
block_size=block_size,
|
||||
state_width=state_width,
|
||||
compress_ratio=self.compress_ratio,
|
||||
pdl_kwargs=pdl_kwargs,
|
||||
)
|
||||
|
||||
# Fused: compress → RMSNorm → RoPE → FP8 quant → KV cache write.
|
||||
@@ -348,161 +335,44 @@ class DeepseekCompressor(nn.Module):
|
||||
k_cache_metadata = cast(Any, attn_metadata[self.k_cache_prefix])
|
||||
kv_cache = self._static_forward_context[self.k_cache_prefix].kv_cache
|
||||
|
||||
if self._use_cutedsl_sparse_compressor:
|
||||
if self._use_cutedsl_fused_sparse_compressor:
|
||||
self._fused_sparse_kernel(
|
||||
state_cache,
|
||||
token_to_req_indices,
|
||||
positions,
|
||||
slot_mapping,
|
||||
block_table,
|
||||
block_size,
|
||||
self.norm.weight,
|
||||
self.rms_norm_eps,
|
||||
cos_sin_cache,
|
||||
kv_cache,
|
||||
k_cache_metadata.slot_mapping,
|
||||
kv_cache.shape[1], # paged KV cache block size
|
||||
kv_cache.stride(0),
|
||||
head_size=self.head_dim,
|
||||
state_width=state_width,
|
||||
rope_head_dim=self.rope_head_dim,
|
||||
fp8_max=448.0,
|
||||
quant_block=self._quant_block,
|
||||
token_stride=self._token_stride,
|
||||
scale_dim=self._scale_dim,
|
||||
compress_ratio=self.compress_ratio,
|
||||
overlap=self.overlap,
|
||||
)
|
||||
if current_platform.is_cuda():
|
||||
# NVIDIA GPUs.
|
||||
if self.head_dim == 512:
|
||||
from .nvidia.ops import compress_norm_rope_store_cutedsl
|
||||
|
||||
# Main compressor path.
|
||||
# Use a cutedsl kernel for better performance.
|
||||
compress_norm_rope_store_fn = compress_norm_rope_store_cutedsl
|
||||
else:
|
||||
compressed_kv = torch.empty(
|
||||
(num_actual, self.head_dim),
|
||||
dtype=torch.float32,
|
||||
device=state_cache.device,
|
||||
)
|
||||
self._compress_kernel(
|
||||
state_cache,
|
||||
token_to_req_indices,
|
||||
positions,
|
||||
slot_mapping,
|
||||
block_table,
|
||||
block_size,
|
||||
compressed_kv,
|
||||
head_size=self.head_dim,
|
||||
state_width=state_width,
|
||||
compress_ratio=self.compress_ratio,
|
||||
overlap=self.overlap,
|
||||
)
|
||||
self._norm_rope_store_kernel(
|
||||
compressed_kv,
|
||||
positions,
|
||||
slot_mapping,
|
||||
self.norm.weight,
|
||||
self.rms_norm_eps,
|
||||
cos_sin_cache,
|
||||
kv_cache,
|
||||
k_cache_metadata.slot_mapping,
|
||||
kv_cache.shape[1], # paged KV cache block size
|
||||
kv_cache.stride(0),
|
||||
head_size=self.head_dim,
|
||||
rope_head_dim=self.rope_head_dim,
|
||||
fp8_max=448.0,
|
||||
quant_block=self._quant_block,
|
||||
token_stride=self._token_stride,
|
||||
scale_dim=self._scale_dim,
|
||||
compress_ratio=self.compress_ratio,
|
||||
)
|
||||
# Indexer path (head_dim == 128).
|
||||
# Use a triton kernel.
|
||||
compress_norm_rope_store_fn = compress_norm_rope_store_triton
|
||||
else:
|
||||
self._fused_kernel[(num_actual,)](
|
||||
# state cache
|
||||
state_cache,
|
||||
state_cache.stride(0),
|
||||
state_cache.stride(1),
|
||||
# metadata
|
||||
token_to_req_indices,
|
||||
positions,
|
||||
slot_mapping,
|
||||
block_table,
|
||||
block_table.stride(0),
|
||||
block_size,
|
||||
# RMSNorm
|
||||
self.norm.weight,
|
||||
self.rms_norm_eps,
|
||||
# RoPE
|
||||
cos_sin_cache,
|
||||
cos_sin_cache.stride(0),
|
||||
# KV cache
|
||||
kv_cache,
|
||||
k_cache_metadata.slot_mapping,
|
||||
kv_cache.shape[1], # paged KV cache block size (tokens per block)
|
||||
# constexprs
|
||||
HEAD_SIZE=self.head_dim,
|
||||
TRITON_BLOCK_SIZE=triton.next_power_of_2(self.head_dim),
|
||||
STATE_WIDTH=state_width,
|
||||
COMPRESS_RATIO=self.compress_ratio,
|
||||
OVERLAP=self.overlap,
|
||||
ROPE_HEAD_DIM=self.rope_head_dim,
|
||||
FP8_MAX=448.0,
|
||||
QUANT_BLOCK=self._quant_block,
|
||||
TOKEN_STRIDE=self._token_stride,
|
||||
SCALE_DIM=self._scale_dim,
|
||||
KV_BLOCK_STRIDE=kv_cache.stride(0),
|
||||
num_warps=self._num_warps,
|
||||
**pdl_kwargs,
|
||||
)
|
||||
# AMD GPUs.
|
||||
# Always use a triton kernel.
|
||||
compress_norm_rope_store_fn = compress_norm_rope_store_triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _save_partial_states_kernel(
|
||||
kv_ptr,
|
||||
kv_stride,
|
||||
score_ptr,
|
||||
score_stride,
|
||||
ape_ptr,
|
||||
ape_stride,
|
||||
positions_ptr,
|
||||
state_cache_ptr,
|
||||
state_cache_stride0,
|
||||
state_cache_stride1,
|
||||
slot_mapping_ptr,
|
||||
block_size,
|
||||
HEAD_SIZE: tl.constexpr,
|
||||
TRITON_BLOCK_SIZE: tl.constexpr,
|
||||
# state_cache last dim packs [kv_state, score_state], each STATE_WIDTH wide.
|
||||
STATE_WIDTH: tl.constexpr,
|
||||
COMPRESS_RATIO: tl.constexpr,
|
||||
):
|
||||
token_idx = tl.program_id(0)
|
||||
slot_id = tl.load(slot_mapping_ptr + token_idx)
|
||||
|
||||
# Skip padded / invalid tokens (slot_id == -1 is the PAD sentinel used
|
||||
# by vLLM). During CUDA graph replay the batch may contain padding
|
||||
# tokens whose slot_mapping is -1; writing to kv_state[-1] would be an
|
||||
# illegal memory access.
|
||||
if slot_id < 0:
|
||||
return
|
||||
|
||||
block_idx = slot_id // block_size
|
||||
pos_in_block = slot_id % block_size
|
||||
base_ptr = (
|
||||
state_cache_ptr
|
||||
+ block_idx * state_cache_stride0
|
||||
+ pos_in_block * state_cache_stride1
|
||||
)
|
||||
|
||||
block = tl.arange(0, TRITON_BLOCK_SIZE)
|
||||
mask = block < HEAD_SIZE
|
||||
|
||||
kv = tl.load(kv_ptr + token_idx * kv_stride + block, mask=mask)
|
||||
tl.store(base_ptr + block, kv, mask=mask)
|
||||
|
||||
# Fused: score += ape[position % compress_ratio]
|
||||
position = tl.load(positions_ptr + token_idx)
|
||||
ape_row = position % COMPRESS_RATIO
|
||||
ape = tl.load(ape_ptr + ape_row * ape_stride + block, mask=mask)
|
||||
score = tl.load(score_ptr + token_idx * score_stride + block, mask=mask)
|
||||
tl.store(
|
||||
base_ptr + STATE_WIDTH + block,
|
||||
score + ape,
|
||||
mask=mask,
|
||||
)
|
||||
compress_norm_rope_store_fn(
|
||||
state_cache=state_cache,
|
||||
num_actual=num_actual,
|
||||
token_to_req_indices=token_to_req_indices,
|
||||
positions=positions,
|
||||
slot_mapping=slot_mapping,
|
||||
block_table=block_table,
|
||||
block_size=block_size,
|
||||
state_width=state_width,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
kv_cache=kv_cache,
|
||||
k_cache_metadata=k_cache_metadata,
|
||||
pdl_kwargs=pdl_kwargs,
|
||||
head_dim=self.head_dim,
|
||||
rope_head_dim=self.rope_head_dim,
|
||||
compress_ratio=self.compress_ratio,
|
||||
overlap=self.overlap,
|
||||
use_fp4_cache=self.use_fp4_cache,
|
||||
rms_norm_weight=self.norm.weight,
|
||||
rms_norm_eps=self.rms_norm_eps,
|
||||
quant_block=self._quant_block,
|
||||
token_stride=self._token_stride,
|
||||
scale_dim=self._scale_dim,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user