Files
fa75d3456d [Feature][PCP] Add DeepSeek-V4.1-Flash support (#59857)
Signed-off-by: wangyicong <wangyicong@bytedance.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
2026-10-15 23:17:51 -07:00

417 lines
16 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import Any, ClassVar, cast
import torch
from torch import nn
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.distributed import get_pcp_group
from vllm.forward_context import get_forward_context
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import MergedColumnParallelLinear
from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import (
fused_save_compress_norm,
rope_quant_insert,
save_ring_rows,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backend import (
AttentionBackend,
AttentionCGSupport,
AttentionMetadataBuilder,
CommonAttentionMetadata,
MultipleOf,
)
from vllm.v1.kv_cache_interface import CircularBufferSpec, KVCacheSpec
class CompressorBackend(AttentionBackend):
def __init__(self):
super().__init__()
@staticmethod
def get_name() -> str:
return "CompressorBackend"
@classmethod
def supports_pcp(cls) -> bool:
return True
@staticmethod
def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]:
return [MultipleOf(1)]
@classmethod
def get_supported_head_sizes(cls) -> list[int]:
return [512, 1024]
@staticmethod
def get_builder_cls() -> type["CompressorMetadataBuilder"]:
return CompressorMetadataBuilder
@dataclass
class CompressorMetadata:
# [num_tokens] ring slot of every token: block * capacity + pos % capacity.
slot_mapping: torch.Tensor
query_start_loc: torch.Tensor # [num_reqs + 1]
token_to_req_indices: torch.Tensor # [num_tokens]
# PCP only: every rank gathers the last row of each rank's local requests.
gather_tokens: torch.Tensor | None = None # [rows] this rank's rows
# [pcp * rows] ring slot of each request's newest row if it is open, else -1.
ring_write_slots: torch.Tensor | None = None
# [num_reqs] gathered row preceding each local chunk, or -1 to use the ring.
prev_row_indices: torch.Tensor | None = None
@triton.jit(do_not_specialize=["block_table_stride", "num_actual_tokens", "num_tokens"])
def _ring_slot_mapping_kernel(
slot_mapping_ptr,
block_table_ptr,
block_table_stride,
token_to_req_ptr,
positions_ptr,
num_actual_tokens,
num_tokens,
CAPACITY: tl.constexpr,
BLOCK: tl.constexpr,
):
offsets = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
valid = offsets < num_actual_tokens
req = tl.load(token_to_req_ptr + offsets, mask=valid, other=0).to(tl.int64)
block = tl.load(block_table_ptr + req * block_table_stride, mask=valid, other=0)
pos = tl.load(positions_ptr + offsets, mask=valid, other=0)
slot = block.to(tl.int64) * CAPACITY + pos % CAPACITY
# Block 0 is the null block. A request without a ring block (dummy or padding
# rows, which the runners fill with the null block) must not write.
tl.store(
slot_mapping_ptr + offsets,
tl.where(valid & (block != 0), slot, -1),
mask=offsets < num_tokens,
)
class CompressorMetadataBuilder(AttentionMetadataBuilder):
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
assert isinstance(self.kv_cache_spec, CircularBufferSpec)
self.capacity = self.kv_cache_spec.block_size
self.pcp_size = self.vllm_config.parallel_config.prefill_context_parallel_size
max_num_batched_tokens = (
self.vllm_config.scheduler_config.max_num_batched_tokens
)
self.token_to_req_indices = torch.zeros(
max_num_batched_tokens, dtype=torch.int32, device=self.device
)
self.slot_mapping_buffer = torch.empty(
max_num_batched_tokens, dtype=torch.int64, device=self.device
)
# The ring keeps only the open row under PCP, too few for spec decoding.
assert self.pcp_size == 1 or self.vllm_config.speculative_config is None
def build(
self,
common_prefix_len: int,
common_attn_metadata: CommonAttentionMetadata,
fast_build: bool = False,
) -> CompressorMetadata:
num_tokens = common_attn_metadata.slot_mapping.numel() // self.pcp_size
positions = common_attn_metadata.positions
assert positions is not None
token_to_req_indices = common_attn_metadata.token_to_req_indices(
self.token_to_req_indices
)
slot_mapping = self.slot_mapping_buffer[:num_tokens]
block_table = common_attn_metadata.block_table_tensor
_ring_slot_mapping_kernel[(triton.cdiv(num_tokens, 256),)](
slot_mapping,
block_table,
block_table.stride(0),
token_to_req_indices,
positions,
common_attn_metadata.num_actual_tokens,
num_tokens,
CAPACITY=self.capacity,
BLOCK=256,
)
metadata = CompressorMetadata(
slot_mapping=slot_mapping,
query_start_loc=common_attn_metadata.query_start_loc,
token_to_req_indices=token_to_req_indices,
)
if self.pcp_size > 1:
self._link_pcp_chunks(metadata, common_attn_metadata.num_reqs, positions)
return metadata
def _link_pcp_chunks(
self, metadata: CompressorMetadata, num_reqs: int, positions: torch.Tensor
) -> None:
"""Link ratio-2 groups split across PCP chunks and pick the ring writes.
Every rank gathers ``(ring block, position)`` of each local request's
last token; the ring block names a request on every rank. Sorted, they
give each local chunk's predecessor row and each request's newest row.
"""
num_tokens = metadata.slot_mapping.numel()
# Equal on every rank, so the gathers line up.
num_rows = min(num_tokens, 2 * self.vllm_config.scheduler_config.max_num_seqs)
num_reqs = min(num_reqs, num_rows)
query_start_loc = metadata.query_start_loc.long()
starts = query_start_loc[:num_reqs].clamp(max=num_tokens - 1)
last = (query_start_loc[1 : num_reqs + 1] - 1).clamp(min=0)
nonempty = query_start_loc[1 : num_reqs + 1] > query_start_loc[:num_reqs]
def key(slots: torch.Tensor, pos: torch.Tensor) -> torch.Tensor:
return torch.where(slots >= 0, (slots // self.capacity) << 32 | pos, -1)
local_keys = torch.full((num_rows,), -1, dtype=torch.int64, device=self.device)
local_keys[:num_reqs] = key(
torch.where(nonempty, metadata.slot_mapping[last], -1), positions[last]
)
sorted_keys, order = get_pcp_group().all_gather(local_keys, dim=0).sort()
blocks, sorted_positions = sorted_keys >> 32, sorted_keys & 0xFFFFFFFF
newest = torch.ones_like(sorted_keys, dtype=torch.bool)
newest[:-1] = blocks[:-1] != blocks[1:]
is_open = (sorted_keys >= 0) & newest & (sorted_positions % 2 == 0)
ring_write_slots = torch.empty_like(sorted_keys)
ring_write_slots[order] = torch.where(
is_open, blocks * self.capacity + sorted_positions % self.capacity, -1
)
first_positions = positions[starts]
wanted = key(metadata.slot_mapping[starts], first_positions - 1)
found = torch.searchsorted(sorted_keys, wanted).clamp_(max=order.numel() - 1)
has_prev = (
nonempty
& (wanted >= 0)
& (first_positions % 2 == 1)
& (sorted_keys[found] == wanted)
)
gather_tokens = torch.zeros_like(local_keys)
gather_tokens[:num_reqs] = last
metadata.gather_tokens = gather_tokens
metadata.ring_write_slots = ring_write_slots
metadata.prev_row_indices = torch.where(has_prev, order[found], -1)
class CompressorStateCache(torch.nn.Module, AttentionLayerBase):
"""Per-request ring of the open group's [kv, score] rows (ratio > 1)."""
def __init__(self, state_dim: int, dtype: torch.dtype, prefix: str):
super().__init__()
self.state_dim = state_dim
self.dtype = dtype
self.prefix = prefix
self.kv_cache = torch.tensor([])
compilation_config = get_current_vllm_config().compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
assert self.dtype == torch.float32
# Drafts + bonus token + the previous row, as a power of two >= 8.
rows_per_step = get_current_vllm_config().num_speculative_tokens + 2
self.block_size = max(8, 1 << (rows_per_step - 1).bit_length())
def bind_kv_cache(self, kv_cache: torch.Tensor) -> None:
# [B, H=1, N, C] -> [B, N, C]
self.kv_cache = kv_cache.squeeze(1)
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
del vllm_config
return CircularBufferSpec(
block_size=self.block_size,
num_kv_heads=1,
head_size=self.state_dim,
head_size_v=0,
dtype=self.dtype,
)
def forward(self): ...
def get_attn_backend(self) -> type[AttentionBackend]:
return CompressorBackend
class DeepseekCompressor(nn.Module):
"""DeepSeek V4.1 KV/score compressor.
Pools ``compress_ratio`` consecutive tokens into one KV latent with a
learned softmax gate (ratio 1 has no gate and no pooling). Owns the
linear, norm and state cache. State saving, compression and RMSNorm share
one Triton kernel. The emitted latent feeds independent main-cache and
indexer K kernels, which the attention layer can schedule concurrently.
"""
def __init__(
self,
vllm_config: VllmConfig,
compress_ratio: int,
hidden_size: int,
head_dim: int,
rotate: bool = False,
prefix: str = "",
k_cache_prefix="",
):
super().__init__()
if compress_ratio not in (1, 2):
raise NotImplementedError(
"DeepSeek V4.1 compressor supports compress_ratio 1 (full-length "
f"compressed cache) and 2; got {compress_ratio}. The ratio-4/128 "
"CuTe-DSL kernels are v4.0-specific and not wired here."
)
self.compress_ratio = compress_ratio
self.hidden_size = hidden_size
self.head_dim = head_dim
self.rotate = rotate
self.prefix = prefix
self.k_cache_prefix = k_cache_prefix
self.pcp_size = vllm_config.parallel_config.prefill_context_parallel_size
self.pcp_rank = get_pcp_group().rank_in_group if self.pcp_size > 1 else 0
# Ratio 1 pools single tokens, so the checkpoint carries no gate.
self.has_gate = compress_ratio > 1
config = vllm_config.model_config.hf_config
self.rope_head_dim = config.qk_rope_head_dim
self.nope_head_dim = self.head_dim - self.rope_head_dim
assert self.head_dim == 512 and self.rope_head_dim == 64
self.rms_norm_eps = config.rms_norm_eps
self.device = current_platform.device_type
self.max_num_reqs = vllm_config.scheduler_config.max_num_seqs
self.max_model_len = vllm_config.model_config.max_model_len
wkv_wgate_sizes = (
[self.head_dim, self.head_dim] if self.has_gate else [self.head_dim]
)
self.fused_wkv_wgate = MergedColumnParallelLinear(
self.hidden_size,
wkv_wgate_sizes,
bias=False,
return_bias=False,
quant_config=None,
disable_tp=True,
prefix=f"{prefix}.fused_wkv_wgate",
)
self.norm = RMSNorm(self.head_dim, self.rms_norm_eps)
self.state_cache = (
CompressorStateCache(
state_dim=2 * self.head_dim, # kv_state + score_state
dtype=torch.float32,
prefix=f"{prefix}.state_cache",
)
if compress_ratio > 1
else None
)
# Save reference to static_forward_context for forward-time KV cache lookup.
# get_current_vllm_config() is only available during __init__, not forward.
self._static_forward_context = (
vllm_config.compilation_config.static_forward_context
)
def forward(
self,
# [num_tokens, (2 if has_gate else 1) * self.head_dim]
kv_score: torch.Tensor,
# [num_tokens]
positions: torch.Tensor,
) -> torch.Tensor | None:
"""Save states and return the BF16 latent for cache insertion and indexing.
Only valid group-boundary rows are written. Under PCP each rank pools
its own rows; the last row of every rank's chunks is gathered, both to
close groups split across chunks and to keep every rank's ring equal.
"""
attn_metadata = get_forward_context().attn_metadata
if not isinstance(attn_metadata, dict):
return None
prev_rows = prev_row_indices = ring_write_slots = None
if self.state_cache is None:
state_cache = query_start_loc = token_to_req_indices = None
slot_mapping = cast(Any, attn_metadata[self.k_cache_prefix]).slot_mapping
if self.pcp_size > 1:
slot_mapping = slot_mapping.chunk(self.pcp_size)[self.pcp_rank]
else:
state_metadata = cast(
CompressorMetadata, attn_metadata[self.state_cache.prefix]
)
state_cache = self.state_cache.kv_cache
slot_mapping = state_metadata.slot_mapping
query_start_loc = state_metadata.query_start_loc
token_to_req_indices = state_metadata.token_to_req_indices
if self.pcp_size > 1:
assert state_metadata.gather_tokens is not None
prev_rows = get_pcp_group().all_gather(
kv_score[state_metadata.gather_tokens], dim=0
)
prev_row_indices = state_metadata.prev_row_indices
ring_write_slots = state_metadata.ring_write_slots
latent = torch.empty(
kv_score.shape[0],
self.head_dim,
dtype=torch.bfloat16,
device=kv_score.device,
)
fused_save_compress_norm(
kv_score,
positions,
state_cache,
slot_mapping,
query_start_loc,
token_to_req_indices,
self.norm.weight,
self.rms_norm_eps,
self.compress_ratio,
latent,
prev_rows=prev_rows,
prev_row_indices=prev_row_indices,
)
if prev_rows is not None:
assert state_cache is not None and ring_write_slots is not None
save_ring_rows(prev_rows, ring_write_slots, state_cache)
return latent
def insert_cache(
self,
latent: torch.Tensor | None,
positions: torch.Tensor,
rotary_emb,
) -> None:
"""Publish compressed main-cache rows after the latent becomes ready."""
if latent is None:
return
attn_metadata = get_forward_context().attn_metadata
assert isinstance(attn_metadata, dict)
k_cache_metadata = cast(Any, attn_metadata[self.k_cache_prefix])
k_cache_layer = self._static_forward_context[self.k_cache_prefix]
kv_cache = k_cache_layer.kv_cache
# Plain-row per-tensor fp8 caches (FlashInfer) carry the layer's scale;
# fp8_ds_mla and bf16 rows need none.
fp8_scale = (
getattr(k_cache_layer, "_flashinfer_fp8_kv_scale", None)
if kv_cache.dtype == torch.float8_e4m3fn
else None
)
rope_quant_insert(
latent,
positions,
rotary_emb.cos_sin_cache,
kv_cache,
k_cache_metadata.slot_mapping,
self.compress_ratio,
fp8_scale=fp8_scale,
)