Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-16 17:35:42 +07:00
347 changed files with 29531 additions and 3472 deletions
+10
View File
@@ -950,6 +950,16 @@ class AttentionImpl(AttentionImplBase[T], Generic[T]):
"""Whether this implementation supports fused QKNorm+MRoPE+KVCache."""
return False
def set_interleaved_v_cache(self):
"""Opt this implementation into the interleaved V-cache layout for all of
its cache writers and readers. QkNormRopeKvCacheFusionPass calls this
for every eligible layer when it is constructed, i.e. before any KV
cache is written, so fused and unfused paths stay consistent whether or
not the pattern matches. Default is a no-op; backends whose kernels
read an interleaved V-cache under this fusion (e.g. ROCM_ATTN) override.
"""
return False
def fused_rope_kvcache_supported(self):
"""Does this attention implementation support RoPE+KVCache fusion.
This is used by the RopeKVCacheFusionPass to only fuse the RoPE ops
@@ -35,6 +35,7 @@ from vllm.v1.attention.backends.mla.sparse_utils import (
flat_kv_row_view,
prepare_sparse_mla_safe_lengths,
)
from vllm.v1.attention.ops.metadata import compute_token_to_req_indices
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheSpec
if TYPE_CHECKING:
@@ -297,6 +298,11 @@ class FlashInferMLASparseTRTLLMMetadataBuilder(FlashInferMLASparseMetadataBuilde
device: torch.device,
) -> None:
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
self.supports_draft_decode_metadata_update = (
self.dcp_world_size == 1
and self.pcp_world_size == 1
and vllm_config.attention_config.hisparse_config is None
)
# Under DCP the workspace must hold the trtllm-gen softmax-stats slab.
# The buffer address is baked into CUDA graphs, so it has to reach its
@@ -314,6 +320,18 @@ class FlashInferMLASparseTRTLLMMetadataBuilder(FlashInferMLASparseMetadataBuilde
),
)
def update_draft_decode_metadata(
self, metadata: FlashInferMLASparseMetadata
) -> None:
num_tokens = metadata.num_decode_tokens
if num_tokens:
compute_token_to_req_indices(
metadata.query_start_loc,
metadata.req_id_per_token,
num_tokens,
num_tokens,
)
# Global workspace buffer (lazily initialized)
_fi_sparse_workspace: torch.Tensor | None = None
@@ -103,10 +103,11 @@ For DeepSeek V4.1, each token's KV cache is 528 Bytes, structured as:
- **Last 16 bytes:** Scale factors, containing 16 `ue8m0` values, one per 32
consecutive `float8_e4m3` values (i.e. MXFP8).
The V4 and V4.1 records are not laid out token-by-token within a page: a page
holds all its data rows first and all its scale rows after, so a page is
`block_size * bytes_per_token` bytes rounded up to the decode kernel's TMA
stride (576 B for V4, 512 B for V4.1).
The V4 record is not laid out token-by-token within a page: a page holds all
its data rows first and all its scale rows after, so a page is
`block_size * bytes_per_token` bytes rounded up to the decode kernel's 576 B
TMA stride. The V4.1 records are interleaved: each token's scales directly
follow its own data, and the decode kernel's TMA stride is the record width.
In the "nvfp4_ds_mla" format (SM100 only, DeepSeek V3.2 geometry), each
token's KV cache is 352 Bytes, structured as:
+3 -5
View File
@@ -205,19 +205,16 @@ class PrepareUniformDecodeKernel(
)
# Below this many rows per rank, stay replicated (TP4 engages at 64K rows).
MIN_TP_SHARD_ROWS_PER_RANK = 16_384
def balanced_prefill_row_shard(
seq_lens_cpu: torch.Tensor,
query_lens_cpu: torch.Tensor,
compress_ratio: int,
tp_size: int,
min_rows: int,
) -> list[int] | None:
"""Return contiguous TP row counts balanced by indexer MQA work."""
num_rows = int(query_lens_cpu.sum())
if tp_size < 2 or num_rows < MIN_TP_SHARD_ROWS_PER_RANK * tp_size:
if tp_size < 2 or num_rows < max(min_rows, tp_size):
return None
query_lens = query_lens_cpu.to(torch.int64)
@@ -1584,6 +1581,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
prefill_query_lens_cpu,
self.compress_ratio,
self.tp_size,
self.vllm_config.attention_config.indexer_tp_shard_min_rows,
)
prefill_metadata = DeepseekV32IndexerPrefillMetadata(
chunks,
+9 -1
View File
@@ -85,6 +85,7 @@ class DeepseekV4SWACache(torch.nn.Module, AttentionLayerBase):
block_size: int = 64,
packed_bytes_per_token: int = 584,
packed_page_alignment: int = 576,
packed_block_stride_alignment: int | None = None,
bounded_replay: bool = False,
):
super().__init__()
@@ -99,6 +100,7 @@ class DeepseekV4SWACache(torch.nn.Module, AttentionLayerBase):
self.dtype = dtype
self.packed_bytes_per_token = packed_bytes_per_token
self.packed_page_alignment = packed_page_alignment
self.packed_block_stride_alignment = packed_block_stride_alignment
compilation_config = get_current_vllm_config().compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
@@ -135,8 +137,11 @@ class DeepseekV4SWACache(torch.nn.Module, AttentionLayerBase):
state_content_bytes=(
self.packed_bytes_per_token if uses_fp8_ds_mla_layout else None
),
# FlashMLA packing stride; 512B for FlashInfer sparse (#44577).
# Packed page alignment; 512B for FlashInfer sparse (#44577).
alignment=self.packed_page_alignment if uses_fp8_ds_mla_layout else 512,
block_stride_alignment=(
self.packed_block_stride_alignment if uses_fp8_ds_mla_layout else None
),
model_version="deepseek_v4",
kv_quant_mode=get_kv_quant_mode(self.cache_config.cache_dtype),
)
@@ -211,6 +216,9 @@ class DeepseekSparseSWAMetadata:
# window attention may read (SWA bounded replay);
# zeros when nothing replays.
replay_start: torch.Tensor | None = None
# PCP cache writes use the gathered slots; queries use rank-local slots.
cache_slot_mapping: torch.Tensor | None = None
cache_positions: torch.Tensor | None = None
# Number of decode/prefill requests/tokens (batch is reordered: decodes first)
num_decodes: int = 0
+3
View File
@@ -117,6 +117,9 @@ class AttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta):
"vllm.models.deepseek_v41.nvidia.flashinfer_sparse."
"DeepseekV4FlashInferMLASparseBackend"
)
ROCM_FLASHMLA_SPARSE_DSV41 = (
"vllm.models.deepseek_v41.amd.rocm.DeepseekV41ROCMAiterMLASparseBackend"
)
FLASHMLA_MEGA_ATTN_DSV41 = (
"vllm.models.deepseek_v41.sparse_mla.FlashMLAMegaAttnBackend"
)
+95 -54
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Attention layer with AiterFlashAttention."""
from dataclasses import dataclass, replace
from dataclasses import dataclass, field, replace
from typing import ClassVar
import torch
@@ -152,8 +152,8 @@ if current_platform.is_rocm():
k_reg = tl.load(key_cache_ptr_offset + k_reg_offset)
v_reg = tl.load(value_cache_ptr_offset + v_reg_offset)
if DEQUANT:
k_scale = 1.0
v_scale = 1.0
k_scale = tl.load(k_scale_ptr)
v_scale = tl.load(v_scale_ptr)
k_reg = k_reg.to(tl.float32) * k_scale
v_reg = v_reg.to(tl.float32) * v_scale
tl.store(key_ptr_offset + col_offsets, k_reg)
@@ -270,8 +270,8 @@ if current_platform.is_rocm():
k_val = tl.load(key_ptr + src_offset_k + offset)
v_val = tl.load(value_ptr + src_offset_v + offset)
if QUANT:
k_scale = 1.0
v_scale = 1.0
k_scale = tl.load(k_scale_ptr)
v_scale = tl.load(v_scale_ptr)
k_dtype = key_cache_ptr.type.element_ty
v_dtype = value_cache_ptr.type.element_ty
k_val = (k_val.to(tl.float32) / k_scale).to(k_dtype)
@@ -410,11 +410,48 @@ class AiterFlashAttentionMetadata:
kv_sharing_metadata: AiterKVSharingMetadata | None = None
@dataclass
class AsmKvScaleTables:
"""Dense per-slot scale tables the asm decode kernel reads for an fp8 cache.
One table per distinct (device, shape, scale), shared by every layer and
every metadata builder in the process, so the target and draft models with
equal scales reuse a single allocation. Tables are constants: never write
to one in place; a per-token or dynamic scale needs its own buffer.
"""
tables: dict[tuple[str, int, int, int, float], torch.Tensor] = field(
default_factory=dict
)
def get(
self,
device: torch.device,
num_blocks: int,
num_kv_heads: int,
block_size: int,
scale: float,
) -> torch.Tensor:
key = (str(device), num_blocks, num_kv_heads, block_size, scale)
table = self.tables.get(key)
if table is None:
table = torch.full(
(num_blocks, num_kv_heads, block_size),
scale,
dtype=torch.float32,
device=device,
)
self.tables[key] = table
return table
class AiterFlashAttentionMetadataBuilder(
AttentionMetadataBuilder[AiterFlashAttentionMetadata]
):
_cudagraph_support = AttentionCGSupport.UNIFORM_BATCH
asm_kv_scale_tables: ClassVar[AsmKvScaleTables] = AsmKvScaleTables()
def __init__(
self,
kv_cache_spec: AttentionSpec,
@@ -466,7 +503,8 @@ class AiterFlashAttentionMetadataBuilder(
dtype=self.model_config.dtype,
device=device,
)
self.scale = torch.tensor([1.0], dtype=torch.float, device=self.device)
self.k_scale: dict[str, torch.Tensor] | None = None
self.v_scale: dict[str, torch.Tensor] | None = None
self.kv_sharing_workspace = (
torch.empty(
(
@@ -488,6 +526,38 @@ class AiterFlashAttentionMetadataBuilder(
common_prefix_len=0, common_attn_metadata=common_attn_metadata
)
def _build_asm_kv_scale_tables(self) -> None:
# Only for fp8 shuffle kv cache: the asm decode kernel reads a dense
# [num_blocks, num_kv_heads, block_size] scale table per layer. Map each
# layer to the shared table for its scale once the KV cache is bound.
if (
self.k_scale is not None
or not rocm_aiter_ops.is_shuffle_kv_cache_enabled()
or not is_quantized_kv_cache(self.vllm_config.cache_config.cache_dtype)
):
return
attn_layers = self.vllm_config.compilation_config.static_forward_context
k_scale_tables: dict[str, torch.Tensor] = {}
v_scale_tables: dict[str, torch.Tensor] = {}
for layer_name in self.layer_names:
layer = attn_layers[layer_name]
num_blocks = layer.kv_cache.shape[0]
k_scale_tables[layer_name] = self.asm_kv_scale_tables.get(
self.device,
num_blocks,
self.num_heads_kv,
self.block_size,
layer._k_scale_float,
)
v_scale_tables[layer_name] = self.asm_kv_scale_tables.get(
self.device,
num_blocks,
self.num_heads_kv,
self.block_size,
layer._v_scale_float,
)
self.k_scale, self.v_scale = k_scale_tables, v_scale_tables
def build(
self,
common_prefix_len: int,
@@ -499,24 +569,7 @@ class AiterFlashAttentionMetadataBuilder(
common_attn_metadata,
decode_threshold=self.reorder_batch_threshold,
)
# Allocate scales for fp8 shuffle kv cache with shuffle_kv_cache enabled
if (
rocm_aiter_ops.is_shuffle_kv_cache_enabled()
and self.scale.numel() == 1
and is_quantized_kv_cache(self.vllm_config.cache_config.cache_dtype)
):
# Size the scales from a layer this builder owns. The draft model
# runs its own builder over its own KV cache, so the first layer of
# the whole config can carry an unrelated block count.
kv_cache_shape = self.vllm_config.compilation_config.static_forward_context[
self.layer_names[0]
].kv_cache.shape
num_blocks = kv_cache_shape[0]
self.scale = torch.ones(
[num_blocks, self.num_heads_kv, self.block_size],
dtype=torch.float32,
device=self.device,
)
self._build_asm_kv_scale_tables()
(
num_decodes,
num_extends,
@@ -709,8 +762,8 @@ class AiterFlashAttentionMetadataBuilder(
decode_metadata=decode_metadata,
prefill_metadata=prefill_metadata,
extend_metadata=extend_metadata,
k_scale=self.scale,
v_scale=self.scale,
k_scale=self.k_scale,
v_scale=self.v_scale,
kv_sharing_metadata=kv_sharing_metadata,
)
return attn_metadata
@@ -726,6 +779,7 @@ class AiterFlashAttentionMetadataBuilder(
skip split_decodes_prefills_and_extends() and avoid all .cpu() /
.item() calls that would otherwise break CUDA graph capture.
"""
self._build_asm_kv_scale_tables()
num_reqs = common_attn_metadata.num_reqs
num_tokens = common_attn_metadata.num_actual_tokens
@@ -749,8 +803,8 @@ class AiterFlashAttentionMetadataBuilder(
decode_metadata=decode_metadata,
prefill_metadata=None,
extend_metadata=None,
k_scale=self.scale,
v_scale=self.scale,
k_scale=self.k_scale,
v_scale=self.v_scale,
)
@@ -1249,9 +1303,6 @@ class AiterFlashAttentionImpl(AttentionImpl):
extend_outputs = output[extend_tokens_slice]
k_scale = layer._k_scale
v_scale = layer._v_scale
if rocm_aiter_ops.is_shuffle_kv_cache_enabled():
k_scale = attn_metadata.k_scale
v_scale = attn_metadata.v_scale
self.extend_forward(
attn_metadata=attn_metadata,
query=extend_queries,
@@ -1428,16 +1479,13 @@ class AiterFlashAttentionImpl(AttentionImpl):
device=query.device,
)
max_logits = torch.empty_like(exp_sums)
k_qscale = (
layer._k_scale
if attn_metadata.k_scale is None
else attn_metadata.k_scale
)
v_qscale = (
layer._v_scale
if attn_metadata.v_scale is None
else attn_metadata.v_scale
)
# asm indexes this layer's dense scale buffer from build();
# the HIP kernel reads the per-tensor scale directly.
k_qscale_asm = v_qscale_asm = None
if attn_metadata.k_scale is not None:
assert attn_metadata.v_scale is not None
k_qscale_asm = attn_metadata.k_scale[layer.layer_name]
v_qscale_asm = attn_metadata.v_scale[layer.layer_name]
rocm_aiter_ops.paged_attention_common(
Q=query[:num_decode_tokens],
K=new_key_cache,
@@ -1452,10 +1500,10 @@ class AiterFlashAttentionImpl(AttentionImpl):
0
),
scale=self.scale,
K_QScale_hip=k_qscale,
V_QScale_hip=v_qscale,
K_QScale_asm=k_qscale,
V_QScale_asm=v_qscale,
K_QScale_hip=layer._k_scale,
V_QScale_hip=layer._v_scale,
K_QScale_asm=k_qscale_asm,
V_QScale_asm=v_qscale_asm,
out_=output[:num_decode_tokens],
kv_cache_dtype=self.kv_cache_dtype,
)
@@ -1542,9 +1590,6 @@ class AiterFlashAttentionImpl(AttentionImpl):
# the reshape_and_cache_flash op uses the slot_mapping's shape
# to determine the number of actual tokens.
if rocm_aiter_ops.is_shuffle_kv_cache_enabled():
# We may calculate per token quant scale in
# reshape_and_cache_shuffle_triton which might differ from
# vllm's style when shuffle layout is used.
k_scale = layer._k_scale
v_scale = layer._v_scale
assert k_scale is not None and v_scale is not None, (
@@ -1581,12 +1626,7 @@ class AiterFlashAttentionImpl(AttentionImpl):
)
def fused_qk_norm_rope_kvcache_supported(self):
# Only fuse when shuffle layout is off; the shuffle write path uses a
# dedicated cache update, mirroring fused_rope_kvcache_supported.
return (
rocm_aiter_ops.is_enabled()
and not rocm_aiter_ops.is_shuffle_kv_cache_enabled()
)
return rocm_aiter_ops.is_enabled()
def do_qk_norm_rope_kvcache_update(
self,
@@ -1624,6 +1664,7 @@ class AiterFlashAttentionImpl(AttentionImpl):
v_scale=layer._v_scale_cpu,
kv_cache_dtype=self.kv_cache_dtype,
use_shuffle_layout=rocm_aiter_ops.is_shuffle_kv_cache_enabled(),
return_kv=True,
)
def do_rope_and_kv_cache_update(
@@ -15,6 +15,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8StaticTensorSym,
)
from vllm.platforms.rocm import on_gfx1250
from vllm.utils.torch_utils import is_quantized_kv_cache
from vllm.v1.attention.backend import (
AttentionLayer,
@@ -28,7 +29,12 @@ from vllm.v1.attention.backends.rocm_attn import (
RocmAttentionMetadata,
RocmAttentionMetadataBuilder,
)
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheLayout
from vllm.v1.kv_cache_interface import (
AttentionSpec,
KVCacheLayout,
KVQuantMode,
get_kv_quant_mode,
)
logger = init_logger(__name__)
@@ -186,12 +192,34 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl):
# RDNA3 has no low precision MMA, preserve query precision
self.supports_quant_query_input = not on_gfx11()
# Layout flag: must mirror get_kv_cache_shape, which is attn_type
# agnostic because the allocation is shared across layers.
self._shuffle_kv_layout = (
rocm_aiter_ops.is_shuffle_kv_cache_enabled() and on_gfx1250()
)
# Kernel flag: the shuffled reader/writer is causal decoder only.
self._use_shuffle_kv = (
self._shuffle_kv_layout and self.attn_type != AttentionType.ENCODER_DECODER
)
def _split_kv_cache(
self, kv_cache: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
if self._shuffle_kv_layout:
# (2, B, N, H, hs) -> ((B, N, H, hs), (B, N, H, hs))
return kv_cache.unbind(0)
# (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs))
return kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
def _shuffle_kv_views(
self, key_cache: torch.Tensor, value_cache: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
nb, bs, nkv, hd = key_cache.shape
x = 16 // key_cache.element_size()
kc = key_cache.reshape(nb, nkv, hd // x, bs, x)
vc = value_cache.reshape(nb, nkv, bs // x, hd, x)
return kc, vc
def forward(
self,
layer: torch.nn.Module,
@@ -264,6 +292,9 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl):
key_cache = key_cache.view(self.fp8_dtype)
value_cache = value_cache.view(self.fp8_dtype)
if self._use_shuffle_kv:
key_cache, value_cache = self._shuffle_kv_views(key_cache, value_cache)
cu_seqlens_q = attn_metadata.query_start_loc
seqused_k = attn_metadata.seq_lens
max_seqlen_q = attn_metadata.max_query_len
@@ -291,6 +322,7 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl):
v_descale=layer._v_scale,
sinks=self.sinks,
output_scale=output_scale,
shuffled_kv_cache=self._use_shuffle_kv,
)
else:
# The aiter kernel is causal-only. Non-causal cross-attention
@@ -339,6 +371,27 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl):
return
key_cache, value_cache = self._split_kv_cache(kv_cache)
if self._use_shuffle_kv:
from vllm.v1.attention.backends.rocm_aiter_fa import (
reshape_and_cache_shuffle_triton,
)
if get_kv_quant_mode(self.kv_cache_dtype) == KVQuantMode.FP8_PER_TENSOR:
key_cache = key_cache.view(self.fp8_dtype)
value_cache = value_cache.view(self.fp8_dtype)
_dummy = torch.empty(1, dtype=torch.float32, device=key.device)
reshape_and_cache_shuffle_triton(
key,
value,
key_cache,
value_cache,
slot_mapping,
self.kv_cache_dtype,
_dummy,
_dummy,
)
return
# Reshape the input keys and values and store them in the cache.
ops.reshape_and_cache_flash(
key,
@@ -360,6 +413,11 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl):
def fused_qk_norm_mrope_kvcache_supported(self) -> bool:
return IS_AITER_FOUND
def set_interleaved_v_cache(self):
# No-op override of RocmAttentionImpl: this backend
# decodes with the AITER triton unified-attention kernel
pass
def do_qk_norm_rope_kvcache_update(
self,
layer: AttentionLayer,
@@ -395,7 +453,8 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl):
k_scale=layer._k_scale_cpu,
v_scale=layer._v_scale_cpu,
kv_cache_dtype=self.kv_cache_dtype,
use_shuffle_layout=False,
use_shuffle_layout=self._use_shuffle_kv,
return_kv=False,
)
def do_qk_norm_mrope_kvcache_update(
@@ -456,13 +515,16 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl):
# we use direct Q, K, V tensors without caching
return
key_cache, value_cache = self._split_kv_cache(kv_cache)
flash_layout = True
is_fp8_kv_cache = is_quantized_kv_cache(self.kv_cache_dtype)
if is_fp8_kv_cache:
key_cache = key_cache.view(self.fp8_dtype)
value_cache = value_cache.view(self.fp8_dtype)
if self._use_shuffle_kv:
key_cache, value_cache = self._shuffle_kv_views(key_cache, value_cache)
flash_layout = not self._use_shuffle_kv
rocm_aiter_ops.triton_rope_and_cache(
query,
key,
+63
View File
@@ -292,6 +292,8 @@ class RocmAttentionImpl(AttentionImpl):
f"num_heads: {num_heads}."
)
self._use_interleaved_v_cache = False
def _forward_encoder_attention(
self,
query: torch.Tensor,
@@ -450,6 +452,7 @@ class RocmAttentionImpl(AttentionImpl):
sm_scale=self.scale,
output_scale=output_scale,
sinks=self.sinks,
use_interleaved_v_cache=self._use_interleaved_v_cache,
causal=attn_metadata.causal,
)
@@ -475,6 +478,12 @@ class RocmAttentionImpl(AttentionImpl):
block_size = value_cache.shape[3]
has_native_layout = has_native_kv_cache_layout(key_cache, value_cache)
# Match the fused writer's interleaved V-cache when fusion is active.
interleaved_v = self._use_interleaved_v_cache
interleaved_v_pack_factor = (
(16 // value_cache.element_size()) if interleaved_v else 0
)
if block_size in (16, 32) and has_native_layout:
# Normal 16, 32 with contiguous blocks: use vLLM native HIP C++ logic.
PagedAttention.write_to_paged_cache(
@@ -486,6 +495,7 @@ class RocmAttentionImpl(AttentionImpl):
self.kv_cache_dtype,
layer._k_scale,
layer._v_scale,
interleaved_v,
)
else:
# Non-standard blocks and hybrid attention/Mamba layouts need the
@@ -501,8 +511,61 @@ class RocmAttentionImpl(AttentionImpl):
self.kv_cache_dtype,
layer._k_scale,
layer._v_scale,
interleaved_v_pack_factor,
)
def fused_qk_norm_rope_kvcache_supported(self):
return rocm_aiter_ops.is_enabled()
def set_interleaved_v_cache(self):
# Decode/prefill kernels must read the interleaved V-cache the AITER
# fused write emits when use_shuffle_layout=True.
self._use_interleaved_v_cache = True
def do_qk_norm_rope_kvcache_update(
self,
layer: AttentionLayer,
qkv: torch.Tensor,
q_out: torch.Tensor,
k_out: torch.Tensor,
positions: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
rms_norm_eps: float,
cos_sin_cache: torch.Tensor,
is_neox: bool,
kv_cache: torch.Tensor,
layer_slot_mapping: torch.Tensor,
):
# kv_cache is logical [num_blocks, 2, block_size, num_kv_heads * head_size]
# (see forward); the AITER shuffle writer takes one (B, N, H, hs) per side.
key_cache, value_cache = kv_cache.unflatten(
-1, (self.num_kv_heads, self.head_size)
).unbind(1)
use_shuffle_layout = self._use_interleaved_v_cache
rocm_aiter_ops.do_qk_norm_rope_kvcache_update(
qkv=qkv,
q_weight=q_weight,
k_weight=k_weight,
cos_sin_cache=cos_sin_cache,
positions=positions,
num_heads_q=self.num_heads,
num_heads_k=self.num_kv_heads,
head_dim=self.head_size,
is_neox=is_neox,
rms_norm_eps=rms_norm_eps,
q_out=q_out,
k_out=k_out,
key_cache=key_cache,
value_cache=value_cache,
slot_mapping=layer_slot_mapping,
k_scale=layer._k_scale_cpu,
v_scale=layer._v_scale_cpu,
kv_cache_dtype=self.kv_cache_dtype,
use_shuffle_layout=use_shuffle_layout,
return_kv=True,
)
def fused_rope_kvcache_supported(self):
return rocm_aiter_ops.is_enabled()
+124 -12
View File
@@ -18,6 +18,7 @@ from typing_extensions import runtime_checkable
from vllm.config import CacheConfig, VllmConfig, get_layers_from_vllm_config
from vllm.config.cache import _layout_from_name
from vllm.triton_utils import tl, triton
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d, np_to_pinned_tensor
@@ -1160,6 +1161,75 @@ def get_dcp_local_seq_lens(
return dcp_local_seq_lens
@triton.jit(
do_not_specialize_on_alignment=["out_ptr", "block_table_ptr", "seq_lens_ptr"]
)
def _mamba_align_block_table_kernel(
out_ptr,
block_table_ptr,
block_table_stride_0: tl.int64,
block_table_stride_1,
seq_lens_ptr,
seq_lens_stride,
block_size,
n_out,
BLOCK: tl.constexpr,
):
"""Gather the last ``n_out`` block-table entries of each request.
One program per request. This replaces a five-statement PyTorch expression
that expanded to seven kernels over a ``(num_reqs,)`` tensor -- one element
in a single-request decode -- so almost all of its cost was launch
overhead rather than work.
Both block-table strides are passed explicitly. The expression this replaces
was ``torch.gather(block_table, 1, ...)``, which honours any layout, and
this is a shared utility -- ``gdn_attn``, ``mamba_attn``, ``short_conv_attn``
and ``linear_attn`` all call it -- so assuming a column stride of 1 would
silently read the wrong slots for a caller holding a non-contiguous view.
``kimi_k3/nvidia/kda_metadata.py`` passes both strides for the same reason.
Two annotations above are load-bearing and easy to drop by accident:
``block_table_stride_0: tl.int64``
Without it ``req_idx * block_table_stride_0`` is computed in i32 --
``program_id`` is i32 and Triton specializes a stride below 2^31 to
i32 -- so the product wraps once a row offset reaches 2^31 elements
(a 4096 x 625000 table faults from row 3436 on MI355X).
``torch.gather``, which this replaces, indexes in 64-bit and has no
such limit, so narrowing it would be a regression.
``mamba_utils.py:43`` annotates its row stride the same way.
``do_not_specialize_on_alignment`` on the three pointers
Triton puts pointer alignment in the specialization key, and the MRV1
DBO path hands this kernel ``seq_lens[request_slice]`` /
``block_table_tensor[request_slice]`` VIEWS (ubatch_utils.py:278/316).
When a ubatch starts at a request index that is not a multiple of 4
the view is differently aligned and a fresh variant JIT-compiles
mid-step -- 25-28 ms cold, ~1.5 ms warm, and an error under
``jit_monitor_mode=error``. The torch expression this replaced never
compiled anything, so that recompile would be a regression introduced
by this change.
"""
req_idx = tl.program_id(0)
seq_len = tl.load(seq_lens_ptr + req_idx * seq_lens_stride)
# A zero-length request in a CUDA graph has an invalid block table; clamp
# to 0 so it reads a valid row, exactly as the previous clamp_ did.
start = tl.maximum((seq_len - 1) // block_size, 0)
offs = tl.arange(0, BLOCK)
mask = offs < n_out
vals = tl.load(
block_table_ptr
+ req_idx * block_table_stride_0
+ (start + offs) * block_table_stride_1,
mask=mask,
other=0,
)
# `out` is freshly allocated by the caller and therefore contiguous, so its
# row stride is exactly n_out.
tl.store(out_ptr + req_idx * n_out + offs, vals, mask=mask)
def mamba_get_block_table_tensor(
block_table: torch.Tensor,
seq_lens: torch.Tensor,
@@ -1180,16 +1250,58 @@ def mamba_get_block_table_tensor(
return block_table
else:
assert isinstance(kv_cache_spec, MambaSpec)
# NOTE: For 0-length requests in CUDA graph, use a start_index of 0
# to handle the invalid block table.
start_indices = (seq_lens - 1) // kv_cache_spec.block_size
start_indices.clamp_(min=0)
# Use int32 for arithmetic to avoid dtype promotion overhead,
# then convert to int64 for gather (which requires Long indices)
offsets = torch.arange(
1 + kv_cache_spec.num_speculative_blocks,
device=block_table.device,
dtype=torch.int32,
# The arithmetic here is trivial -- a divide, a clamp, an arange, an
# add, a cast and a gather -- but as PyTorch ops it was seven kernel
# launches over a (num_reqs,) tensor, which holds a single element in a
# one-request decode. One Triton kernel does the same work in one
# launch. Zero-length requests in a CUDA graph still clamp to row 0.
num_reqs = block_table.shape[0]
n_out = 1 + kv_cache_spec.num_speculative_blocks
# The expression this replaced took its row count from `seq_lens` (a
# `torch.gather` whose index tensor is built from it), so a mismatch
# raised and a cross-device pair raised. Taking `num_reqs` from
# `block_table` instead silently changed all of that: with 4 block-table
# rows and 2 `seq_lens`, the old code returned 2 correct rows and the new
# kernel returns 4, two of them read from memory past the end of
# `seq_lens`; a pinned-CPU `seq_lens` with a GPU `block_table` used to
# raise and would now be read zero-copy from host memory.
#
# No in-tree caller does any of that today, but this helper is shared by
# gdn_attn, mamba_attn, short_conv_attn and linear_attn. Both are host
# metadata, so this costs no synchronisation, and it keeps the CPU
# branch below and the GPU kernel agreeing on the same inputs.
assert seq_lens.shape[0] == num_reqs, (
f"seq_lens has {seq_lens.shape[0]} rows, block_table has {num_reqs}"
)
indices_to_gather = (start_indices.unsqueeze(1) + offsets).to(torch.int64)
return torch.gather(block_table, 1, indices_to_gather)
assert seq_lens.device == block_table.device, (
f"seq_lens is on {seq_lens.device}, block_table on {block_table.device}"
)
if not block_table.is_cuda:
# Triton runs on the accelerator only, and this helper IS called
# with CPU tensors: the GDN and linear-attention metadata builders
# are unit-tested on CPU (`DEVICE = torch.device("cpu")`). The
# expression this replaced was plain torch and handled that, so
# keep it for the CPU path rather than narrowing the contract.
# kimi_k3/nvidia/kda_metadata.py asserts `is_cuda` instead; a
# shared utility cannot, because its callers legitimately pass CPU.
start_indices = (seq_lens - 1) // kv_cache_spec.block_size
start_indices = start_indices.clamp(min=0)
offsets = torch.arange(n_out, device=block_table.device, dtype=torch.int32)
return torch.gather(
block_table, 1, (start_indices.unsqueeze(1) + offsets).to(torch.int64)
)
out = block_table.new_empty((num_reqs, n_out))
if num_reqs == 0:
return out
_mamba_align_block_table_kernel[(num_reqs,)](
out,
block_table,
block_table.stride(0),
block_table.stride(1),
seq_lens,
seq_lens.stride(0),
kv_cache_spec.block_size,
n_out,
BLOCK=triton.next_power_of_2(n_out),
)
return out
@@ -84,6 +84,7 @@ def kernel_paged_attention_2d(
query_start_len_ptr, # [num_seqs+1]
USE_SINKS: tl.constexpr, # bool
USE_FP8: tl.constexpr,
INTERLEAVED_V_PACK_FACTOR: tl.constexpr,
FP8_MIN: tl.constexpr = float8_info.min,
FP8_MAX: tl.constexpr = float8_info.max,
):
@@ -170,13 +171,24 @@ def kernel_paged_attention_2d(
+ (offs_d[:, None] % x) * stride_k_cache_4
)
# 4D addressing logic of V (Slot is innermost)
v_offset = (
p_block_idx[:, None] * stride_v_cache_0
+ kv_head_idx * stride_v_cache_1
+ offs_d[None, :] * stride_v_cache_2
+ internal_offsets[:, None] * stride_v_cache_3
)
# V addressing: standard 4D or interleaved 5D
if INTERLEAVED_V_PACK_FACTOR > 0:
v_offset = (
p_block_idx[:, None] * stride_v_cache_0
+ kv_head_idx * stride_v_cache_1
+ (internal_offsets[:, None] // INTERLEAVED_V_PACK_FACTOR)
* HEAD_SIZE
* INTERLEAVED_V_PACK_FACTOR
+ offs_d[None, :] * INTERLEAVED_V_PACK_FACTOR
+ (internal_offsets[:, None] % INTERLEAVED_V_PACK_FACTOR)
)
else:
v_offset = (
p_block_idx[:, None] * stride_v_cache_0
+ kv_head_idx * stride_v_cache_1
+ offs_d[None, :] * stride_v_cache_2
+ internal_offsets[:, None] * stride_v_cache_3
)
# Only the final tile can straddle seq_len. Slots >= seq_len are
# unwritten KV cache that may hold NaN/garbage; they are score-masked
@@ -304,6 +316,7 @@ def chunked_prefill_paged_decode(
# Optional tensor for sinks
sinks=None,
is_block_table_ptr: bool = False,
use_interleaved_v_cache: bool = False,
causal: bool = True,
):
if sm_scale is None:
@@ -314,6 +327,10 @@ def chunked_prefill_paged_decode(
if sliding_window is None or sliding_window <= 0:
sliding_window = 0
interleaved_v_pack_factor = (
(16 // value_cache.element_size()) if use_interleaved_v_cache else 0
)
if max_query_len > 1:
context_attention_fwd(
q=query,
@@ -336,6 +353,7 @@ def chunked_prefill_paged_decode(
skip_decode=True,
fp8_out_scale=output_scale,
sinks=sinks,
interleaved_v_pack_factor=interleaved_v_pack_factor,
causal=causal,
)
@@ -428,6 +446,7 @@ def chunked_prefill_paged_decode(
k_scale=k_scale,
v_scale=v_scale,
fp8_out_scale=output_scale,
use_interleaved_v_cache=use_interleaved_v_cache,
)
else:
logger.warning_once(
@@ -504,4 +523,5 @@ def chunked_prefill_paged_decode(
query_start_len_ptr=query_start_loc,
USE_SINKS=sinks is not None,
USE_FP8=output_scale is not None,
INTERLEAVED_V_PACK_FACTOR=interleaved_v_pack_factor,
)
+2
View File
@@ -38,6 +38,7 @@ class PagedAttention:
kv_cache_dtype: str,
k_scale: torch.Tensor,
v_scale: torch.Tensor,
interleaved_v: bool,
) -> None:
ops.reshape_and_cache(
key,
@@ -48,4 +49,5 @@ class PagedAttention:
kv_cache_dtype,
k_scale,
v_scale,
interleaved_v,
)
+51 -13
View File
@@ -56,6 +56,8 @@ def _paged_kv_cache_offsets(
stride_v_cache_d,
stride_v_cache_bl,
PHYSICAL_BLOCK_SIZE: tl.constexpr,
BLOCK_DMODEL: tl.constexpr,
INTERLEAVED_V_PACK_FACTOR: tl.constexpr = 0,
MASK_BLOCK_TABLE: tl.constexpr = False,
):
"""Compute paged K/V cache element offsets for a tile of token positions.
@@ -65,7 +67,9 @@ def _paged_kv_cache_offsets(
tiles), then the physical element offsets are built with the same layout the
kernel uses everywhere:
K cache: [num_blocks, num_kv_heads, head_size // x, block_size, x]
V cache: [num_blocks, num_kv_heads, head_size, block_size]
V cache: [num_blocks, num_kv_heads, head_size, block_size], or the
interleaved layout [num_blocks, num_kv_heads, block_size // x,
head_size, x] when `INTERLEAVED_V_PACK_FACTOR` (= x) > 0.
Returns `(off_k, off_v)` with shapes [D, N] and [N, D] respectively.
When `MASK_BLOCK_TABLE` is set, the block-table load is masked with
@@ -94,12 +98,25 @@ def _paged_kv_cache_offsets(
+ internal[None, :] * stride_k_cache_bl
+ (offs_d[:, None] % x) * stride_k_cache_x
)
off_v = (
bn[:, None] * stride_v_cache_bs
+ cur_kv_head * stride_v_cache_h
+ offs_d[None, :] * stride_v_cache_d
+ internal[:, None] * stride_v_cache_bl
)
if INTERLEAVED_V_PACK_FACTOR > 0:
# Interleaved V-cache: [num_blocks, num_kv_heads, block_size/x,
# head_size, x] flattened onto the 4D stride layout.
off_v = (
bn[:, None] * stride_v_cache_bs
+ cur_kv_head * stride_v_cache_h
+ (internal[:, None] // INTERLEAVED_V_PACK_FACTOR)
* BLOCK_DMODEL
* INTERLEAVED_V_PACK_FACTOR
+ offs_d[None, :] * INTERLEAVED_V_PACK_FACTOR
+ (internal[:, None] % INTERLEAVED_V_PACK_FACTOR)
)
else:
off_v = (
bn[:, None] * stride_v_cache_bs
+ cur_kv_head * stride_v_cache_h
+ offs_d[None, :] * stride_v_cache_d
+ internal[:, None] * stride_v_cache_bl
)
return off_k, off_v
@@ -159,6 +176,7 @@ def _fwd_kernel(
USE_FP8: tl.constexpr,
CAUSAL: tl.constexpr = True,
MAX_Q_LEN: tl.constexpr = 0,
INTERLEAVED_V_PACK_FACTOR: tl.constexpr = 0,
MAX_CTX_LEN: tl.constexpr = 0,
FP8_MIN: tl.constexpr = float8_info.min,
FP8_MAX: tl.constexpr = float8_info.max,
@@ -263,6 +281,8 @@ def _fwd_kernel(
stride_v_cache_d,
stride_v_cache_bl,
PHYSICAL_BLOCK_SIZE,
BLOCK_DMODEL,
INTERLEAVED_V_PACK_FACTOR,
)
if (
@@ -403,6 +423,8 @@ def _fwd_kernel(
stride_v_cache_d,
stride_v_cache_bl,
PHYSICAL_BLOCK_SIZE,
BLOCK_DMODEL,
INTERLEAVED_V_PACK_FACTOR,
MASK_BLOCK_TABLE=True,
)
k_cur_load = tl.load(
@@ -540,6 +562,7 @@ def _fwd_kernel_alibi(
BLOCK_DMODEL_PADDED: tl.constexpr, # head size padded to a power of 2
BLOCK_N: tl.constexpr,
SKIP_DECODE: tl.constexpr,
INTERLEAVED_V_PACK_FACTOR: tl.constexpr,
):
# attn_bias[]
cur_batch = tl.program_id(0)
@@ -608,12 +631,24 @@ def _fwd_kernel_alibi(
+ ((start_n + offs_n[None, :]) % block_size) * stride_k_cache_bl
+ (offs_d[:, None] % x) * stride_k_cache_x
)
off_v = (
bn[:, None] * stride_v_cache_bs
+ cur_kv_head * stride_v_cache_h
+ offs_d[None, :] * stride_v_cache_d
+ (start_n + offs_n[:, None]) % block_size * stride_v_cache_bl
)
if INTERLEAVED_V_PACK_FACTOR > 0:
v_internal = (start_n + offs_n[:, None]) % block_size
off_v = (
bn[:, None] * stride_v_cache_bs
+ cur_kv_head * stride_v_cache_h
+ (v_internal // INTERLEAVED_V_PACK_FACTOR)
* BLOCK_DMODEL
* INTERLEAVED_V_PACK_FACTOR
+ offs_d[None, :] * INTERLEAVED_V_PACK_FACTOR
+ (v_internal % INTERLEAVED_V_PACK_FACTOR)
)
else:
off_v = (
bn[:, None] * stride_v_cache_bs
+ cur_kv_head * stride_v_cache_h
+ offs_d[None, :] * stride_v_cache_d
+ (start_n + offs_n[:, None]) % block_size * stride_v_cache_bl
)
k_load = tl.load(
K_cache + off_k,
mask=dim_mask[:, None] & ((start_n + offs_n[None, :]) < cur_batch_ctx_len),
@@ -796,6 +831,7 @@ def context_attention_fwd(
fp8_out_scale=None,
sinks=None,
is_block_table_ptr: bool = False,
interleaved_v_pack_factor: int = 0,
causal: bool = True,
):
q_dtype_is_f32 = q.dtype is torch.float32
@@ -937,6 +973,7 @@ def context_attention_fwd(
BLOCK_DMODEL_PADDED=Lk_padded,
BLOCK_N=BLOCK,
SKIP_DECODE=skip_decode,
INTERLEAVED_V_PACK_FACTOR=interleaved_v_pack_factor,
num_warps=NUM_WARPS,
num_stages=1,
)
@@ -1010,6 +1047,7 @@ def context_attention_fwd(
num_warps=4,
num_stages=1,
USE_SINKS=sinks is not None,
INTERLEAVED_V_PACK_FACTOR=interleaved_v_pack_factor,
CAUSAL=causal,
KV_FROM_CACHE=kv_from_cache,
**extra_kargs,
@@ -52,6 +52,9 @@ def reshape_and_cache_kernel_flash(
block_size: tl.constexpr,
x: tl.constexpr,
USE_HEAD_MAJOR_LAYOUT: tl.constexpr,
# >0 writes V interleaved as [Block, Head, Slot//kx, Dim, kx] (head-major
# layout only); 0 keeps the standard [Block, Head, Dim, Slot] V layout.
INTERLEAVED_V_PACK_FACTOR: tl.constexpr,
# FP8 flags
FP8_KV_CACHE: tl.constexpr,
# tune parameters
@@ -76,13 +79,25 @@ def reshape_and_cache_kernel_flash(
# Decompose the tile index back into head and dim coordinates.
cur_head = tile_pos // head_size
cur_dim = tile_pos % head_size
# Value addressing (4D): [Block, Head, Dim, Slot]
tgt_idx_v = (
block_idx * block_stride
+ cur_head * head_stride
+ cur_dim * dim_stride_v
+ block_offset * 1
)
# Value addressing: standard 4D [Block, Head, Dim, Slot], or the
# interleaved 5D [Block, Head, Slot//kx, Dim, kx] the fused AITER
# writer emits and the interleaved read path expects.
if INTERLEAVED_V_PACK_FACTOR > 0:
tgt_idx_v = (
block_idx * block_stride
+ cur_head * head_stride
+ (block_offset // INTERLEAVED_V_PACK_FACTOR)
* (head_size * INTERLEAVED_V_PACK_FACTOR)
+ cur_dim * INTERLEAVED_V_PACK_FACTOR
+ (block_offset % INTERLEAVED_V_PACK_FACTOR)
)
else:
tgt_idx_v = (
block_idx * block_stride
+ cur_head * head_stride
+ cur_dim * dim_stride_v
+ block_offset * 1
)
# Key addressing (5D): [Block, Head, Dim//8, Slot, 8]
tgt_idx_k = (
block_idx * block_stride
@@ -371,11 +386,18 @@ def triton_reshape_and_cache_flash(
kv_cache_dtype: str, # "auto", "fp8"
k_scale: torch.Tensor, # float32
v_scale: torch.Tensor, # float32
interleaved_v_pack_factor: int = 0, # >0: interleave V (head-major); 0: standard
):
num_heads = key.shape[1]
head_size = key.shape[2]
use_head_major_layout = key_cache.ndim == 5
if interleaved_v_pack_factor and not use_head_major_layout:
raise NotImplementedError(
"Interleaved V-cache write is only implemented for the head-major "
"(5D) cache layout; got a non-head-major cache with "
f"interleaved_v_pack_factor={interleaved_v_pack_factor}."
)
if use_head_major_layout:
block_size = key_cache.shape[3]
x = key_cache.shape[4]
@@ -452,6 +474,7 @@ def triton_reshape_and_cache_flash(
block_size=block_size,
x=x,
USE_HEAD_MAJOR_LAYOUT=use_head_major_layout,
INTERLEAVED_V_PACK_FACTOR=interleaved_v_pack_factor,
FP8_KV_CACHE=FP8_KV_CACHE,
# autotune parameters
TILE_SIZE=TILE_SIZE,
+6 -2
View File
@@ -759,9 +759,11 @@ class BlockPool:
"""
for block in blocks:
if block.is_null:
continue
# ref_cnt=0 means this block is in the free list (i.e. eviction
# candidate), so remove it.
if block.ref_cnt == 0 and not block.is_null:
if block.ref_cnt == 0:
self.free_block_queue.remove(block)
block.ref_cnt += 1
if self.metrics_collector:
@@ -785,11 +787,13 @@ class BlockPool:
blocks_to_evict_first = []
other_pools: dict[BlockPool, list[KVCacheBlock]] = {}
for block in ordered_blocks:
if block.is_null: # never pooled; its ref_cnt is not maintained
continue
if block.pool is not None and block.pool is not self:
other_pools.setdefault(block.pool, []).append(block)
continue
block.ref_cnt -= 1
if block.ref_cnt == 0 and not block.is_null:
if block.ref_cnt == 0:
if block.block_hash is None or not self.enable_caching:
# LIFO reuse of non-cached blocks for better GPU locality.
blocks_to_evict_first.append(block)
+2 -2
View File
@@ -14,7 +14,7 @@ from vllm.v1.core.kv_cache_coordinator import (
get_kv_cache_coordinator,
)
from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector
from vllm.v1.core.kv_cache_utils import KVCacheBlock, KVCacheBlockCopy
from vllm.v1.core.kv_cache_utils import KVCacheBlock, KVCacheBlockCopy, trim_null_head
from vllm.v1.core.single_type_kv_cache_manager import MambaManager
from vllm.v1.kv_cache_interface import (
AttentionSpec,
@@ -117,7 +117,7 @@ class KVCacheBlocks:
return [
[
block.block_id
for block in group
for block in trim_null_head(group)
if block.block_hash is None and not block.is_null
]
for group in self.blocks
+35 -1
View File
@@ -8,7 +8,15 @@ import math
import os
from collections.abc import Callable, Iterable, Iterator, Sequence
from dataclasses import dataclass, field, replace
from typing import TYPE_CHECKING, Any, NamedTuple, NewType, TypeAlias, overload
from typing import (
TYPE_CHECKING,
Any,
NamedTuple,
NewType,
TypeAlias,
TypeVar,
overload,
)
from vllm import envs
from vllm.config import VllmConfig
@@ -216,6 +224,32 @@ class KVCacheBlock:
)
def null_head_len(blocks: Sequence[KVCacheBlock]) -> int:
"""Number of leading null blocks, e.g. a sliding-window table's skipped prefix."""
if not blocks or not blocks[0].is_null:
return 0
lo, hi = 1, len(blocks)
while lo < hi:
mid = (lo + hi) // 2
if blocks[mid].is_null:
lo = mid + 1
else:
hi = mid
# All null entries are one shared object, so count() checks the run at C speed.
if blocks[:lo].count(blocks[0]) == lo:
return lo
return next(i for i, b in enumerate(blocks) if not b.is_null)
_BlockSeq = TypeVar("_BlockSeq", list[KVCacheBlock], Sequence[KVCacheBlock])
def trim_null_head(blocks: _BlockSeq) -> _BlockSeq:
"""``blocks`` without its leading null blocks; ``blocks`` itself if none."""
nhl = null_head_len(blocks)
return blocks[nhl:] if nhl else blocks
class KVCacheBlockCopy(NamedTuple):
src_block_id: int
dst_block_id: int
+4 -2
View File
@@ -60,14 +60,16 @@ class AsyncScheduler(Scheduler):
self, request: Request, new_token_ids: list[int], is_stale: bool = False
) -> tuple[list[int], bool]:
status_before_update = request.status
# Tokens trimmed at a stop resolve their placeholders too.
num_new_tokens = len(new_token_ids)
new_token_ids, stopped = super()._update_request_with_output(
request, new_token_ids
request, new_token_ids, is_stale
)
# Placeholders were zeroed at preemption; a stale delivery must not
# decrement them (it would underflow).
if not is_stale:
request.num_output_placeholders -= len(new_token_ids)
request.num_output_placeholders -= num_new_tokens
assert request.num_output_placeholders >= 0
# Cache the new tokens. Preempted requests should be skipped.
+82 -1
View File
@@ -368,6 +368,36 @@ class Scheduler(SchedulerInterface):
# prefill batch fully drained the waiting queue. Prefill throttling
# is disabled in this case.
self.prefill_capacity_bound = False
# The DP engine core drives this cadence from a counter shared across
# ranks; an engine without one never throttles, so apply the interval
# locally, timed from the last prefill-carrying step. Three regimes:
# 1. no DP -- local cadence applies;
# 2. dense DP rank -- `reconfigure_for_independent_dp_rank` already
# set data_parallel_size=1, so each rank gets its own cadence.
# Intended, but a behaviour change: this used to be a no-op;
# 3. MoE DP rank (size > 1) -- the engine core drives it, no-op here.
self.local_prefill_interval = (
self.scheduler_config.prefill_schedule_interval
if self.parallel_config.data_parallel_size == 1
else 1
)
if self.local_prefill_interval > 1:
# Neither the hold nor its release is otherwise visible in logs or
# stats, which makes a misapplied cadence hard to diagnose.
logger.info_once(
"Prefill cadence: applying prefill_schedule_interval=%d "
"locally (this engine is not DP-coordinated); prefill is held "
"until %d executed steps have passed since the last "
"prefill-carrying step.",
self.local_prefill_interval,
self.local_prefill_interval,
)
# A full interval back, so the first step is never gated.
self.last_prefill_step = -self.local_prefill_interval
# Steps that actually scheduled tokens. `current_step` counts every
# schedule() call including empty ones, so it would expire the hold
# early; it keeps its own meaning (next_decode_eligible_step).
self.num_executed_steps = 0
self.scheduler_reserve_full_isl = (
self.scheduler_config.scheduler_reserve_full_isl
)
@@ -623,6 +653,9 @@ class Scheduler(SchedulerInterface):
scheduled_spec_decode_tokens: dict[str, list[int]] = {}
# Whether the running batch contains any prefill requests.
prefill_scheduled = False
# Whether a waiting request was admitted with prompt left to compute.
# Separate from `prefill_scheduled`, which covers only the running loop.
prefill_admitted = False
# Whether any scheduled request has a synchronous connector KV load.
has_sync_kv_loads = False
@@ -654,6 +687,28 @@ class Scheduler(SchedulerInterface):
long_prefill_token_threshold, input_budget // num_eligible_reqs
)
# Local prefill cadence: hold prefill until the interval since the last
# prefill-carrying step has elapsed, so decode runs in longer
# uninterrupted stretches. Mirrors the DP gate above, including its
# capacity escape: once a release step cannot drain the waiting queue,
# throttling stops (#44558, test_throttle_capacity_bound_guard_admits).
# Without that escape a backed-up engine holds both admissions and
# in-flight chunks for interval-1 of every interval steps, capping
# prefill at roughly one budget per interval and growing the waiting
# queue without bound.
if (
not defer_prefills
and self.local_prefill_interval > 1
and not self.prefill_capacity_bound
# This call becomes executed step num_executed_steps + 1 if it
# schedules anything, and last_prefill_step is stamped on the same
# convention below.
and (self.num_executed_steps + 1) - self.last_prefill_step
< self.local_prefill_interval
and any(not r.is_prefill_chunk for r in self.running)
):
defer_prefills = True
# First, schedule the RUNNING requests.
req_index = 0
while req_index < len(self.running) and token_budget > 0:
@@ -1318,6 +1373,12 @@ class Scheduler(SchedulerInterface):
)
if load_kv_async:
# This path deliberately does not reach `prefill_admitted`
# below, so it does not restart the prefill cadence:
# `num_new_tokens` was set to 0 above, so the step carries
# no prefill compute for this request. The interval is
# timed from steps that actually prefilled; refreshing it
# here would start the cadence from a step that did none.
# If loading async, allocate memory and put request
# into the WAITING_FOR_REMOTE_KV state.
request.status = RequestStatus.WAITING_FOR_REMOTE_KVS
@@ -1373,6 +1434,14 @@ class Scheduler(SchedulerInterface):
scheduled_spec_decode_tokens[request_id] = [
-1
] * self.num_spec_tokens
# Must use the SAME predicate as the deferral gate in the
# waiting loop (`num_computed_tokens < num_tokens - 1`).
# Stamping on `< num_prompt_tokens` instead let a request with
# exactly one token left pass the gate on a held step and still
# restart the interval -- which is every full prefix-cache hit,
# and every remote-KV resume, since those set
# num_computed_tokens = num_tokens - 1.
prefill_admitted |= num_computed_tokens < request.num_tokens - 1
# Only track requests that will still be prefilling after this chunk.
if num_computed_tokens + num_new_tokens < request.num_tokens:
self._inflight_prefills.add(request)
@@ -1403,6 +1472,15 @@ class Scheduler(SchedulerInterface):
# Check if the scheduling constraints are satisfied.
total_num_scheduled_tokens = sum(num_scheduled_tokens.values())
assert total_num_scheduled_tokens <= self.max_num_scheduled_tokens
# Count this step before stamping the cadence below, so that
# last_prefill_step includes the step that carried the prefill.
if total_num_scheduled_tokens > 0:
self.num_executed_steps += 1
# Restart the cadence interval from any step that carried prefill,
# whether a new admission or a running chunk.
if self.local_prefill_interval > 1 and (prefill_scheduled or prefill_admitted):
self.last_prefill_step = self.num_executed_steps
assert token_budget >= 0
assert input_budget >= 0
@@ -2496,7 +2574,6 @@ class Scheduler(SchedulerInterface):
def _update_request_with_output(
self, request: Request, new_token_ids: list[int], is_stale: bool = False
) -> tuple[list[int], bool]:
# is_stale is only used by the AsyncScheduler override.
# Append generated tokens and check for stop. Note that if
# a request is still being prefilled, we expect the model runner
# to return empty token ids for the request.
@@ -2508,6 +2585,10 @@ class Scheduler(SchedulerInterface):
# This must be called before we make the EngineCoreOutput.
stopped = check_stop(request, self.max_model_len)
if stopped:
# num_computed_tokens covers all new tokens but the last one.
# Keep that true after trimming: resumable requests resume from it.
if not is_stale:
request.num_computed_tokens -= len(new_token_ids) - num_new
del new_token_ids[num_new:] # Trim new tokens if needed.
break
return new_token_ids, stopped
+5 -3
View File
@@ -16,6 +16,7 @@ from vllm.v1.core.kv_cache_utils import (
BlockHashWithGroupId,
KVCacheBlock,
resolve_block_hashes,
trim_null_head,
)
from vllm.v1.hisparse.block_pool import SharedEventQueueBlockPool
from vllm.v1.kv_cache_interface import (
@@ -569,14 +570,15 @@ class SingleTypeKVCacheManager(ABC):
request_id: The request ID.
Returns:
The request's blocks in allocation order.
The request's blocks in allocation order, without the leading
null blocks (e.g. a sliding-window table's skipped prefix).
"""
# Default to [] in case a request is freed (aborted) before alloc.
req_blocks = self.req_to_blocks.pop(request_id, [])
self.num_cached_block.pop(request_id, None)
self._partial_hit_reqs.pop(request_id, None)
return req_blocks
return trim_null_head(req_blocks)
def free(self, request_id: str) -> None:
"""Free the blocks for the request.
@@ -2547,7 +2549,7 @@ class HiSparseResidentManager(_HiSparseAuxiliaryManager):
blocks = self.get_residency_row(request_id)
self.residency_changes.pop(request_id, None)
super().pop_blocks_for_free(request_id)
return blocks
return trim_null_head(blocks)
def adopt_resident_page(
self, request_id: str, block_idx: int, block: KVCacheBlock
+3
View File
@@ -104,6 +104,9 @@ class EngineCoreReadyResponse:
supports_draft_weight_updates: bool = False
# Full-attention block size in tokens after initialization, or unavailable.
effective_attention_block_size: int | None = None
coord_store_port: int = 0
coordinator_input_address: str | None = None
coordinator_output_address: str | None = None
class EngineCoreRequest(
+75 -11
View File
@@ -22,7 +22,11 @@ from vllm.distributed.weight_transfer.base import (
)
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.engine.protocol import EngineClient, StreamingInput
from vllm.entrypoints.serve.elastic_ep.middleware import set_scaling_elastic_ep
from vllm.entrypoints.serve.elastic_ep.middleware import (
get_elastic_ep_rank_retired,
set_elastic_ep_rank_retired,
set_scaling_elastic_ep,
)
from vllm.exceptions import (
GracefulHTTPError,
MaxQueuedTokensError,
@@ -53,7 +57,11 @@ from vllm.v1.engine.admission_control import SharedAdmissionStats
from vllm.v1.engine.core_client import EngineCoreClient
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
from vllm.v1.engine.input_processor import InputProcessor
from vllm.v1.engine.output_processor import OutputProcessor, RequestOutputCollector
from vllm.v1.engine.output_processor import (
OutputProcessor,
RequestOutputCollector,
StreamingInputOutputCollector,
)
from vllm.v1.engine.parallel_sampling import ParentRequest
from vllm.v1.executor import Executor
from vllm.v1.fault_tolerance.utils import FaultToleranceRequest, FaultToleranceResult
@@ -620,7 +628,9 @@ class AsyncLLM(EngineClient):
self.input_processor.assign_request_id(final_req)
internal_req_id = final_req.request_id
queue = RequestOutputCollector(sampling_params.output_kind, internal_req_id)
queue = StreamingInputOutputCollector(
sampling_params.output_kind, internal_req_id
)
async def handle_inputs():
cancelled = False
@@ -628,6 +638,13 @@ class AsyncLLM(EngineClient):
any_added = False
try:
async for input_chunk in input_stream:
# Stop once the session has finished (aborted or errored).
# There is no await between this check and the ADD send,
# so any chunk already sent precedes the engine abort.
if any_added and not self.output_processor.has_request(
internal_req_id
):
break
sp = input_chunk.sampling_params
if sp:
self._validate_streaming_input_sampling_params(sp)
@@ -663,8 +680,10 @@ class AsyncLLM(EngineClient):
if not cancelled:
if any_added:
# Send empty final request to indicate that inputs have
# finished. Don't send if cancelled (session was aborted).
await self._add_request(final_req, None, None, 0, queue)
# finished. Don't send if cancelled or if the session
# already finished (aborted or errored).
if self.output_processor.has_request(internal_req_id):
await self._add_request(final_req, None, None, 0, queue)
elif not errored:
queue.put(STREAM_FINISHED)
@@ -903,6 +922,12 @@ class AsyncLLM(EngineClient):
mm_cache_stats=renderer.stat_mm_cache(),
)
except Exception as e:
if get_elastic_ep_rank_retired() and isinstance(e, EngineDeadError):
logger.info(
"[Elastic EP] Retired rank stopped receiving EngineCore outputs"
)
output_processor.propagate_error(e)
return
logger.exception("AsyncLLM output_handler failed.")
output_processor.propagate_error(e)
@@ -1088,6 +1113,8 @@ class AsyncLLM(EngineClient):
async def check_health(self) -> None:
logger.debug("Called check_health.")
if get_elastic_ep_rank_retired():
return
if self.errored:
raise self.dead_error
@@ -1237,8 +1264,13 @@ class AsyncLLM(EngineClient):
"waiting for requests to drain before scaling"
)
await self.wait_for_requests_to_drain(drain_timeout)
except BaseException:
set_scaling_elastic_ep(False)
except BaseException as e:
if self.vllm_config.parallel_config.data_parallel_external_lb:
await self.engine_core.abort_external_elastic_ep(
str(e) or type(e).__name__
)
else:
set_scaling_elastic_ep(False)
raise
async def scale_elastic_ep(
@@ -1251,7 +1283,8 @@ class AsyncLLM(EngineClient):
async def _scale_elastic_ep(
self, new_data_parallel_size: int, drain_timeout: int
) -> None:
old_data_parallel_size = self.vllm_config.parallel_config.data_parallel_size
parallel_config = self.vllm_config.parallel_config
old_data_parallel_size = parallel_config.data_parallel_size
if old_data_parallel_size == new_data_parallel_size:
logger.info(
"Data parallel size is already %s, skipping scale",
@@ -1282,9 +1315,36 @@ class AsyncLLM(EngineClient):
if envs.VLLM_ELASTIC_EP_DRAIN_REQUESTS:
await self._drain_requests_for_elastic_ep(drain_timeout)
await self.engine_core.commit_elastic_ep()
self.vllm_config.parallel_config.data_parallel_size = new_data_parallel_size
set_scaling_elastic_ep(False)
rank_will_retire = (
parallel_config.data_parallel_external_lb
and new_data_parallel_size < old_data_parallel_size
and parallel_config.data_parallel_rank >= new_data_parallel_size
)
if rank_will_retire:
set_elastic_ep_rank_retired(True)
commit_succeeded = False
try:
await self.engine_core.commit_elastic_ep()
parallel_config.data_parallel_size = new_data_parallel_size
commit_succeeded = True
if rank_will_retire:
logger.info(
"[Elastic EP] Data-parallel rank %s is retired and awaiting "
"external process shutdown",
parallel_config.data_parallel_rank,
)
finally:
if rank_will_retire and not commit_succeeded:
set_elastic_ep_rank_retired(False)
# A failed commit may leave engines partially reconfigured or paused.
if commit_succeeded:
set_scaling_elastic_ep(False)
async def get_external_elastic_ep_status(
self,
) -> dict[str, str | int | None] | None:
return await self.engine_core.get_external_elastic_ep_status()
async def handle_fault(
self, fault_tolerance_request: FaultToleranceRequest
@@ -1308,6 +1368,10 @@ class AsyncLLM(EngineClient):
def errored(self) -> bool:
return self.engine_core.resources.engine_dead or not self.is_running
@property
def should_keep_api_server_alive(self) -> bool:
return get_elastic_ep_rank_retired()
@property
def dead_error(self) -> BaseException:
return EngineDeadError()
+1
View File
@@ -126,6 +126,7 @@ class DPCoordinator:
self.stats_publish_address = front_publish_address
self.coord_in_address = back_publish_address
self.coord_out_address = back_output_address
self._coord_store: object | None = None
self._finalizer = weakref.finalize(self, shutdown, [self.proc])
def get_stats_publish_address(self) -> str:
+50 -9
View File
@@ -1346,6 +1346,16 @@ class EngineCoreProc(EngineCore):
ready_msg["parallel_config_hash"] = (
vllm_config.parallel_config.compute_hash()
)
if vllm_config.parallel_config.enable_elastic_ep:
ready_msg["coord_store_port"] = (
vllm_config.parallel_config._coord_store_port
)
ready_msg["elastic_ep_max_dp_size"] = (
vllm_config.parallel_config.elastic_ep_max_dp_size
)
ready_msg["num_redundant_experts"] = (
vllm_config.parallel_config.eplb_config.num_redundant_experts
)
handshake_socket.send(msgspec.msgpack.encode(ready_msg))
@@ -1376,14 +1386,37 @@ class EngineCoreProc(EngineCore):
f"minutes"
)
init_bytes = handshake_socket.recv()
init_message: EngineHandshakeMetadata = msgspec.msgpack.decode(
init_bytes, type=EngineHandshakeMetadata
)
num_redundant_experts: int | None = None
if (
envs.VLLM_ELASTIC_EP_SCALE_UP_LAUNCH
and parallel_config is not None
and parallel_config.data_parallel_external_lb
):
from vllm.distributed.elastic_ep.external_elastic_ep import (
ExternalElasticEPScaleUpHandshakeMetadata,
)
scale_up_message = msgspec.msgpack.decode(
init_bytes, type=ExternalElasticEPScaleUpHandshakeMetadata
)
init_message = scale_up_message.engine_metadata
num_redundant_experts = scale_up_message.num_redundant_experts
parallel_config.elastic_ep_max_dp_size = (
scale_up_message.elastic_ep_max_dp_size
)
else:
init_message = msgspec.msgpack.decode(
init_bytes, type=EngineHandshakeMetadata
)
logger.debug("Received init message: %s", init_message)
if parallel_config is not None:
for key, value in init_message.parallel_config.items():
setattr(parallel_config, key, value)
if num_redundant_experts is not None:
parallel_config.eplb_config.num_redundant_experts = (
num_redundant_experts
)
return init_message.addresses
@@ -1826,6 +1859,9 @@ class EngineCoreProc(EngineCore):
if self.vllm_config.weight_transfer_config is not None
else False
),
coord_store_port=parallel_config._coord_store_port,
coordinator_input_address=self.addresses.coordinator_input,
coordinator_output_address=self.addresses.coordinator_output,
)
def process_input_sockets(
@@ -2342,6 +2378,13 @@ class DPEngineCoreProc(EngineCoreProc):
if state.is_complete():
if state.worker_type == "removing":
raise SystemExit
if (
state.worker_type == "new"
and self.vllm_config.parallel_config.data_parallel_external_lb
):
# An independently launched rank has no external utility
# caller to join the post-commit resume barrier.
self.resume_scheduler()
self.process_input_queue_block = True
self.eep_scaling_state = None
elif not state.commit_requested and state.is_ready_for_switch():
@@ -2512,12 +2555,10 @@ class DPEngineCoreProc(EngineCoreProc):
self.output_queue.put_nowait((0, outputs))
else:
encoder = MsgpackEncoder()
with (
zmq.Context() as ctx,
make_zmq_socket(
ctx, self.addresses.outputs[0], zmq.PUSH, linger=4000
) as socket,
):
ctx = zmq.Context.instance()
with make_zmq_socket(
ctx, self.addresses.outputs[0], zmq.PUSH, linger=4000
) as socket:
socket.send_multipart(encoder.encode(outputs))
def _eep_scale_up_before_kv_init(self):
+167 -58
View File
@@ -23,6 +23,9 @@ import zmq.asyncio
from vllm import envs
from vllm.config import VllmConfig
from vllm.config.kv_events import KVEventsConfig
from vllm.distributed.elastic_ep.external_elastic_ep import (
ExternalElasticEPScaleCoordinator,
)
from vllm.envs import VLLM_ENGINE_READY_TIMEOUT_S
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
@@ -271,6 +274,14 @@ class EngineCoreClient(ABC):
async def prepare_elastic_ep(self, new_data_parallel_size: int) -> None:
raise NotImplementedError
async def abort_external_elastic_ep(self, error: str) -> None:
raise NotImplementedError
async def get_external_elastic_ep_status(
self,
) -> dict[str, str | int | None] | None:
return None
async def get_output_async(self) -> EngineCoreOutputs:
raise NotImplementedError
@@ -626,6 +637,8 @@ class MPClient(EngineCoreClient):
enable_input_socket_handover = parallel_config.enable_elastic_ep
self.stats_update_address: str | None = None
self.coordinator_input_address: str | None = None
self.coordinator_output_address: str | None = None
tensor_queue: Queue | None = None
if client_addresses:
# Engines are managed externally to this client.
@@ -702,6 +715,11 @@ class MPClient(EngineCoreClient):
coordinator.get_stats_publish_address()
)
self.input_address = self.input_socket.getsockopt_string(zmq.LAST_ENDPOINT)
output_socket = self.resources.output_socket
assert output_socket is not None
self.output_address = output_socket.getsockopt_string(zmq.LAST_ENDPOINT)
# Serialization setup with tensor queues for multimodal tensor IPC.
tensor_ipc_sender: TensorIpcSender | None = None
model_config = getattr(vllm_config, "model_config", None)
@@ -885,6 +903,25 @@ class MPClient(EngineCoreClient):
response.kv_events_config
)
parallel_config = vllm_config.parallel_config
if response.coord_store_port:
if parallel_config._coord_store_port:
assert response.coord_store_port == parallel_config._coord_store_port
else:
parallel_config._coord_store_port = response.coord_store_port
for attribute in (
"coordinator_input_address",
"coordinator_output_address",
):
address = getattr(response, attribute)
if address is not None:
existing_address = getattr(self, attribute)
if existing_address is None:
setattr(self, attribute, address)
else:
assert address == existing_address
def get_kv_event_sources(self) -> dict[int, KVEventsConfig]:
return dict(self._kv_event_sources)
@@ -1431,6 +1468,12 @@ class DPAsyncMPClient(AsyncMPClient):
renderer: BaseRenderer | None = None,
):
self.current_wave = 0
parallel_config = vllm_config.parallel_config
self.external_eep_coordinator: ExternalElasticEPScaleCoordinator | None = None
if parallel_config.enable_elastic_ep and (
parallel_config.data_parallel_external_lb
):
self.external_eep_coordinator = ExternalElasticEPScaleCoordinator(self)
super().__init__(
vllm_config,
@@ -1514,23 +1557,19 @@ class DPAsyncMPClient(AsyncMPClient):
dp_rank = parallel_config.data_parallel_rank
assert dp_rank == 0
assert dp_size == new_engine_count
assert not (
parallel_config.data_parallel_hybrid_lb
or parallel_config.data_parallel_external_lb
)
num_ranks = dp_size
self.engine_ranks_managed = list(
range(dp_rank, dp_rank + num_ranks)
)
if len(self.lb_engines) < new_engine_count:
self.lb_engines = self.lb_engines + [
[0, 0, 0.0]
for _ in range(
new_engine_count - len(self.lb_engines)
)
]
else:
self.lb_engines = self.lb_engines[:new_engine_count]
assert not parallel_config.data_parallel_hybrid_lb
# External LB clients continue to manage only local ranks.
if not parallel_config.data_parallel_external_lb:
self.engine_ranks_managed = list(range(dp_size))
if len(self.lb_engines) < new_engine_count:
self.lb_engines += [
[0, 0, 0.0]
for _ in range(
new_engine_count - len(self.lb_engines)
)
]
else:
self.lb_engines = self.lb_engines[:new_engine_count]
# Send scale up notification to coordinator
scale_msg = msgspec.msgpack.encode(
("SCALE_ELASTIC_EP", new_engine_count)
@@ -1599,6 +1638,93 @@ class DPAsyncMPClient(AsyncMPClient):
def get_core_engine_for_request(self, request: EngineCoreRequest):
return self.core_engine
def _eep_commit_pause_mode(self) -> PauseMode:
from vllm.distributed.elastic_ep.elastic_execute import (
can_reuse_fused_moe_kernel,
)
# MRV2 re-warms through the request pool, so running requests must
# finish first while new ones stay queued in the scheduler.
parallel_config = self.vllm_config.parallel_config
if self.vllm_config.use_v2_model_runner and not can_reuse_fused_moe_kernel(
parallel_config
):
return "wait"
return "keep"
def _setup_elastic_ep_reconfig_bootstrap(self) -> tuple[str, int]:
from vllm.distributed.utils import create_tcp_store
from vllm.utils.network_utils import get_open_ports_list
parallel_config = self.vllm_config.parallel_config
parallel_config._data_parallel_master_port_list = get_open_ports_list(5)
parallel_config.data_parallel_master_port = (
parallel_config._data_parallel_master_port_list.pop()
)
ip = parallel_config.data_parallel_master_ip
store = create_tcp_store(
ip,
0,
is_master=True,
world_size=-1,
wait_for_workers=False,
)
parallel_config._coord_store_port = store.port
self._coord_store = store
return ip, store.port
@staticmethod
async def eep_process_engine_core_notification(
self: "DPAsyncMPClient", notification_data: tuple[str, int]
) -> None:
if self.external_eep_coordinator is not None:
await self.external_eep_coordinator.process_engine_core_notification(
notification_data
)
def _get_external_eep_coordinator(self) -> ExternalElasticEPScaleCoordinator:
parallel_config = self.vllm_config.parallel_config
if not parallel_config.enable_elastic_ep:
raise NotImplementedError(
"Elastic EP scaling requires enable_elastic_ep=True."
)
if not parallel_config.data_parallel_external_lb:
raise NotImplementedError(
"DPAsyncMPClient only supports Elastic EP scaling in external "
"load-balancer mode."
)
coordinator = self.external_eep_coordinator
assert coordinator is not None
return coordinator
async def prepare_elastic_ep(self, new_data_parallel_size: int) -> None:
cur_data_parallel_size = self.vllm_config.parallel_config.data_parallel_size
assert new_data_parallel_size != cur_data_parallel_size, (
f"new_data_parallel_size {new_data_parallel_size} must be "
f"different from cur_data_parallel_size {cur_data_parallel_size}"
)
await self._get_external_eep_coordinator().prepare(
cur_data_parallel_size, new_data_parallel_size
)
async def commit_elastic_ep(self) -> None:
await self._get_external_eep_coordinator().commit()
async def abort_external_elastic_ep(self, error: str) -> None:
self._get_external_eep_coordinator().abort(error)
async def get_external_elastic_ep_status(
self,
) -> dict[str, str | int | None] | None:
coordinator = self.external_eep_coordinator
return (
None
if coordinator is None
else await asyncio.to_thread(coordinator.get_status)
)
class DPLBAsyncMPClient(DPAsyncMPClient):
"""Asyncio-compatible client for multi-proc, multi-engine (data parallel)
@@ -1618,6 +1744,8 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
# To route aborts to the correct engine.
self.reqs_in_flight: dict[str, EngineIdentity] = {}
# Open streaming-input sessions, whose aborts go to all engines.
self.streaming_reqs: set[str] = set()
# Exact per-engine count of this client's unfinished requests.
self.engine_inflight: Counter[EngineIdentity] = Counter()
@@ -1640,6 +1768,12 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
) // client_count
def get_core_engine_for_request(self, request: EngineCoreRequest) -> EngineIdentity:
if request.resumable:
self.streaming_reqs.add(request.request_id)
else:
# Also removes a session on its final (end-of-stream) request.
self.streaming_reqs.discard(request.request_id)
if (engine := self.reqs_in_flight.get(request.request_id)) is not None:
# This is for streaming-input session affinity.
return engine
@@ -1711,8 +1845,9 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
@staticmethod
async def eep_process_engine_core_notification(
self: "DPLBAsyncMPClient", notification_data: tuple[str, int]
):
self: "DPAsyncMPClient", notification_data: tuple[str, int]
) -> None:
assert isinstance(self, DPLBAsyncMPClient)
cache = self.eep_scaling_cache
notification_type_str, dp_rank = notification_data
try:
@@ -1759,15 +1894,24 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
if not request_ids or self.resources.engine_dead:
return
if len(request_ids) == 1:
if (
len(request_ids) == 1
and (req_id := request_ids[0]) not in self.streaming_reqs
and (engine := self.reqs_in_flight.get(req_id))
):
# Fast-path common case.
if engine := self.reqs_in_flight.get(request_ids[0]):
await self._abort_requests(request_ids, engine)
await self._abort_requests(request_ids, engine)
return
by_engine = defaultdict[EngineIdentity, list[str]](list)
for req_id in request_ids:
if engine := self.reqs_in_flight.get(req_id):
if req_id in self.streaming_reqs:
# A chunk in flight when the session finished may have
# re-created it on any engine.
self.streaming_reqs.discard(req_id)
for engine in self.core_engines:
by_engine[engine].append(req_id)
elif engine := self.reqs_in_flight.get(req_id):
by_engine[engine].append(req_id)
for engine, req_ids in by_engine.items():
await self._abort_requests(req_ids, engine)
@@ -1864,27 +2008,6 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
new_engine_identities.discard(identity)
self._apply_ready_response(payload)
def _setup_elastic_ep_reconfig_bootstrap(self) -> None:
from vllm.distributed.utils import create_tcp_store
from vllm.utils.network_utils import get_open_ports_list
parallel_config = self.vllm_config.parallel_config
parallel_config._data_parallel_master_port_list = get_open_ports_list(5)
parallel_config.data_parallel_master_port = (
parallel_config._data_parallel_master_port_list.pop()
)
ip = parallel_config.data_parallel_master_ip
store = create_tcp_store(
ip,
0,
is_master=True,
world_size=-1,
wait_for_workers=False,
)
parallel_config._coord_store_port = store.port
self._coord_store = store
def _make_reconfig_request(
self,
new_data_parallel_size: int,
@@ -1939,20 +2062,6 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
await asyncio.to_thread(self._coord_store.wait, ready_keys)
logger.info("[Elastic EP] Successfully started new engines")
def _eep_commit_pause_mode(self) -> PauseMode:
from vllm.distributed.elastic_ep.elastic_execute import (
can_reuse_fused_moe_kernel,
)
# MRV2 re-warms through the request pool, so running requests must
# finish first while new ones stay queued in the scheduler.
parallel_config = self.vllm_config.parallel_config
if self.vllm_config.use_v2_model_runner and not can_reuse_fused_moe_kernel(
parallel_config
):
return "wait"
return "keep"
async def _commit_scale_up_elastic_ep(self, new_data_parallel_size: int) -> None:
new_core_engines = [
rank.to_bytes(2, "little")
+57 -5
View File
@@ -113,6 +113,53 @@ class RequestOutputCollector:
self._input_stream_task = None
def _is_chunk_boundary(
output: RequestOutput | PoolingRequestOutput | Exception | None,
) -> bool:
"""Whether the output marks the end of a streaming-input chunk: a
finish_reason on a request that has not finished."""
return (
isinstance(output, RequestOutput)
and not output.finished
and any(completion.finished() for completion in output.outputs)
)
class StreamingInputOutputCollector(RequestOutputCollector):
"""Collector for streaming-input requests. Doesn't merge across input
chunk boundaries (a finish_reason on a non-finished output)."""
def __init__(self, output_kind: RequestOutputKind, request_id: str):
super().__init__(output_kind, request_id)
self._backlog: deque[RequestOutput] = deque()
def put(self, output: RequestOutput | PoolingRequestOutput | Exception) -> None:
if isinstance(output, Exception):
self._backlog.clear()
elif self._backlog or _is_chunk_boundary(self.output):
assert isinstance(output, RequestOutput)
if self._backlog and not _is_chunk_boundary(self._backlog[-1]):
self._backlog[-1].add(output, aggregate=self.aggregate)
else:
self._backlog.append(output)
return
super().put(output)
async def get(self) -> RequestOutput | PoolingRequestOutput:
output = await super().get()
if self._backlog:
self.output = self._backlog.popleft()
self.ready.set()
return output
def get_nowait(self) -> RequestOutput | PoolingRequestOutput | None:
output = super().get_nowait()
if self._backlog:
self.output = self._backlog.popleft()
self.ready.set()
return output
@dataclass
class OutputProcessorOutput:
request_outputs: list[RequestOutput | PoolingRequestOutput]
@@ -784,6 +831,10 @@ class OutputProcessor:
# if required.
req_state.logprobs_processor.update_from_output(engine_core_output)
streaming_chunk_boundary = req_state.streaming_input and (
finish_reason not in (FinishReason.ABORT, FinishReason.ERROR)
)
# 4) Create and handle RequestOutput objects.
if request_output := req_state.make_request_output(
new_token_ids,
@@ -794,7 +845,7 @@ class OutputProcessor:
ec_transfer_params,
error=request_error,
):
if req_state.streaming_input:
if streaming_chunk_boundary:
request_output.finished = False
if req_state.queue is not None:
@@ -806,7 +857,7 @@ class OutputProcessor:
# Free completed requests.
if finish_reason is not None:
if req_state.streaming_input:
if streaming_chunk_boundary:
if req_state.input_chunk_queue:
update = req_state.input_chunk_queue.popleft()
req_state.apply_streaming_update(update)
@@ -814,9 +865,10 @@ class OutputProcessor:
req_state.input_chunk_queue = None
else:
self._finish_request(req_state)
if not engine_core_output.finished:
# If req not finished in EngineCore, but Detokenizer
# detected stop string, abort needed in EngineCore.
if req_state.streaming_input or not engine_core_output.finished:
# EngineCore may still have the request: a stop string
# was detected here, or an in-flight streaming-input
# chunk may re-create the session.
reqs_to_abort.append(req_id)
# Track per-request stats
+75 -14
View File
@@ -290,6 +290,7 @@ class CoreEngineProcManager:
self._finalizer = weakref.finalize(self, shutdown, self.processes)
self.manager_stopped = threading.Event()
self.failed_proc_name: str | None = None
self._coord_store: object | None = None
# All ranks share this config object: capture the user-provided
# --device-ids list before the per-rank shard overwrites it. Mutating
@@ -1139,7 +1140,9 @@ def get_engine_zmq_bind_addresses(
offline_mode or local_engines_only or (local_engine_count == dp_size)
)
# NOTE(yongji): handling scaling from intra-node to inter-node
if parallel_config.enable_elastic_ep:
if parallel_config.enable_elastic_ep and (
not local_engines_only or parallel_config.data_parallel_rank == 0
):
client_local_only = False
def _addr() -> str:
@@ -1228,6 +1231,32 @@ def launch_core_engines(
vllm_config.needs_dp_coordinator and not offline_mode and dp_rank == 0
)
coord_store = None
create_coord_store = (
parallel_config.enable_elastic_ep
and parallel_config.data_parallel_backend != "ray"
and dp_rank == 0
and not parallel_config._coord_store_port
)
if create_coord_store:
from vllm.distributed.utils import create_tcp_store
coord_store = create_tcp_store(
host,
0,
is_master=True,
world_size=-1,
wait_for_workers=False,
)
parallel_config._coord_store_port = coord_store.port
logger.info(
"Created Elastic EP coordination TCPStore on %s:%d",
host,
coord_store.port,
)
if run_coordinator:
coordinator = DPCoordinator(
parallel_config,
@@ -1243,12 +1272,13 @@ def launch_core_engines(
)
logger.info("Started DP Coordinator process (PID: %d)", coordinator.proc.pid)
if coord_store is not None:
coordinator._coord_store = coord_store
else:
coordinator = None
# Hold a coordination TCPStore (alive for this frame) so engines pick DP
# master ports at bind time; pre-allocated ports can be taken before use.
coord_store = None
if (
dp_size > 1
and not offline_mode
@@ -1344,6 +1374,13 @@ def launch_core_engines(
else:
local_engine_manager = None
if (
coord_store is not None
and coordinator is None
and (local_engine_manager is not None)
):
local_engine_manager._coord_store = coord_store
launch = CoreEngineLaunch(
local_engine_manager, coordinator, addresses, tensor_queue
)
@@ -1474,21 +1511,33 @@ def wait_for_engine_startup(
if status == "HELLO" and engine.state == CoreEngineState.NEW:
# Send init message with DP config info.
parallel_config_payload: dict[str, int | str | list[int]] = {}
if coordinated_dp and not (
parallel_config.local_engines_only
and parallel_config.data_parallel_rank > 0
):
parallel_config_payload = {
k: getattr(parallel_config, k)
for k in (
"data_parallel_master_ip",
"data_parallel_master_port",
"_data_parallel_master_port_list",
"_coord_store_port",
"data_parallel_size",
)
}
if (
parallel_config.enable_elastic_ep
and not parallel_config._coord_store_port
):
raise RuntimeError(
"Elastic EP startup requires a coordination TCPStore "
"port before engine handshake."
)
init_message = msgspec.msgpack.encode(
EngineHandshakeMetadata(
addresses=launch.addresses,
parallel_config={
k: getattr(parallel_config, k)
for k in (
"data_parallel_master_ip",
"data_parallel_master_port",
"_data_parallel_master_port_list",
"_coord_store_port",
"data_parallel_size",
)
}
if coordinated_dp
else {},
parallel_config=parallel_config_payload,
)
)
handshake_socket.send_multipart((eng_identity, init_message), copy=False)
@@ -1496,6 +1545,18 @@ def wait_for_engine_startup(
start_pending[0 if local else 1] += 1
engine.state = CoreEngineState.CONNECTED
elif status == "READY" and engine.state == CoreEngineState.CONNECTED:
coord_store_port = msg.get("coord_store_port")
if coord_store_port is not None and local:
parallel_config._coord_store_port = coord_store_port
num_redundant_experts = msg.get("num_redundant_experts")
if num_redundant_experts is not None and local:
parallel_config.eplb_config.num_redundant_experts = (
num_redundant_experts
)
elastic_ep_max_dp_size = msg.get("elastic_ep_max_dp_size")
if elastic_ep_max_dp_size is not None and local:
parallel_config.elastic_ep_max_dp_size = elastic_ep_max_dp_size
# Validate config hash consistency across DP workers for MoE models.
if coordinated_dp:
worker_config_hash = msg.get("parallel_config_hash")
+1 -1
View File
@@ -292,7 +292,7 @@ class RayExecutorV2(MultiprocExecutor):
# Step 1: Initialize Ray cluster and retrieve placement group
if ray is None:
raise ImportError("Using Ray backend requires installation of ray.")
initialize_ray_cluster(self.parallel_config, require_gpu_on_driver=False)
initialize_ray_cluster(self.parallel_config)
placement_group = self.parallel_config.placement_group
tp_size, pp_size, pcp_size = self._get_parallel_sizes()
+2 -55
View File
@@ -8,7 +8,6 @@ from collections import defaultdict
from vllm.config import ParallelConfig
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_ip
logger = init_logger(__name__)
PG_WAIT_TIMEOUT = 1800
@@ -30,14 +29,6 @@ try:
from ray.util import placement_group_table
from ray.util.placement_group import PlacementGroup
try:
from ray._private.state import available_resources_per_node
except ImportError:
# Ray 2.9.x doesn't expose `available_resources_per_node`
from ray._private.state import state as _state
available_resources_per_node = _state._available_resources_per_node
ray_import_err = None
except ImportError as e:
@@ -65,15 +56,8 @@ def _verify_bundles(
placement_group: "PlacementGroup",
parallel_config: ParallelConfig,
device_str: str,
require_gpu_on_driver: bool = True,
):
"""Verify a given placement group has bundles located in the right place.
There are 2 rules.
- Warn if all tensor parallel workers cannot fit in a single node.
- Fail if driver node is not included in a placement group
(only when require_gpu_on_driver is True).
"""
"""Warn if all tensor parallel workers cannot fit in a single node."""
assert ray.is_initialized(), (
"Ray is not initialized although distributed-executor-backend is ray."
)
@@ -87,17 +71,6 @@ def _verify_bundles(
for bundle_idx, node_id in bundle_to_node_ids.items():
node_id_to_bundle[node_id].append(bundles[bundle_idx])
driver_node_id = ray.get_runtime_context().get_node_id()
if require_gpu_on_driver and driver_node_id not in node_id_to_bundle:
raise RuntimeError(
f"driver node id {driver_node_id} is not included in a placement "
f"group {placement_group.id}. Node id -> bundles "
f"{node_id_to_bundle}. "
"You don't have enough GPUs available in a current node. Check "
"`ray status` and `ray list nodes` to see if you have available "
"GPUs in a node `{driver_node_id}` before starting an vLLM engine."
)
for node_id, bundles in node_id_to_bundle.items():
if len(bundles) < parallel_config.tensor_parallel_size:
@@ -293,7 +266,6 @@ def _wait_until_pg_ready(current_placement_group: "PlacementGroup"):
def initialize_ray_cluster(
parallel_config: ParallelConfig,
ray_address: str | None = None,
require_gpu_on_driver: bool = True,
):
"""Initialize the distributed cluster with Ray.
@@ -305,10 +277,6 @@ def initialize_ray_cluster(
parallel_config: The configurations for parallel execution.
ray_address: The address of the Ray cluster. If None, uses
the default Ray cluster address.
require_gpu_on_driver: If True (default), require at least one GPU
on the current (driver) node and pin the first PG bundle to it.
Set to False for executors like RayExecutorV2 where all GPU work
is delegated to remote Ray actors.
"""
assert_ray_available()
@@ -404,25 +372,6 @@ def initialize_ray_cluster(
{device_str: 1.0} for _ in range(parallel_config.world_size)
]
# vLLM engine is also a worker to execute model with an accelerator,
# so it requires to have the device in a current node. Check if
# the current node has at least one device.
current_ip = get_ip()
current_node_id = ray.get_runtime_context().get_node_id()
current_node_resource = available_resources_per_node()[current_node_id]
if require_gpu_on_driver:
if current_node_resource.get(device_str, 0) < 1:
raise ValueError(
f"Current node has no {device_str} available. "
f"{current_node_resource=}. vLLM engine cannot start "
f"without {device_str}. Make sure you have at least 1 "
f"{device_str} available in a node "
f"{current_node_id=} {current_ip=}."
)
# This way, at least bundle is required to be created in a
# current node.
placement_group_specs[0][f"node:{current_ip}"] = 0.001
# By default, Ray packs resources as much as possible.
current_placement_group = ray.util.placement_group(
placement_group_specs, strategy="PACK"
@@ -430,8 +379,6 @@ def initialize_ray_cluster(
_wait_until_pg_ready(current_placement_group)
assert current_placement_group is not None
_verify_bundles(
current_placement_group, parallel_config, device_str, require_gpu_on_driver
)
_verify_bundles(current_placement_group, parallel_config, device_str)
# Set the placement group in the parallel config
parallel_config.placement_group = current_placement_group
+11 -29
View File
@@ -17,6 +17,8 @@ from vllm.v1.hisparse.layout import (
from vllm.v1.hisparse.runtime import (
HiSparseCacheHandle,
HiSparseHostPool,
drop_hisparse_host_pool,
get_hisparse_host_pool,
initialize_hisparse_runtime_buffers,
release_pinned_state,
)
@@ -126,7 +128,7 @@ def init_hisparse_kv_cache(
block_tables: "BlockTables",
) -> dict[str, torch.Tensor]:
"""Allocate and bind HiSparse caches within the caller's allocation context."""
host_pool = HiSparseHostPool(vllm_config, kv_cache_config)
host_pool = get_hisparse_host_pool(vllm_config, kv_cache_config)
try:
kv_caches = allocate_hisparse_kv_caches(
kv_cache_config,
@@ -150,6 +152,9 @@ def init_hisparse_kv_cache(
except Exception:
if host_pool.shared_region is not None:
host_pool.shared_region.cleanup()
elif host_pool.registered is not None:
release_pinned_state([], [host_pool.registered])
drop_hisparse_host_pool(vllm_config)
raise
@@ -163,37 +168,14 @@ def _get_hisparse_cache(
def release_hisparse_profiling_cache(forward_context: dict[str, Any]) -> None:
cache_handles = [
cache
for layer in forward_context.values()
if (cache := getattr(layer, "hisparse_cache", None)) is not None
]
"""Detach the profiling caches but keep the host pool for the real KV cache."""
runtimes = {
id(cache.runtime): cache.runtime
for cache in cache_handles
if hasattr(cache.runtime, "_host_cache")
for layer in forward_context.values()
if (cache := getattr(layer, "hisparse_cache", None)) is not None
and hasattr(cache.runtime, "_host_cache")
}
if not runtimes:
return
shared_regions = {
id(runtime.shared_host_region): runtime.shared_host_region
for runtime in runtimes.values()
if runtime.shared_host_region is not None
}
assert len(shared_regions) <= 1
shared_region = next(iter(shared_regions.values()), None)
registered_pools = (
[]
if shared_region is not None
else list(
{
runtime.registered_host_pool.data_ptr(): runtime.registered_host_pool
for runtime in runtimes.values()
}.values()
)
)
release_pinned_state(list(runtimes.values()), registered_pools, shared_region)
release_pinned_state(list(runtimes.values()), [])
def bind_hisparse_kv_caches(
+20 -1
View File
@@ -223,7 +223,9 @@ class HiSparseHostPool:
self.shared_region: SharedOffloadRegion | None = None
def allocate(self, size: int) -> torch.Tensor:
assert self.backing is None, "HiSparse host pool is already allocated."
if self.backing is not None:
assert self.backing.numel() == size, "HiSparse host pool size changed."
return self.backing
config = self.kv_cache_config
if config.hisparse_shared_host_pool:
tensor_sizes = [
@@ -253,6 +255,23 @@ class HiSparseHostPool:
return self.backing
_host_pools: dict[str, HiSparseHostPool] = {}
def get_hisparse_host_pool(
vllm_config: VllmConfig, kv_cache_config: KVCacheConfig
) -> HiSparseHostPool:
"""Return the engine's host pool, which outlives the profiling KV cache."""
instance_id = vllm_config.instance_id
if instance_id not in _host_pools:
_host_pools[instance_id] = HiSparseHostPool(vllm_config, kv_cache_config)
return _host_pools[instance_id]
def drop_hisparse_host_pool(vllm_config: VllmConfig) -> None:
_host_pools.pop(vllm_config.instance_id, None)
def use_shared_hisparse_host_pool(vllm_config: VllmConfig) -> bool:
"""Whether replicated MLA host KV can share one local mmap."""
parallel = vllm_config.parallel_config
@@ -1,11 +1,14 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import contextlib
import ctypes
import errno
import functools
import mmap
import os
import time
from collections.abc import Callable, Iterator
from ctypes import c_int, c_long, c_size_t, c_void_p
import numpy as np
import torch
@@ -21,6 +24,10 @@ logger = init_logger(__name__)
# MADV_POPULATE_WRITE was added in Linux 5.14 (value 23).
_MADV_POPULATE_WRITE = getattr(mmap, "MADV_POPULATE_WRITE", 23)
_MAP_FIXED = 0x10
# Page-cache inserts are serialized per shm file, capping pre-faulting of one
# file near 10 GiB/s; smaller segments are not worth a file.
_MIN_SEGMENT_BYTES = 1 << 30
def _wait_for_file_size(fd: int, expected_size: int, timeout: float = 30.0) -> None:
@@ -40,6 +47,14 @@ def _wait_for_file_size(fd: int, expected_size: int, timeout: float = 30.0) -> N
time.sleep(0.005)
@functools.cache
def _libc_mmap() -> Callable[..., int | None]:
mmap_fn = ctypes.CDLL(None, use_errno=True).mmap
mmap_fn.argtypes = [c_void_p, c_size_t, c_int, c_int, c_int, c_long]
mmap_fn.restype = c_void_p
return mmap_fn
def _madvise_populate_write(mmap_obj: mmap.mmap, offset: int, length: int) -> None:
mmap_obj.madvise(_MADV_POPULATE_WRITE, offset, length)
@@ -84,6 +99,7 @@ class SharedOffloadRegion:
``populate_shard=(index, count)`` pre-faults only this worker's contiguous
share, for layouts where every worker maps the same bytes. The barrier
also waits for all shards to finish pre-faulting before returning.
``num_segments`` splits backing files beneath the contiguous mapping.
"""
BLOCK_SIZE_ALIGNMENT: int = mmap.PAGESIZE
@@ -99,6 +115,7 @@ class SharedOffloadRegion:
*,
creator_memory_check: Callable[[int], None] | None = None,
populate_shard: tuple[int, int] | None = None,
num_segments: int = 1,
) -> None:
self.page_size = mmap.PAGESIZE
assert kv_bytes_per_chunk % self.page_size == 0
@@ -109,6 +126,17 @@ class SharedOffloadRegion:
self.cpu_page_size = cpu_page_size
self.mmap_path = f"/dev/shm/vllm_offload_{engine_id}.mmap"
num_segments = min(num_segments, self.total_size_bytes // _MIN_SEGMENT_BYTES)
rows = max(1, cdiv(num_chunks, max(1, num_segments)))
# (path, offset, length) per file.
self._segments = [
(
f"{self.mmap_path}.{i}" if i else self.mmap_path,
row * self._row_stride,
min(rows, num_chunks - row) * self._row_stride,
)
for i, row in enumerate(range(0, max(num_chunks, 1), rows))
]
self._creator = False # set True only if this worker creates the file
self.rank = rank
if rank is not None:
@@ -126,7 +154,7 @@ class SharedOffloadRegion:
# Joiner path — another worker won O_EXCL. Reopen and wait
# for the file to reach expected size.
self.fd = os.open(self.mmap_path, os.O_RDWR)
_wait_for_file_size(self.fd, self.total_size_bytes)
_wait_for_file_size(self.fd, self._segments[0][2])
logger.info("Opened existing mmap file %s", self.mmap_path)
else:
# Creator path. We won O_EXCL, so we own the file: any
@@ -140,24 +168,32 @@ class SharedOffloadRegion:
self.total_size_bytes,
allocation_name="CPU KV offload shared region in /dev/shm",
)
os.ftruncate(self.fd, self.total_size_bytes)
for path, _, length in self._segments[1:]:
fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600)
os.ftruncate(fd, length)
os.close(fd)
# Joiners wait for the first file, so size it last.
os.ftruncate(self.fd, self._segments[0][2])
logger.info(
"Created mmap file %s (%.2f GB)",
self.mmap_path,
self.total_size_bytes / 1e9,
)
single = len(self._segments) == 1
self.mmap_obj: mmap.mmap | None = mmap.mmap(
self.fd,
self.fd if single else -1,
self.total_size_bytes,
flags=mmap.MAP_SHARED,
flags=mmap.MAP_SHARED if single else mmap.MAP_PRIVATE,
prot=mmap.PROT_READ | mmap.PROT_WRITE,
)
if not single:
self._map_segments()
if barrier is not None:
barriers_remaining -= 1
barrier()
if self._creator:
os.unlink(self.mmap_path)
self._unlink()
self._creator = False
logger.info("Unlinked mmap file %s", self.mmap_path)
if populate_shard is not None:
@@ -173,8 +209,7 @@ class SharedOffloadRegion:
barrier()
except Exception:
if self._creator:
with contextlib.suppress(FileNotFoundError):
os.unlink(self.mmap_path)
self._unlink()
self._creator = False
if hasattr(self, "mmap_obj") and self.mmap_obj is not None:
# The prefault traceback may retain an exported buffer.
@@ -225,11 +260,42 @@ class SharedOffloadRegion:
"MADV_POPULATE_WRITE entire region: %.3f s", time.perf_counter() - _t0
)
def _map_segments(self) -> None:
"""Replace the anonymous mapping with the segment files, back to back."""
assert self.mmap_obj is not None and self.fd is not None
base = ctypes.addressof(ctypes.c_char.from_buffer(self.mmap_obj))
for path, offset, length in self._segments:
fd = self.fd if offset == 0 else os.open(path, os.O_RDWR)
address = _libc_mmap()(
base + offset,
length,
mmap.PROT_READ | mmap.PROT_WRITE,
mmap.MAP_SHARED | _MAP_FIXED,
fd,
0,
)
if fd != self.fd:
os.close(fd)
if address != base + offset:
raise OSError(ctypes.get_errno(), f"Failed to map {path}")
def _unlink(self) -> None:
for path, _, _ in self._segments:
with contextlib.suppress(FileNotFoundError):
os.unlink(path)
def iter_worker_slot_ranges(self) -> Iterator[tuple[int, int]]:
"""Yield this worker's slots as page-aligned (offset, length) ranges."""
assert self.rank is not None
worker_offset = self.rank * self.cpu_page_size
for row in range(self.num_chunks):
first = 0
if len(self._segments) > 1:
# Start each worker on a different file.
first = (
self.rank * self.num_chunks // (self._row_stride // self.cpu_page_size)
)
for i in range(self.num_chunks):
row = (first + i) % self.num_chunks
start = row * self._row_stride + worker_offset
aligned = start - start % self.page_size
end = round_up(start + self.cpu_page_size, self.page_size)
@@ -380,7 +446,7 @@ class SharedOffloadRegion:
self.fd = None
if self._creator and getattr(self, "mmap_path", None):
try:
os.unlink(self.mmap_path)
self._unlink()
logger.info("Removed mmap file %s", self.mmap_path)
except Exception:
logger.warning(
+5
View File
@@ -22,6 +22,10 @@ from vllm.v1.kv_offload.cpu.gpu_worker import CPUOffloadingWorker
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
# Split the shared region into this many shm files so workers pre-fault them
# in parallel (page-cache inserts are serialized per file).
_REGION_SEGMENTS = 32
def _all_workers_barrier() -> None:
"""Block until every worker rank has reached this point (gloo cpu group).
@@ -188,6 +192,7 @@ class CPUOffloadingSpec(OffloadingSpec):
kv_bytes_per_chunk=self.kv_bytes_per_chunk,
cpu_page_size=self.cpu_page_size_per_worker,
barrier=_all_workers_barrier,
num_segments=_REGION_SEGMENTS,
populate_shard=(
(worker_index, world_size) if self.replicated_layout else None
),
+42 -6
View File
@@ -34,11 +34,34 @@ def register_top_k_top_p_warmups() -> None:
_topp_split_mask.register_warmup()
def _skip_aiter_sampler_on_gfx1250() -> bool:
# Lazy ROCm-only import; keeps arch detection out of import time on CUDA/CPU.
from vllm.platforms.rocm import on_gfx1250
def _aiter_temp_gumbel_sample(
logits: torch.Tensor, generators: dict[int, torch.Generator]
) -> torch.Tensor:
"""Fused temperature Gumbel-max sampling via aiter.
return on_gfx1250()
Logits are already temperature-scaled by the parent Sampler, so temps=1.
Returns int64 token ids of shape (num_tokens,).
"""
from aiter import mixed_sample_outer_exponential
n, vocab = logits.shape
out = torch.empty(n, dtype=torch.int32, device=logits.device)
if not generators:
# Common case: shared (1, vocab) exponential noise (cheap RNG).
exp = (
torch.empty((1, vocab), dtype=torch.float32, device=logits.device)
.exponential_()
.expand(n, vocab)
)
else:
exp = torch.empty((n, vocab), dtype=torch.float32, device=logits.device)
if len(generators) != n:
exp.exponential_()
for i, g in generators.items():
exp[i].exponential_(generator=g)
temps = torch.ones(n, dtype=torch.float32, device=logits.device)
mixed_sample_outer_exponential(out, logits, exp, temps, eps=1e-10)
return out.to(torch.int64)
def _flashinfer_jit_unsupported_reason(capability: DeviceCapability) -> str | None:
@@ -246,8 +269,12 @@ class TopKTopPSampler(nn.Module):
elif (
logprobs_mode not in PROCESSED_LOGPROBS_MODES
and rocm_aiter_ops.is_enabled()
and not _skip_aiter_sampler_on_gfx1250() # TODO (JPVILLAM): Enable
):
# Lazy ROCm-only import; keeps arch detection out of import time
# on CUDA/CPU.
from vllm.platforms.rocm import on_gfx1250
self.on_gfx1250 = on_gfx1250()
self.aiter_ops = None
self._aiter_ops_import_failed = False
logger.info_once(
@@ -375,7 +402,16 @@ class TopKTopPSampler(nn.Module):
k: torch.Tensor | None,
p: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
"""Optimized ROCm/aiter path (same structure as forward_cuda)."""
"""Optimized ROCm/aiter path."""
if self.on_gfx1250:
if (
k is None
and p is None
and not self.use_fp64_gumbel
and self.logprobs_mode not in PROCESSED_LOGPROBS_MODES
):
return _aiter_temp_gumbel_sample(logits, generators), None
return self.forward_native(logits, generators, k, p)
if (k is None and p is None) or generators:
if generators:
logger.warning_once(
+61
View File
@@ -5,6 +5,7 @@
import torch
import torch.nn as nn
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config.model import LogprobsMode
from vllm.platforms import current_platform
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
@@ -19,6 +20,61 @@ from vllm.v1.sample.ops.topk_topp_sampler import TopKTopPSampler
_SAMPLING_EPS = 1e-5
def mixed_sample_eligible(sm: SamplingMetadata, predict_bonus_token: bool) -> bool:
"""Whether the fused aiter sample-from-raw-logits kernel can be used."""
if not rocm_aiter_ops.is_enabled():
return False
from vllm.platforms.rocm import on_gfx1250
if not on_gfx1250():
return False
if predict_bonus_token:
return False
if not sm.all_random or sm.temperature is None:
return False
if sm.top_k is not None or sm.top_p is not None:
return False
if sm.max_num_logprobs is not None or sm.logprob_token_ids:
return False
if (
not sm.no_penalties
or sm.allowed_token_ids_mask is not None
or sm.bad_words_token_ids
):
return False
lp = sm.logitsprocs
if lp.argmax_invariant or lp.non_argmax_invariant:
return False
h = sm.thinking_budget_state_holder
return h is None or not h.has_tracked_requests()
def _aiter_raw_fused_sample(logits: torch.Tensor, sm: SamplingMetadata) -> torch.Tensor:
"""Sample token ids from raw (unscaled) logits via a fused aiter kernel.
Returns int32 ids of shape (num_tokens, 1).
"""
from aiter import mixed_sample_outer_exponential
n, vocab = logits.shape
out = torch.empty(n, dtype=torch.int32, device=logits.device)
gens = sm.generators
if not gens:
exp = (
torch.empty((1, vocab), dtype=torch.float32, device=logits.device)
.exponential_()
.expand(n, vocab)
)
else:
exp = torch.empty((n, vocab), dtype=torch.float32, device=logits.device)
exp.exponential_()
for i, g in gens.items():
exp[i].exponential_(generator=g)
mixed_sample_outer_exponential(out, logits, exp, sm.temperature, eps=1e-10)
return out.unsqueeze(-1)
class Sampler(nn.Module):
"""A layer that samples the next tokens from the model's outputs
with the following steps in order:
@@ -77,6 +133,11 @@ class Sampler(nn.Module):
predict_bonus_token: bool = False,
logprobs_mode_override: LogprobsMode | None = None,
) -> SamplerOutput:
if mixed_sample_eligible(sampling_metadata, predict_bonus_token):
return SamplerOutput(
sampled_token_ids=_aiter_raw_fused_sample(logits, sampling_metadata),
logprobs_tensors=None,
)
logprobs_mode = logprobs_mode_override or self.logprobs_mode
# NOTE(woosuk): Use the original logits (before any penalties or
# temperature scaling) for the top-k logprobs.
+58 -7
View File
@@ -5,6 +5,8 @@ from collections.abc import Iterable, Sequence
from concurrent.futures import Future, ThreadPoolExecutor
from typing import TYPE_CHECKING, NamedTuple
import numpy as np
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.parser.engine.adapters import ParserEngineReasoningAdapter
@@ -20,7 +22,6 @@ from vllm.v1.structured_output.backend_xgrammar import XgrammarBackend
from vllm.v1.structured_output.utils import strip_speculative_padding
if TYPE_CHECKING:
import numpy as np
import numpy.typing as npt
import torch
@@ -211,10 +212,16 @@ class StructuredOutputManager:
raise
def _fill_bitmasks(
self, batch: Iterable[tuple[StructuredOutputGrammar, int, bool]]
self,
batch: Iterable[tuple[StructuredOutputGrammar, int, bool, tuple[int, ...]]],
) -> None:
"""Fill one bitmask row per entry.
A row the grammar does not constrain allows every token except
`masked_ids`, the implicit reasoning ends masked while reasoning.
"""
assert self._grammar_bitmask is not None
for grammar, index, apply_bitmask in batch:
for grammar, index, apply_bitmask, masked_ids in batch:
if apply_bitmask and not grammar.is_terminated():
grammar.fill_bitmask(self._grammar_bitmask, index)
else:
@@ -222,12 +229,41 @@ class StructuredOutputManager:
# reset the relevant part of the bitmask for consequent
# requests here.
self._grammar_bitmask[index].fill_(self._full_mask)
if masked_ids:
row = self._grammar_bitmask[index].numpy().view(np.uint32)
for token_id in masked_ids:
row[token_id >> 5] &= ~np.uint32(1 << (token_id & 31))
def _async_submit_fill_bitmask(
self, batch: list[tuple[StructuredOutputGrammar, int, bool]]
self, batch: list[tuple[StructuredOutputGrammar, int, bool, tuple[int, ...]]]
) -> Future:
return self.executor_for_fillmask.submit(self._fill_bitmasks, batch)
def _masked_reasoning_ends(self, request: "Request") -> tuple[int, ...]:
"""Implicit reasoning-end tokens the grammar rejects as first content.
Such a token (e.g. a tool-call start) ends reasoning and is also the
first content token, so sampling one where the grammar cannot start
with it would make the output invalid. These stay masked while the
request reasons. The grammar does not advance before reasoning ends,
so the set is fixed per request.
"""
structured_req = request.structured_output_request
assert structured_req is not None
if structured_req.masked_reasoning_end_ids is None:
masked: tuple[int, ...] = ()
reasoner = self._get_reasoner(request)
if isinstance(reasoner, ParserEngineReasoningAdapter):
grammar = structured_req.grammar
assert isinstance(grammar, StructuredOutputGrammar)
masked = tuple(
token_id
for token_id in sorted(reasoner.implicit_reasoning_end_token_ids)
if not grammar.validate_tokens([token_id])
)
structured_req.masked_reasoning_end_ids = masked
return structured_req.masked_reasoning_end_ids
def _get_constraint_bounds(
self,
request: "Request",
@@ -377,7 +413,10 @@ class StructuredOutputManager:
apply_bitmask = (
self._get_constraint_bounds(request, ()).constraint_start == 0
)
batch.append((grammar, cumulative_index, apply_bitmask))
masked_ids = (
() if apply_bitmask else self._masked_reasoning_ends(request)
)
batch.append((grammar, cumulative_index, apply_bitmask, masked_ids))
if len(batch) == self.fill_bitmask_parallel_batch_size:
promises.append(self._async_submit_fill_bitmask(batch))
batch = []
@@ -405,6 +444,10 @@ class StructuredOutputManager:
grammar_start, constraint_start = self._get_constraint_bounds(
request, strip_speculative_padding(req_tokens)
)
# Rows before constraint_start are still reasoning.
masked_ids = (
self._masked_reasoning_ends(request) if constraint_start else ()
)
state_advancements = 0
seen_padding = False
# Row filled from the last valid grammar state before a draft
@@ -417,7 +460,11 @@ class StructuredOutputManager:
cumulative_index += 1
continue
apply_bitmask = not seen_padding and i >= constraint_start
self._fill_bitmasks(((grammar, cumulative_index, apply_bitmask),))
reasoning = not seen_padding and i < constraint_start
row_masked = masked_ids if reasoning else ()
self._fill_bitmasks(
((grammar, cumulative_index, apply_bitmask, row_masked),)
)
if token == -1:
seen_padding = True
elif not seen_padding and i >= grammar_start:
@@ -444,7 +491,11 @@ class StructuredOutputManager:
bonus_apply = not seen_padding and constraint_start <= len(
req_tokens
)
self._fill_bitmasks(((grammar, cumulative_index, bonus_apply),))
bonus_reasoning = not seen_padding and not bonus_apply
bonus_masked = masked_ids if bonus_reasoning else ()
self._fill_bitmasks(
((grammar, cumulative_index, bonus_apply, bonus_masked),)
)
cumulative_index += 1
if state_advancements > 0:
+3
View File
@@ -28,6 +28,9 @@ class StructuredOutputRequest:
# Cached per request; do not share reasoning parsers across requests because
# their behavior can depend on reasoning_parser_kwargs.
reasoner: "ReasoningParser | None" = None
# Implicit reasoning-end tokens the grammar rejects as the first content
# token; masked while the request reasons. Computed on first use.
masked_reasoning_end_ids: tuple[int, ...] | None = None
@staticmethod
def from_sampling_params(
+37
View File
@@ -14,6 +14,7 @@ from vllm.v1.watermarking.detector import (
)
from vllm.v1.watermarking.prfs import PhiloxPRF, WatermarkPRF, create_prf
from vllm.v1.watermarking.watermarker import (
DraftContextState,
RandomSampler,
SupportsSpeculativeDecoding,
Watermarker,
@@ -101,6 +102,42 @@ class GumbelWatermarker(Watermarker):
)
return WatermarkSample(token_ids, logits)
def _try_sample_block(
self,
logits: torch.Tensor,
num_steps: int,
random_sampler: RandomSampler,
state: DraftContextState,
) -> torch.Tensor | None:
if type(self.prf) is not PhiloxPRF or logits.device.type != "cuda":
return None
from vllm.v1.worker.gpu.sample.watermark import draft_philox_gumbel_sample
assert random_sampler.logits_cache is not None
assert random_sampler.logits_cache_col is not None
return draft_philox_gumbel_sample(
logits,
state.contexts,
self.prf.key,
num_steps=num_steps,
expanded_idx_mapping=random_sampler.expanded_idx_mapping,
temperatures=random_sampler.temperatures,
seeds=random_sampler.seeds,
positions=random_sampler.positions,
enabled=state.enabled,
logits_cache=random_sampler.logits_cache,
logits_cache_col=random_sampler.logits_cache_col,
use_fp64=random_sampler.use_fp64,
prior_contexts=state.prior_contexts,
all_token_ids=state.all_token_ids,
prompt_lens=state.prompt_lens,
total_lens=state.total_lens,
deduplicate=state.deduplicate_contexts != "none",
max_history=state.deduplicate_contexts_max_history,
include_prompt=state.deduplicate_contexts == "all",
)
class DualKeyGumbelWatermarker(Watermarker, SupportsSpeculativeDecoding):
def __init__(
+111 -34
View File
@@ -8,6 +8,7 @@ from vllm.v1.watermarking.factory import create_watermarker
from vllm.v1.watermarking.gumbel import GumbelWatermarker
from vllm.v1.watermarking.prfs import PhiloxPRF
from vllm.v1.watermarking.watermarker import (
DraftContextState,
RandomSampler,
SupportsSpeculativeDecoding,
Watermarker,
@@ -28,25 +29,25 @@ class DraftWatermarker:
) -> None:
self.watermarker = watermarker
self.num_speculative_steps = num_speculative_steps
self.deduplicate_contexts = deduplicate_contexts
self.deduplicate_contexts_max_history = deduplicate_contexts_max_history
self.contexts = torch.zeros(
max_num_reqs,
watermarker.context_width,
dtype=torch.int64,
device=device,
self.state = DraftContextState(
contexts=torch.zeros(
max_num_reqs,
watermarker.context_width,
dtype=torch.int64,
device=device,
),
enabled=torch.zeros(max_num_reqs, dtype=torch.bool, device=device),
prior_contexts=torch.zeros(
max_num_reqs,
num_speculative_steps,
watermarker.context_width,
dtype=torch.int64,
device=device,
),
prompt_lens=torch.zeros(max_num_reqs, dtype=torch.int32, device=device),
deduplicate_contexts=deduplicate_contexts,
deduplicate_contexts_max_history=deduplicate_contexts_max_history,
)
self.enabled = torch.zeros(max_num_reqs, dtype=torch.bool, device=device)
self.prior_contexts = torch.zeros(
max_num_reqs,
num_speculative_steps,
watermarker.context_width,
dtype=torch.int64,
device=device,
)
self.prompt_lens = torch.zeros(max_num_reqs, dtype=torch.int32, device=device)
self.all_token_ids: torch.Tensor | None = None
self.total_lens: torch.Tensor | None = None
def prepare(
self,
@@ -57,11 +58,12 @@ class DraftWatermarker:
total_lens: torch.Tensor,
) -> None:
num_reqs = contexts.shape[0]
self.contexts[:num_reqs].copy_(contexts)
self.enabled[:num_reqs].copy_(enabled)
self.all_token_ids = all_token_ids
self.prompt_lens.copy_(prompt_lens)
self.total_lens = total_lens
state = self.state
state.contexts[:num_reqs].copy_(contexts)
state.enabled[:num_reqs].copy_(enabled)
state.all_token_ids = all_token_ids
state.prompt_lens.copy_(prompt_lens)
state.total_lens = total_lens
def _sampling_state(
self,
@@ -75,22 +77,23 @@ class DraftWatermarker:
steps = torch.as_tensor(draft_step, device=logits.device, dtype=torch.int64)
if steps.ndim == 0:
steps = steps.expand(num_rows)
enabled = self.enabled[:num_rows] & (request_temperatures != 0)
if self.deduplicate_contexts == "none":
state = self.state
enabled = state.enabled[:num_rows] & (request_temperatures != 0)
if state.deduplicate_contexts == "none":
return steps, enabled
assert self.all_token_ids is not None
assert self.total_lens is not None
assert state.all_token_ids is not None
assert state.total_lens is not None
enabled = draft_watermarking_mask(
self.all_token_ids,
state.all_token_ids,
idx_mapping,
self.prompt_lens,
self.total_lens,
self.prior_contexts,
state.prompt_lens,
state.total_lens,
state.prior_contexts,
contexts,
steps,
enabled,
self.deduplicate_contexts_max_history,
include_prompt=self.deduplicate_contexts == "all",
state.deduplicate_contexts_max_history,
include_prompt=state.deduplicate_contexts == "all",
)
return steps, enabled
@@ -107,14 +110,26 @@ class DraftWatermarker:
logits_cache_col: torch.Tensor | None = None,
use_fp64: bool = False,
logits_cache_source: torch.Tensor | None = None,
num_steps: int = 1,
) -> torch.Tensor:
if num_steps > 1:
return self.sample_block(
logits,
idx_mapping,
temperature,
seed,
pos,
logits_cache=logits_cache,
logits_cache_col=logits_cache_col,
use_fp64=use_fp64,
)
assert apply_temperature
assert is_drafting
assert logits_cache is not None
assert logits_cache_col is not None
num_rows = logits.shape[0]
request_temperatures = temperature[idx_mapping]
contexts = self.contexts[:num_rows]
contexts = self.state.contexts[:num_rows]
steps, enabled = self._sampling_state(
logits, idx_mapping, request_temperatures, logits_cache_col, contexts
)
@@ -145,6 +160,68 @@ class DraftWatermarker:
contexts.copy_(torch.cat((contexts[:, 1:], sampled.unsqueeze(-1)), dim=-1))
return sampled
def sample_block(
self,
logits: torch.Tensor,
idx_mapping: torch.Tensor,
temperature: torch.Tensor,
seed: torch.Tensor,
pos: torch.Tensor,
logits_cache: torch.Tensor | None = None,
logits_cache_col: torch.Tensor | None = None,
use_fp64: bool = False,
) -> torch.Tensor:
"""Sample drafts flattened as (request, step), one step at a time.
The target verifies each draft with a context that includes the earlier
drafts of its request, so steps must be sampled in order even when
their logits were computed in parallel.
"""
assert logits_cache_col is not None
num_steps = self.num_speculative_steps
sampled_block = self.watermarker._try_sample_block(
logits,
num_steps,
RandomSampler(
expanded_idx_mapping=idx_mapping,
temperatures=temperature,
seeds=seed,
positions=pos,
use_fp64=use_fp64,
is_drafting=True,
logits_cache=logits_cache,
logits_cache_col=logits_cache_col,
),
self.state,
)
if sampled_block is not None:
return sampled_block
num_reqs = logits.shape[0] // num_steps
def by_step(tensor: torch.Tensor) -> torch.Tensor:
return tensor.view(num_reqs, num_steps, *tensor.shape[1:])
logits = by_step(logits)
idx_mapping = by_step(idx_mapping)
pos = by_step(pos)
logits_cache_col = by_step(logits_cache_col)
sampled = torch.empty(
num_reqs, num_steps, dtype=torch.int64, device=logits.device
)
for step in range(num_steps):
sampled[:, step] = self.sample(
logits[:, step],
idx_mapping[:, step],
temperature,
seed,
pos[:, step],
apply_temperature=True,
is_drafting=True,
logits_cache=logits_cache,
logits_cache_col=logits_cache_col[:, step],
use_fp64=use_fp64,
)
return sampled.view(-1)
def create_speculative_target_watermarker(watermarker: Watermarker) -> Watermarker:
if isinstance(watermarker, SupportsSpeculativeDecoding):
+27
View File
@@ -7,6 +7,7 @@ from typing import Protocol, runtime_checkable
import torch
from vllm.config.watermarking import WatermarkContextScope
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
@@ -44,6 +45,18 @@ class RandomSampler:
)
@dataclass
class DraftContextState:
contexts: torch.Tensor
enabled: torch.Tensor
prior_contexts: torch.Tensor
prompt_lens: torch.Tensor
deduplicate_contexts: WatermarkContextScope
deduplicate_contexts_max_history: int | None
all_token_ids: torch.Tensor | None = None
total_lens: torch.Tensor | None = None
class Watermarker(ABC):
@property
@abstractmethod
@@ -94,6 +107,20 @@ class Watermarker(ABC):
) -> WatermarkSample | None:
return None
def _try_sample_block(
self,
logits: torch.Tensor,
num_steps: int,
random_sampler: RandomSampler,
state: DraftContextState,
) -> torch.Tensor | None:
"""Sample a (request, step) draft block in one fused pass.
Returns None when there is no fused implementation, in which case the
caller samples the block one step at a time.
"""
return None
@runtime_checkable
class SupportsSpeculativeDecoding(Protocol):
+18 -4
View File
@@ -234,10 +234,24 @@ class CPUWorker(Worker):
else:
consumed_memory = psutil.Process(os.getpid()).memory_info().rss
requested_memory_for_kv = int(self.requested_cpu_memory - consumed_memory)
if (
requested_memory_for_kv <= 0
or requested_memory_for_kv > available_memory
):
if requested_memory_for_kv <= 0:
# The worker's own footprint already fills its reservation, so
# freeing memory elsewhere cannot help: only a larger
# reservation or an explicit KV cache size can.
in_use = math.ceil(consumed_memory / memory_status.total_memory * 100)
raise ValueError(
f"The worker already uses {format_gib(consumed_memory)} GiB "
"(model weights and runtime), more than the "
f"{format_gib(self.requested_cpu_memory)} GiB of CPU memory "
f"reserved for it on node {cpu_core.numa_node}, leaving none "
"for the KV cache. On the CPU backend, the "
"`--gpu-memory-utilization` flag controls the fraction of CPU "
"memory reserved (despite its name). To resolve: increase "
f"`--gpu-memory-utilization` to more than {in_use / 100:.2f}, "
"or set the KV cache size explicitly with "
"`--kv-cache-memory-bytes` or VLLM_CPU_KVCACHE_SPACE."
)
if requested_memory_for_kv > available_memory:
raise ValueError(
f"Available memory on node {cpu_core.numa_node} "
f"({format_gib(available_memory)}/"
+3 -1
View File
@@ -33,6 +33,7 @@ from vllm.distributed.parallel_state import (
)
from vllm.forward_context import BatchDescriptor, set_forward_context
from vllm.logger import init_logger
from vllm.model_executor.models.interfaces import requires_raw_input_tokens
from vllm.model_executor.offloader.base import get_offloader
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
@@ -690,7 +691,8 @@ class ModelCudaGraphManager(CudaGraphManager):
}
if not self.is_first_pp_rank:
# Update for non-first PP ranks.
model_inputs["input_ids"] = None
if not requires_raw_input_tokens(model):
model_inputs["input_ids"] = None
model_inputs["inputs_embeds"] = None
assert intermediate_tensors is not None
model_inputs["intermediate_tensors"] = intermediate_tensors[:num_tokens]
+1 -3
View File
@@ -5,7 +5,6 @@ from __future__ import annotations
from dataclasses import dataclass, replace
import torch
import torch.distributed as dist
from vllm.config import ParallelConfig
from vllm.config.compilation import CUDAGraphMode
@@ -63,7 +62,6 @@ def sync_cudagraph_and_dp_padding(
Returns (synced_batch_desc, sync). `sync` is None when no rank has work.
"""
assert dp_size > 1, "DP size must be greater than 1"
group = get_dp_group().cpu_group
tensor = torch.zeros(6, dp_size, dtype=torch.int32, device="cpu")
tensor[0][dp_rank] = num_tokens
tensor[1][dp_rank] = desired_batch_desc.cg_mode.value
@@ -74,7 +72,7 @@ def sync_cudagraph_and_dp_padding(
if should_skip_dp_coordination():
tensor[:] = tensor[:, dp_rank, None].clone()
else:
dist.all_reduce(tensor, group=group)
get_dp_group().cpu_all_reduce_(tensor)
num_tokens_across_dp = tensor[0]
cg_mode_across_dp = tensor[1]
+15 -2
View File
@@ -33,7 +33,7 @@ import vllm.envs as envs
from vllm.compilation.counter import compilation_counter
from vllm.compilation.cuda_graph import CUDAGraphStat
from vllm.compilation.wrapper import compile_model_with_stock_torch
from vllm.config import VllmConfig
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.config.compilation import CompilationMode, CUDAGraphMode
from vllm.distributed.aux_output_connector.worker import (
AuxOutputWorkerConnector,
@@ -46,6 +46,9 @@ from vllm.model_executor.layers.fused_moe.all2all_utils import get_ep_all2all_ma
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
initialize_mamba_ssu_backend,
)
from vllm.model_executor.layers.quantization.online.lm_head import (
refresh_quantized_lm_heads,
)
from vllm.model_executor.model_loader import get_model_loader
from vllm.model_executor.model_loader.reload.inplace import reload_weights
from vllm.model_executor.models.interfaces import requires_raw_input_tokens
@@ -569,6 +572,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
return None
return speculator.model
def reset_lora_state(self) -> None:
super().reset_lora_state()
# Called whenever the target's weights are replaced (reload, weight
# transfer), so draft weights derived from them must follow.
draft_model = self.get_draft_model()
if draft_model is not None:
with set_current_vllm_config(self.vllm_config):
refresh_quantized_lm_heads(draft_model)
def reload_weights(
self,
weights_iterator: Iterable[tuple[str, torch.Tensor]] | None = None,
@@ -1970,7 +1982,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
}
if not self.is_first_pp_rank:
# Update for non-first PP ranks.
model_inputs["input_ids"] = None
if not requires_raw_input_tokens(self.model):
model_inputs["input_ids"] = None
model_inputs["inputs_embeds"] = None
# Prepare the intermediate tensors.
+8
View File
@@ -70,6 +70,12 @@ def tl_rand32(seed, offset, includes_zero: tl.constexpr):
return u
@triton.jit
def spec_step_seed(seed, step):
"""Mix a speculative step into the high bits of a request seed."""
return seed ^ (tl.cast(step, tl.int64) << 32)
@triton.jit
def _murmur3_rotl32(value, shift: tl.constexpr):
return (value << shift) | (value >> (32 - shift))
@@ -257,6 +263,8 @@ def gumbel_block_argmax(
seed = tl.load(seeds_ptr + req_state_idx, mask=is_valid_req, other=0)
pos = tl.load(pos_ptr + token_idx)
if logits_cache_ptr is not None:
seed = spec_step_seed(seed, col)
return gumbel_noised_argmax(
logits,
block,
+252 -6
View File
@@ -25,7 +25,7 @@ from vllm.v1.watermarking.prfs.philox import (
from vllm.v1.watermarking.prfs.philox import (
_UINT32_MASK as _UINT32_MASK_VALUE,
)
from vllm.v1.worker.gpu.sample.gumbel import gumbel_noised_argmax
from vllm.v1.worker.gpu.sample.gumbel import gumbel_noised_argmax, spec_step_seed
if HAS_TRITON:
from triton.language import math as tl_math
@@ -530,8 +530,10 @@ def _philox_gumbel_from_logits(logits, word):
@triton.jit
def _gumbel_value(logits_ptr, output, mask):
def _gumbel_value(logits_ptr, output, mask, temperature):
logits = tl.load(logits_ptr, mask=mask, other=float("-inf")).to(tl.float32)
if temperature is not None:
logits = tl_math.div_rn(logits, temperature)
return _philox_gumbel_from_logits(logits, output)
@@ -606,6 +608,7 @@ def _philox_gumbel_kernel(
USE_FP64: tl.constexpr,
IS_DRAFTING: tl.constexpr,
PER_TOKEN_COL: tl.constexpr,
APPLY_TEMPERATURE: tl.constexpr,
):
row = tl.program_id(0).to(tl.int64)
block_index = tl.program_id(1)
@@ -619,6 +622,13 @@ def _philox_gumbel_kernel(
req_state_idx = tl.load(expanded_idx_mapping_ptr + row).to(tl.int64)
valid_req = req_state_idx >= 0
row_temperature = None
if APPLY_TEMPERATURE:
row_temperature = tl.load(
temp_ptr + req_state_idx, mask=valid_req, other=0.0
).to(tl.float32)
row_temperature = tl.where(row_temperature == 0, 1.0, row_temperature)
if logits_cache_ptr is not None:
if PER_TOKEN_COL:
col = tl.load(logits_cache_col_ptr + row)
@@ -644,10 +654,14 @@ def _philox_gumbel_kernel(
)
seed = tl.load(seeds_ptr + req_state_idx, mask=valid_req, other=0)
pos = tl.load(pos_ptr + row)
if logits_cache_ptr is not None:
seed = spec_step_seed(seed, col)
logits_row = logits_ptr + row * logits_stride
logits = tl.load(logits_row + candidate, mask=mask, other=float("-inf")).to(
tl.float32
)
if row_temperature is not None:
logits = tl_math.div_rn(logits, row_temperature)
value, index = gumbel_noised_argmax(
logits,
candidate,
@@ -681,16 +695,16 @@ def _philox_gumbel_kernel(
candidate_3 = candidate_0 + 3
logits_row = logits_ptr + row * logits_stride
value_0 = _gumbel_value(
logits_row + candidate_0, output_0, candidate_0 < vocab_size
logits_row + candidate_0, output_0, candidate_0 < vocab_size, row_temperature
)
value_1 = _gumbel_value(
logits_row + candidate_1, output_1, candidate_1 < vocab_size
logits_row + candidate_1, output_1, candidate_1 < vocab_size, row_temperature
)
value_2 = _gumbel_value(
logits_row + candidate_2, output_2, candidate_2 < vocab_size
logits_row + candidate_2, output_2, candidate_2 < vocab_size, row_temperature
)
value_3 = _gumbel_value(
logits_row + candidate_3, output_3, candidate_3 < vocab_size
logits_row + candidate_3, output_3, candidate_3 < vocab_size, row_temperature
)
best_value = value_0
best_token = candidate_0
@@ -797,6 +811,238 @@ def philox_gumbel_sample(
USE_FP64=use_fp64,
IS_DRAFTING=is_drafting,
PER_TOKEN_COL=logits_cache_col is not None and logits_cache_col.dim() > 0,
APPLY_TEMPERATURE=False,
)
max_block_index = local_max.argmax(dim=-1, keepdim=True)
return local_argmax.gather(dim=-1, index=max_block_index).view(-1)
@triton.jit(do_not_specialize=["step"])
def _draft_step_kernel(
sampled_ptr,
sampled_stride,
step,
local_argmax_ptr,
local_max_ptr,
num_blocks,
contexts_ptr,
context_stride,
skip_mask_ptr,
next_req_indices_ptr,
next_steps_ptr,
temp_ptr,
enabled_ptr,
all_token_ids_ptr,
all_token_ids_stride,
prompt_lens_ptr,
total_lens_ptr,
prior_contexts_ptr,
prior_contexts_stride_0,
prior_contexts_stride_1,
CONTEXT_WIDTH: tl.constexpr,
NUM_SPECULATIVE_STEPS: tl.constexpr,
MAX_HISTORY: tl.constexpr,
INCLUDE_PROMPT: tl.constexpr,
DEDUPLICATE: tl.constexpr,
SAMPLE: tl.constexpr,
PREPARE_NEXT: tl.constexpr,
BLOCKS_POW2: tl.constexpr,
):
"""Finish draft step `step` (if SAMPLE) and build the next step's skip mask."""
row = tl.program_id(0).to(tl.int64)
contexts_row_ptr = contexts_ptr + row * context_stride
if SAMPLE:
# Same pick as local_max.argmax(dim=-1): the first NaN, else the first max.
blocks = tl.arange(0, BLOCKS_POW2)
values = tl.load(
local_max_ptr + row * num_blocks + blocks,
mask=blocks < num_blocks,
other=float("-inf"),
)
is_nan = values != values
best = tl.max(tl.where(is_nan, float("-inf"), values), axis=0)
first_nan = tl.min(tl.where(is_nan, blocks, BLOCKS_POW2), axis=0)
first_max = tl.min(
tl.where((values == best) & (blocks < num_blocks), blocks, BLOCKS_POW2),
axis=0,
)
block = tl.where(first_nan < BLOCKS_POW2, first_nan, first_max)
token = tl.load(local_argmax_ptr + row * num_blocks + block)
tl.store(sampled_ptr + row * sampled_stride + step, token)
offsets = tl.arange(0, BLOCKS_POW2)
context = tl.load(
contexts_row_ptr + offsets + 1,
mask=offsets < CONTEXT_WIDTH - 1,
other=token,
)
tl.debug_barrier()
tl.store(contexts_row_ptr + offsets, context, mask=offsets < CONTEXT_WIDTH)
tl.debug_barrier()
if PREPARE_NEXT:
req_idx = tl.load(next_req_indices_ptr + row).to(tl.int64)
temperature = tl.load(temp_ptr + req_idx, mask=req_idx >= 0, other=0.0)
active = tl.load(enabled_ptr + row) & (temperature != 0)
if DEDUPLICATE:
# The dedup kernel body runs in place on the skip mask buffer.
tl.store(skip_mask_ptr + row, active)
tl.debug_barrier()
_repeated_context_mask_kernel(
skip_mask_ptr,
all_token_ids_ptr,
all_token_ids_stride,
next_req_indices_ptr,
prompt_lens_ptr,
total_lens_ptr,
next_steps_ptr,
1,
contexts_ptr,
context_stride,
None,
prior_contexts_ptr,
prior_contexts_stride_0,
prior_contexts_stride_1,
next_steps_ptr,
1,
skip_mask_ptr,
CONTEXT_WIDTH=CONTEXT_WIDTH,
NUM_SPECULATIVE_STEPS=NUM_SPECULATIVE_STEPS,
MAX_HISTORY=MAX_HISTORY,
INCLUDE_PROMPT=INCLUDE_PROMPT,
SKIP_PARTIAL_CONTEXT=False,
HAS_HISTORY_OFFSETS=True,
SPECULATIVE_CONTEXTS=False,
DRAFT_CONTEXTS=True,
BLOCK=512,
)
tl.debug_barrier()
active = tl.load(skip_mask_ptr + row)
tl.store(skip_mask_ptr + row, active == 0)
def draft_philox_gumbel_sample(
logits: torch.Tensor,
contexts: torch.Tensor,
key: int,
*,
num_steps: int,
expanded_idx_mapping: torch.Tensor,
temperatures: torch.Tensor,
seeds: torch.Tensor,
positions: torch.Tensor,
enabled: torch.Tensor,
logits_cache: torch.Tensor,
logits_cache_col: torch.Tensor,
use_fp64: bool,
prior_contexts: torch.Tensor,
all_token_ids: torch.Tensor | None,
prompt_lens: torch.Tensor,
total_lens: torch.Tensor | None,
deduplicate: bool,
max_history: int | None,
include_prompt: bool,
) -> torch.Tensor:
"""Sample `num_steps` draft steps flattened as (request, step), in step order.
Bit-exact with sampling each step by `DraftWatermarker.sample`, but with two
kernel launches per step: the Philox block argmax (with the temperature
divide and the logits cache write), then a per-row kernel that picks the
token, shifts `contexts` and builds the next step's skip mask. Padding rows
(index -1) read temperature 0 and are never watermarked, so they may differ.
"""
num_rows, vocab_size = logits.shape
num_reqs = num_rows // num_steps
assert logits.stride(-1) == 1 and contexts.stride(-1) == 1
assert logits.dtype == logits_cache.dtype, (
"logits cache source and destination must have the same dtype"
)
assert logits_cache.size(-1) >= vocab_size, (
f"draft logits cache vocab dim ({logits_cache.size(-1)}) is narrower "
f"than the sampled logits ({vocab_size}). Cached logits would be truncated."
)
assert not deduplicate or (all_token_ids is not None and total_lens is not None), (
"context deduplication requires all_token_ids and total_lens"
)
logits = logits.view(num_reqs, num_steps, vocab_size)
# Step-major copies, so each step reads contiguous per-request slices.
expanded_idx_mapping, positions, logits_cache_col = (
tensor.view(num_reqs, num_steps).t().contiguous()
for tensor in (expanded_idx_mapping, positions, logits_cache_col)
)
block_size = 1024
num_blocks = triton.cdiv(vocab_size, block_size)
local_argmax = logits.new_empty(num_reqs, num_blocks, dtype=torch.int64)
local_max = logits.new_empty(
num_reqs, num_blocks, dtype=torch.float64 if use_fp64 else torch.float32
)
skip_mask = torch.empty(num_reqs, dtype=torch.bool, device=logits.device)
sampled = logits.new_empty(num_reqs, num_steps, dtype=torch.int64)
def finish_step(step: int) -> None:
next_step = min(step + 1, num_steps - 1)
_draft_step_kernel[(num_reqs,)](
sampled,
sampled.stride(0),
step,
local_argmax,
local_max,
num_blocks,
contexts,
contexts.stride(0),
skip_mask,
expanded_idx_mapping[next_step],
logits_cache_col[next_step],
temperatures,
enabled,
all_token_ids,
0 if all_token_ids is None else all_token_ids.stride(0),
prompt_lens,
total_lens,
prior_contexts,
prior_contexts.stride(0),
prior_contexts.stride(1),
CONTEXT_WIDTH=contexts.shape[-1],
NUM_SPECULATIVE_STEPS=prior_contexts.shape[1],
MAX_HISTORY=0 if max_history is None else max_history,
INCLUDE_PROMPT=include_prompt,
DEDUPLICATE=deduplicate,
SAMPLE=step >= 0,
PREPARE_NEXT=step + 1 < num_steps,
BLOCKS_POW2=triton.next_power_of_2(max(num_blocks, contexts.shape[-1])),
)
finish_step(-1)
for step in range(num_steps):
_philox_gumbel_kernel[(num_reqs, num_blocks)](
local_argmax,
local_argmax.stride(0),
local_max,
local_max.stride(0),
logits[:, step],
logits.stride(0),
contexts,
contexts.stride(0),
skip_mask,
expanded_idx_mapping[step],
seeds,
positions[step],
temperatures,
logits_cache,
logits_cache.stride(0),
logits_cache.stride(1),
logits_cache_col[step],
logits[:, step],
logits.stride(0),
key & _UINT32_MASK_VALUE,
key >> 32,
vocab_size,
CONTEXT_WIDTH=contexts.shape[-1],
BLOCK_SIZE=block_size,
USE_FP64=use_fp64,
IS_DRAFTING=True,
PER_TOKEN_COL=True,
APPLY_TEMPERATURE=True,
)
finish_step(step)
return sampled.view(-1)
@@ -366,6 +366,7 @@ class DFlashSpeculator(DraftModelSpeculator):
self.seeds,
self.sample_col[:num_sample],
self.draft_logits,
num_steps=self.num_speculative_steps,
)
self.draft_tokens[:num_reqs] = draft_tokens.view(
num_reqs, self.num_speculative_steps
@@ -8,7 +8,7 @@ import torch
from vllm.config import VllmConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.triton_utils import tl, triton
from vllm.v1.worker.gpu.sample.gumbel import gumbel_noised_argmax
from vllm.v1.worker.gpu.sample.gumbel import gumbel_noised_argmax, spec_step_seed
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator
@@ -58,7 +58,7 @@ def _selector_walk_kernel(
scores,
candidates,
mask & valid,
seed,
spec_step_seed(seed, step),
sample_pos,
temperature if SAMPLE_PROBABILISTIC else 0.0,
IS_DRAFTING=True,
+14 -2
View File
@@ -6,6 +6,9 @@ import torch.nn as nn
from vllm.config import VllmConfig, replace
from vllm.distributed.parallel_state import get_pp_group
from vllm.lora.layers.base import BaseLayerWithLoRA
from vllm.model_executor.layers.quantization.online.lm_head import (
quantized_lm_head_copy,
)
from vllm.model_executor.model_loader import get_model
from vllm.model_executor.model_loader.utils import get_draft_load_config
from vllm.model_executor.models.utils import PPMissingLayer
@@ -106,9 +109,14 @@ def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mod
if target_lm_head is not None and _should_share(
eagle_model, "has_own_lm_head", draft_lm_head, target_lm_head
):
draft_head = target_lm_head
if speculative_config.draft_lm_head_quantization is not None:
draft_head = quantized_lm_head_copy(
target_lm_head, speculative_config.draft_lm_head_quantization
)
if draft_lm_head is not None:
del eagle_model.lm_head
eagle_model.lm_head = target_lm_head
eagle_model.lm_head = draft_head
# MTP layers route logits through layer.shared_head.head, not
# eagle_model.lm_head, so the per-layer copies need fixing up too.
@@ -119,7 +127,11 @@ def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mod
sh = getattr(layer, "shared_head", None)
if sh is not None and hasattr(sh, "head"):
del sh.head
sh.head = target_lm_head
sh.head = draft_head
elif speculative_config.draft_lm_head_quantization is not None:
raise ValueError(
"draft_lm_head_quantization needs a drafter that shares the target lm_head."
)
# MTP shares topk_indices_buffer with the target model. We update
# every module in the draft that holds a buffer reference so that
@@ -3,7 +3,11 @@
import torch
from vllm.triton_utils import tl, tldevice, triton
from vllm.v1.worker.gpu.sample.gumbel import gumbel_block_argmax, tl_rand32
from vllm.v1.worker.gpu.sample.gumbel import (
gumbel_block_argmax,
spec_step_seed,
tl_rand32,
)
from vllm.v1.worker.gpu.sample.watermark import philox_gumbel_block_argmax
@@ -569,7 +573,8 @@ def _rejection_kernel(
verifying &= is_valid_draft
if verifying:
pos = tl.load(pos_ptr + logit_idx)
u = tl_rand32(seed, pos, includes_zero=False)
step_seed = spec_step_seed(seed, i) if USE_BLOCK_VERIFICATION else seed
u = tl_rand32(step_seed, pos, includes_zero=False)
if is_greedy:
# Greedy sampling. Accept IFF draft matches target argmax.
# NOTE: Target argmax is stored directly so that resampling
+3 -1
View File
@@ -3,6 +3,7 @@
from abc import ABC, abstractmethod
from collections.abc import Mapping
from dataclasses import replace
from functools import partial
from typing import TYPE_CHECKING, Any
import numpy as np
@@ -446,6 +447,7 @@ class DraftModelSpeculator(BaseSpeculator):
draft_step: torch.Tensor,
draft_logits: torch.Tensor | None,
spec_step_idx: int = 0,
num_steps: int = 1,
) -> torch.Tensor:
if draft_logits is None and self.use_local_argmax_reduction:
return self.get_draft_top_tokens(hidden_states, spec_step_idx)
@@ -455,7 +457,7 @@ class DraftModelSpeculator(BaseSpeculator):
sampler = (
gumbel_sample
if self.draft_watermarker is None
else self.draft_watermarker.sample
else partial(self.draft_watermarker.sample, num_steps=num_steps)
)
sampled = sampler(
logits,
+19
View File
@@ -62,6 +62,10 @@ from vllm.distributed.weight_transfer import (
)
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
from vllm.model_executor.layers.quantization.online.lm_head import (
restore_quantized_lm_heads,
share_source_lm_heads,
)
from vllm.model_executor.model_loader import get_model_loader
from vllm.model_executor.model_loader.weight_checksum import (
compute_tensor_digests,
@@ -1544,12 +1548,19 @@ class Worker(WorkerBase):
"active. Call finish_weight_update first."
)
draft_model = self.get_draft_model() if is_draft else None
try:
if is_draft:
self._set_draft_weight_update_target()
if draft_model is not None:
# Load through the unquantized target head, as without
# draft_lm_head_quantization; finish re-derives the copies.
share_source_lm_heads(draft_model)
self.weight_transfer_engine.start_weight_update()
except BaseException:
self.weight_transfer_engine.reset_weight_update_target()
if draft_model is not None:
restore_quantized_lm_heads(draft_model, refresh=False)
raise
self._weight_update_active = True
self._weight_update_is_draft = is_draft
@@ -1589,6 +1600,10 @@ class Worker(WorkerBase):
except BaseException:
self._weight_update_active = False
self.weight_transfer_engine.reset_weight_update_target()
if self._weight_update_is_draft and (
(draft_model := self.get_draft_model()) is not None
):
restore_quantized_lm_heads(draft_model, refresh=False)
raise
def finish_weight_update(self) -> None:
@@ -1605,6 +1620,10 @@ class Worker(WorkerBase):
self.weight_transfer_engine.finish_weight_update()
self.weight_transfer_engine.reset_weight_update_target()
self._weight_update_active = False
if self._weight_update_is_draft and (
(draft_model := self.get_draft_model()) is not None
):
restore_quantized_lm_heads(draft_model)
# Weight transfer bypasses GPUModelRunner.reload_weights().
if not self._weight_update_is_draft: