Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -950,6 +950,16 @@ class AttentionImpl(AttentionImplBase[T], Generic[T]):
|
||||
"""Whether this implementation supports fused QKNorm+MRoPE+KVCache."""
|
||||
return False
|
||||
|
||||
def set_interleaved_v_cache(self):
|
||||
"""Opt this implementation into the interleaved V-cache layout for all of
|
||||
its cache writers and readers. QkNormRopeKvCacheFusionPass calls this
|
||||
for every eligible layer when it is constructed, i.e. before any KV
|
||||
cache is written, so fused and unfused paths stay consistent whether or
|
||||
not the pattern matches. Default is a no-op; backends whose kernels
|
||||
read an interleaved V-cache under this fusion (e.g. ROCM_ATTN) override.
|
||||
"""
|
||||
return False
|
||||
|
||||
def fused_rope_kvcache_supported(self):
|
||||
"""Does this attention implementation support RoPE+KVCache fusion.
|
||||
This is used by the RopeKVCacheFusionPass to only fuse the RoPE ops
|
||||
|
||||
@@ -35,6 +35,7 @@ from vllm.v1.attention.backends.mla.sparse_utils import (
|
||||
flat_kv_row_view,
|
||||
prepare_sparse_mla_safe_lengths,
|
||||
)
|
||||
from vllm.v1.attention.ops.metadata import compute_token_to_req_indices
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheSpec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -297,6 +298,11 @@ class FlashInferMLASparseTRTLLMMetadataBuilder(FlashInferMLASparseMetadataBuilde
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||
self.supports_draft_decode_metadata_update = (
|
||||
self.dcp_world_size == 1
|
||||
and self.pcp_world_size == 1
|
||||
and vllm_config.attention_config.hisparse_config is None
|
||||
)
|
||||
|
||||
# Under DCP the workspace must hold the trtllm-gen softmax-stats slab.
|
||||
# The buffer address is baked into CUDA graphs, so it has to reach its
|
||||
@@ -314,6 +320,18 @@ class FlashInferMLASparseTRTLLMMetadataBuilder(FlashInferMLASparseMetadataBuilde
|
||||
),
|
||||
)
|
||||
|
||||
def update_draft_decode_metadata(
|
||||
self, metadata: FlashInferMLASparseMetadata
|
||||
) -> None:
|
||||
num_tokens = metadata.num_decode_tokens
|
||||
if num_tokens:
|
||||
compute_token_to_req_indices(
|
||||
metadata.query_start_loc,
|
||||
metadata.req_id_per_token,
|
||||
num_tokens,
|
||||
num_tokens,
|
||||
)
|
||||
|
||||
|
||||
# Global workspace buffer (lazily initialized)
|
||||
_fi_sparse_workspace: torch.Tensor | None = None
|
||||
|
||||
@@ -103,10 +103,11 @@ For DeepSeek V4.1, each token's KV cache is 528 Bytes, structured as:
|
||||
- **Last 16 bytes:** Scale factors, containing 16 `ue8m0` values, one per 32
|
||||
consecutive `float8_e4m3` values (i.e. MXFP8).
|
||||
|
||||
The V4 and V4.1 records are not laid out token-by-token within a page: a page
|
||||
holds all its data rows first and all its scale rows after, so a page is
|
||||
`block_size * bytes_per_token` bytes rounded up to the decode kernel's TMA
|
||||
stride (576 B for V4, 512 B for V4.1).
|
||||
The V4 record is not laid out token-by-token within a page: a page holds all
|
||||
its data rows first and all its scale rows after, so a page is
|
||||
`block_size * bytes_per_token` bytes rounded up to the decode kernel's 576 B
|
||||
TMA stride. The V4.1 records are interleaved: each token's scales directly
|
||||
follow its own data, and the decode kernel's TMA stride is the record width.
|
||||
|
||||
In the "nvfp4_ds_mla" format (SM100 only, DeepSeek V3.2 geometry), each
|
||||
token's KV cache is 352 Bytes, structured as:
|
||||
|
||||
@@ -205,19 +205,16 @@ class PrepareUniformDecodeKernel(
|
||||
)
|
||||
|
||||
|
||||
# Below this many rows per rank, stay replicated (TP4 engages at 64K rows).
|
||||
MIN_TP_SHARD_ROWS_PER_RANK = 16_384
|
||||
|
||||
|
||||
def balanced_prefill_row_shard(
|
||||
seq_lens_cpu: torch.Tensor,
|
||||
query_lens_cpu: torch.Tensor,
|
||||
compress_ratio: int,
|
||||
tp_size: int,
|
||||
min_rows: int,
|
||||
) -> list[int] | None:
|
||||
"""Return contiguous TP row counts balanced by indexer MQA work."""
|
||||
num_rows = int(query_lens_cpu.sum())
|
||||
if tp_size < 2 or num_rows < MIN_TP_SHARD_ROWS_PER_RANK * tp_size:
|
||||
if tp_size < 2 or num_rows < max(min_rows, tp_size):
|
||||
return None
|
||||
|
||||
query_lens = query_lens_cpu.to(torch.int64)
|
||||
@@ -1584,6 +1581,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
prefill_query_lens_cpu,
|
||||
self.compress_ratio,
|
||||
self.tp_size,
|
||||
self.vllm_config.attention_config.indexer_tp_shard_min_rows,
|
||||
)
|
||||
prefill_metadata = DeepseekV32IndexerPrefillMetadata(
|
||||
chunks,
|
||||
|
||||
@@ -85,6 +85,7 @@ class DeepseekV4SWACache(torch.nn.Module, AttentionLayerBase):
|
||||
block_size: int = 64,
|
||||
packed_bytes_per_token: int = 584,
|
||||
packed_page_alignment: int = 576,
|
||||
packed_block_stride_alignment: int | None = None,
|
||||
bounded_replay: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
@@ -99,6 +100,7 @@ class DeepseekV4SWACache(torch.nn.Module, AttentionLayerBase):
|
||||
self.dtype = dtype
|
||||
self.packed_bytes_per_token = packed_bytes_per_token
|
||||
self.packed_page_alignment = packed_page_alignment
|
||||
self.packed_block_stride_alignment = packed_block_stride_alignment
|
||||
compilation_config = get_current_vllm_config().compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
@@ -135,8 +137,11 @@ class DeepseekV4SWACache(torch.nn.Module, AttentionLayerBase):
|
||||
state_content_bytes=(
|
||||
self.packed_bytes_per_token if uses_fp8_ds_mla_layout else None
|
||||
),
|
||||
# FlashMLA packing stride; 512B for FlashInfer sparse (#44577).
|
||||
# Packed page alignment; 512B for FlashInfer sparse (#44577).
|
||||
alignment=self.packed_page_alignment if uses_fp8_ds_mla_layout else 512,
|
||||
block_stride_alignment=(
|
||||
self.packed_block_stride_alignment if uses_fp8_ds_mla_layout else None
|
||||
),
|
||||
model_version="deepseek_v4",
|
||||
kv_quant_mode=get_kv_quant_mode(self.cache_config.cache_dtype),
|
||||
)
|
||||
@@ -211,6 +216,9 @@ class DeepseekSparseSWAMetadata:
|
||||
# window attention may read (SWA bounded replay);
|
||||
# zeros when nothing replays.
|
||||
replay_start: torch.Tensor | None = None
|
||||
# PCP cache writes use the gathered slots; queries use rank-local slots.
|
||||
cache_slot_mapping: torch.Tensor | None = None
|
||||
cache_positions: torch.Tensor | None = None
|
||||
|
||||
# Number of decode/prefill requests/tokens (batch is reordered: decodes first)
|
||||
num_decodes: int = 0
|
||||
|
||||
@@ -117,6 +117,9 @@ class AttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta):
|
||||
"vllm.models.deepseek_v41.nvidia.flashinfer_sparse."
|
||||
"DeepseekV4FlashInferMLASparseBackend"
|
||||
)
|
||||
ROCM_FLASHMLA_SPARSE_DSV41 = (
|
||||
"vllm.models.deepseek_v41.amd.rocm.DeepseekV41ROCMAiterMLASparseBackend"
|
||||
)
|
||||
FLASHMLA_MEGA_ATTN_DSV41 = (
|
||||
"vllm.models.deepseek_v41.sparse_mla.FlashMLAMegaAttnBackend"
|
||||
)
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Attention layer with AiterFlashAttention."""
|
||||
|
||||
from dataclasses import dataclass, replace
|
||||
from dataclasses import dataclass, field, replace
|
||||
from typing import ClassVar
|
||||
|
||||
import torch
|
||||
@@ -152,8 +152,8 @@ if current_platform.is_rocm():
|
||||
k_reg = tl.load(key_cache_ptr_offset + k_reg_offset)
|
||||
v_reg = tl.load(value_cache_ptr_offset + v_reg_offset)
|
||||
if DEQUANT:
|
||||
k_scale = 1.0
|
||||
v_scale = 1.0
|
||||
k_scale = tl.load(k_scale_ptr)
|
||||
v_scale = tl.load(v_scale_ptr)
|
||||
k_reg = k_reg.to(tl.float32) * k_scale
|
||||
v_reg = v_reg.to(tl.float32) * v_scale
|
||||
tl.store(key_ptr_offset + col_offsets, k_reg)
|
||||
@@ -270,8 +270,8 @@ if current_platform.is_rocm():
|
||||
k_val = tl.load(key_ptr + src_offset_k + offset)
|
||||
v_val = tl.load(value_ptr + src_offset_v + offset)
|
||||
if QUANT:
|
||||
k_scale = 1.0
|
||||
v_scale = 1.0
|
||||
k_scale = tl.load(k_scale_ptr)
|
||||
v_scale = tl.load(v_scale_ptr)
|
||||
k_dtype = key_cache_ptr.type.element_ty
|
||||
v_dtype = value_cache_ptr.type.element_ty
|
||||
k_val = (k_val.to(tl.float32) / k_scale).to(k_dtype)
|
||||
@@ -410,11 +410,48 @@ class AiterFlashAttentionMetadata:
|
||||
kv_sharing_metadata: AiterKVSharingMetadata | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class AsmKvScaleTables:
|
||||
"""Dense per-slot scale tables the asm decode kernel reads for an fp8 cache.
|
||||
|
||||
One table per distinct (device, shape, scale), shared by every layer and
|
||||
every metadata builder in the process, so the target and draft models with
|
||||
equal scales reuse a single allocation. Tables are constants: never write
|
||||
to one in place; a per-token or dynamic scale needs its own buffer.
|
||||
"""
|
||||
|
||||
tables: dict[tuple[str, int, int, int, float], torch.Tensor] = field(
|
||||
default_factory=dict
|
||||
)
|
||||
|
||||
def get(
|
||||
self,
|
||||
device: torch.device,
|
||||
num_blocks: int,
|
||||
num_kv_heads: int,
|
||||
block_size: int,
|
||||
scale: float,
|
||||
) -> torch.Tensor:
|
||||
key = (str(device), num_blocks, num_kv_heads, block_size, scale)
|
||||
table = self.tables.get(key)
|
||||
if table is None:
|
||||
table = torch.full(
|
||||
(num_blocks, num_kv_heads, block_size),
|
||||
scale,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
self.tables[key] = table
|
||||
return table
|
||||
|
||||
|
||||
class AiterFlashAttentionMetadataBuilder(
|
||||
AttentionMetadataBuilder[AiterFlashAttentionMetadata]
|
||||
):
|
||||
_cudagraph_support = AttentionCGSupport.UNIFORM_BATCH
|
||||
|
||||
asm_kv_scale_tables: ClassVar[AsmKvScaleTables] = AsmKvScaleTables()
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_spec: AttentionSpec,
|
||||
@@ -466,7 +503,8 @@ class AiterFlashAttentionMetadataBuilder(
|
||||
dtype=self.model_config.dtype,
|
||||
device=device,
|
||||
)
|
||||
self.scale = torch.tensor([1.0], dtype=torch.float, device=self.device)
|
||||
self.k_scale: dict[str, torch.Tensor] | None = None
|
||||
self.v_scale: dict[str, torch.Tensor] | None = None
|
||||
self.kv_sharing_workspace = (
|
||||
torch.empty(
|
||||
(
|
||||
@@ -488,6 +526,38 @@ class AiterFlashAttentionMetadataBuilder(
|
||||
common_prefix_len=0, common_attn_metadata=common_attn_metadata
|
||||
)
|
||||
|
||||
def _build_asm_kv_scale_tables(self) -> None:
|
||||
# Only for fp8 shuffle kv cache: the asm decode kernel reads a dense
|
||||
# [num_blocks, num_kv_heads, block_size] scale table per layer. Map each
|
||||
# layer to the shared table for its scale once the KV cache is bound.
|
||||
if (
|
||||
self.k_scale is not None
|
||||
or not rocm_aiter_ops.is_shuffle_kv_cache_enabled()
|
||||
or not is_quantized_kv_cache(self.vllm_config.cache_config.cache_dtype)
|
||||
):
|
||||
return
|
||||
attn_layers = self.vllm_config.compilation_config.static_forward_context
|
||||
k_scale_tables: dict[str, torch.Tensor] = {}
|
||||
v_scale_tables: dict[str, torch.Tensor] = {}
|
||||
for layer_name in self.layer_names:
|
||||
layer = attn_layers[layer_name]
|
||||
num_blocks = layer.kv_cache.shape[0]
|
||||
k_scale_tables[layer_name] = self.asm_kv_scale_tables.get(
|
||||
self.device,
|
||||
num_blocks,
|
||||
self.num_heads_kv,
|
||||
self.block_size,
|
||||
layer._k_scale_float,
|
||||
)
|
||||
v_scale_tables[layer_name] = self.asm_kv_scale_tables.get(
|
||||
self.device,
|
||||
num_blocks,
|
||||
self.num_heads_kv,
|
||||
self.block_size,
|
||||
layer._v_scale_float,
|
||||
)
|
||||
self.k_scale, self.v_scale = k_scale_tables, v_scale_tables
|
||||
|
||||
def build(
|
||||
self,
|
||||
common_prefix_len: int,
|
||||
@@ -499,24 +569,7 @@ class AiterFlashAttentionMetadataBuilder(
|
||||
common_attn_metadata,
|
||||
decode_threshold=self.reorder_batch_threshold,
|
||||
)
|
||||
# Allocate scales for fp8 shuffle kv cache with shuffle_kv_cache enabled
|
||||
if (
|
||||
rocm_aiter_ops.is_shuffle_kv_cache_enabled()
|
||||
and self.scale.numel() == 1
|
||||
and is_quantized_kv_cache(self.vllm_config.cache_config.cache_dtype)
|
||||
):
|
||||
# Size the scales from a layer this builder owns. The draft model
|
||||
# runs its own builder over its own KV cache, so the first layer of
|
||||
# the whole config can carry an unrelated block count.
|
||||
kv_cache_shape = self.vllm_config.compilation_config.static_forward_context[
|
||||
self.layer_names[0]
|
||||
].kv_cache.shape
|
||||
num_blocks = kv_cache_shape[0]
|
||||
self.scale = torch.ones(
|
||||
[num_blocks, self.num_heads_kv, self.block_size],
|
||||
dtype=torch.float32,
|
||||
device=self.device,
|
||||
)
|
||||
self._build_asm_kv_scale_tables()
|
||||
(
|
||||
num_decodes,
|
||||
num_extends,
|
||||
@@ -709,8 +762,8 @@ class AiterFlashAttentionMetadataBuilder(
|
||||
decode_metadata=decode_metadata,
|
||||
prefill_metadata=prefill_metadata,
|
||||
extend_metadata=extend_metadata,
|
||||
k_scale=self.scale,
|
||||
v_scale=self.scale,
|
||||
k_scale=self.k_scale,
|
||||
v_scale=self.v_scale,
|
||||
kv_sharing_metadata=kv_sharing_metadata,
|
||||
)
|
||||
return attn_metadata
|
||||
@@ -726,6 +779,7 @@ class AiterFlashAttentionMetadataBuilder(
|
||||
skip split_decodes_prefills_and_extends() and avoid all .cpu() /
|
||||
.item() calls that would otherwise break CUDA graph capture.
|
||||
"""
|
||||
self._build_asm_kv_scale_tables()
|
||||
num_reqs = common_attn_metadata.num_reqs
|
||||
num_tokens = common_attn_metadata.num_actual_tokens
|
||||
|
||||
@@ -749,8 +803,8 @@ class AiterFlashAttentionMetadataBuilder(
|
||||
decode_metadata=decode_metadata,
|
||||
prefill_metadata=None,
|
||||
extend_metadata=None,
|
||||
k_scale=self.scale,
|
||||
v_scale=self.scale,
|
||||
k_scale=self.k_scale,
|
||||
v_scale=self.v_scale,
|
||||
)
|
||||
|
||||
|
||||
@@ -1249,9 +1303,6 @@ class AiterFlashAttentionImpl(AttentionImpl):
|
||||
extend_outputs = output[extend_tokens_slice]
|
||||
k_scale = layer._k_scale
|
||||
v_scale = layer._v_scale
|
||||
if rocm_aiter_ops.is_shuffle_kv_cache_enabled():
|
||||
k_scale = attn_metadata.k_scale
|
||||
v_scale = attn_metadata.v_scale
|
||||
self.extend_forward(
|
||||
attn_metadata=attn_metadata,
|
||||
query=extend_queries,
|
||||
@@ -1428,16 +1479,13 @@ class AiterFlashAttentionImpl(AttentionImpl):
|
||||
device=query.device,
|
||||
)
|
||||
max_logits = torch.empty_like(exp_sums)
|
||||
k_qscale = (
|
||||
layer._k_scale
|
||||
if attn_metadata.k_scale is None
|
||||
else attn_metadata.k_scale
|
||||
)
|
||||
v_qscale = (
|
||||
layer._v_scale
|
||||
if attn_metadata.v_scale is None
|
||||
else attn_metadata.v_scale
|
||||
)
|
||||
# asm indexes this layer's dense scale buffer from build();
|
||||
# the HIP kernel reads the per-tensor scale directly.
|
||||
k_qscale_asm = v_qscale_asm = None
|
||||
if attn_metadata.k_scale is not None:
|
||||
assert attn_metadata.v_scale is not None
|
||||
k_qscale_asm = attn_metadata.k_scale[layer.layer_name]
|
||||
v_qscale_asm = attn_metadata.v_scale[layer.layer_name]
|
||||
rocm_aiter_ops.paged_attention_common(
|
||||
Q=query[:num_decode_tokens],
|
||||
K=new_key_cache,
|
||||
@@ -1452,10 +1500,10 @@ class AiterFlashAttentionImpl(AttentionImpl):
|
||||
0
|
||||
),
|
||||
scale=self.scale,
|
||||
K_QScale_hip=k_qscale,
|
||||
V_QScale_hip=v_qscale,
|
||||
K_QScale_asm=k_qscale,
|
||||
V_QScale_asm=v_qscale,
|
||||
K_QScale_hip=layer._k_scale,
|
||||
V_QScale_hip=layer._v_scale,
|
||||
K_QScale_asm=k_qscale_asm,
|
||||
V_QScale_asm=v_qscale_asm,
|
||||
out_=output[:num_decode_tokens],
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
)
|
||||
@@ -1542,9 +1590,6 @@ class AiterFlashAttentionImpl(AttentionImpl):
|
||||
# the reshape_and_cache_flash op uses the slot_mapping's shape
|
||||
# to determine the number of actual tokens.
|
||||
if rocm_aiter_ops.is_shuffle_kv_cache_enabled():
|
||||
# We may calculate per token quant scale in
|
||||
# reshape_and_cache_shuffle_triton which might differ from
|
||||
# vllm's style when shuffle layout is used.
|
||||
k_scale = layer._k_scale
|
||||
v_scale = layer._v_scale
|
||||
assert k_scale is not None and v_scale is not None, (
|
||||
@@ -1581,12 +1626,7 @@ class AiterFlashAttentionImpl(AttentionImpl):
|
||||
)
|
||||
|
||||
def fused_qk_norm_rope_kvcache_supported(self):
|
||||
# Only fuse when shuffle layout is off; the shuffle write path uses a
|
||||
# dedicated cache update, mirroring fused_rope_kvcache_supported.
|
||||
return (
|
||||
rocm_aiter_ops.is_enabled()
|
||||
and not rocm_aiter_ops.is_shuffle_kv_cache_enabled()
|
||||
)
|
||||
return rocm_aiter_ops.is_enabled()
|
||||
|
||||
def do_qk_norm_rope_kvcache_update(
|
||||
self,
|
||||
@@ -1624,6 +1664,7 @@ class AiterFlashAttentionImpl(AttentionImpl):
|
||||
v_scale=layer._v_scale_cpu,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
use_shuffle_layout=rocm_aiter_ops.is_shuffle_kv_cache_enabled(),
|
||||
return_kv=True,
|
||||
)
|
||||
|
||||
def do_rope_and_kv_cache_update(
|
||||
|
||||
@@ -15,6 +15,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kFp8StaticTensorSym,
|
||||
)
|
||||
from vllm.platforms.rocm import on_gfx1250
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionLayer,
|
||||
@@ -28,7 +29,12 @@ from vllm.v1.attention.backends.rocm_attn import (
|
||||
RocmAttentionMetadata,
|
||||
RocmAttentionMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheLayout
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
KVCacheLayout,
|
||||
KVQuantMode,
|
||||
get_kv_quant_mode,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -186,12 +192,34 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl):
|
||||
# RDNA3 has no low precision MMA, preserve query precision
|
||||
self.supports_quant_query_input = not on_gfx11()
|
||||
|
||||
# Layout flag: must mirror get_kv_cache_shape, which is attn_type
|
||||
# agnostic because the allocation is shared across layers.
|
||||
self._shuffle_kv_layout = (
|
||||
rocm_aiter_ops.is_shuffle_kv_cache_enabled() and on_gfx1250()
|
||||
)
|
||||
# Kernel flag: the shuffled reader/writer is causal decoder only.
|
||||
self._use_shuffle_kv = (
|
||||
self._shuffle_kv_layout and self.attn_type != AttentionType.ENCODER_DECODER
|
||||
)
|
||||
|
||||
def _split_kv_cache(
|
||||
self, kv_cache: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if self._shuffle_kv_layout:
|
||||
# (2, B, N, H, hs) -> ((B, N, H, hs), (B, N, H, hs))
|
||||
return kv_cache.unbind(0)
|
||||
# (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs))
|
||||
return kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
|
||||
|
||||
def _shuffle_kv_views(
|
||||
self, key_cache: torch.Tensor, value_cache: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
nb, bs, nkv, hd = key_cache.shape
|
||||
x = 16 // key_cache.element_size()
|
||||
kc = key_cache.reshape(nb, nkv, hd // x, bs, x)
|
||||
vc = value_cache.reshape(nb, nkv, bs // x, hd, x)
|
||||
return kc, vc
|
||||
|
||||
def forward(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
@@ -264,6 +292,9 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl):
|
||||
key_cache = key_cache.view(self.fp8_dtype)
|
||||
value_cache = value_cache.view(self.fp8_dtype)
|
||||
|
||||
if self._use_shuffle_kv:
|
||||
key_cache, value_cache = self._shuffle_kv_views(key_cache, value_cache)
|
||||
|
||||
cu_seqlens_q = attn_metadata.query_start_loc
|
||||
seqused_k = attn_metadata.seq_lens
|
||||
max_seqlen_q = attn_metadata.max_query_len
|
||||
@@ -291,6 +322,7 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl):
|
||||
v_descale=layer._v_scale,
|
||||
sinks=self.sinks,
|
||||
output_scale=output_scale,
|
||||
shuffled_kv_cache=self._use_shuffle_kv,
|
||||
)
|
||||
else:
|
||||
# The aiter kernel is causal-only. Non-causal cross-attention
|
||||
@@ -339,6 +371,27 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl):
|
||||
return
|
||||
key_cache, value_cache = self._split_kv_cache(kv_cache)
|
||||
|
||||
if self._use_shuffle_kv:
|
||||
from vllm.v1.attention.backends.rocm_aiter_fa import (
|
||||
reshape_and_cache_shuffle_triton,
|
||||
)
|
||||
|
||||
if get_kv_quant_mode(self.kv_cache_dtype) == KVQuantMode.FP8_PER_TENSOR:
|
||||
key_cache = key_cache.view(self.fp8_dtype)
|
||||
value_cache = value_cache.view(self.fp8_dtype)
|
||||
_dummy = torch.empty(1, dtype=torch.float32, device=key.device)
|
||||
reshape_and_cache_shuffle_triton(
|
||||
key,
|
||||
value,
|
||||
key_cache,
|
||||
value_cache,
|
||||
slot_mapping,
|
||||
self.kv_cache_dtype,
|
||||
_dummy,
|
||||
_dummy,
|
||||
)
|
||||
return
|
||||
|
||||
# Reshape the input keys and values and store them in the cache.
|
||||
ops.reshape_and_cache_flash(
|
||||
key,
|
||||
@@ -360,6 +413,11 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl):
|
||||
def fused_qk_norm_mrope_kvcache_supported(self) -> bool:
|
||||
return IS_AITER_FOUND
|
||||
|
||||
def set_interleaved_v_cache(self):
|
||||
# No-op override of RocmAttentionImpl: this backend
|
||||
# decodes with the AITER triton unified-attention kernel
|
||||
pass
|
||||
|
||||
def do_qk_norm_rope_kvcache_update(
|
||||
self,
|
||||
layer: AttentionLayer,
|
||||
@@ -395,7 +453,8 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl):
|
||||
k_scale=layer._k_scale_cpu,
|
||||
v_scale=layer._v_scale_cpu,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
use_shuffle_layout=False,
|
||||
use_shuffle_layout=self._use_shuffle_kv,
|
||||
return_kv=False,
|
||||
)
|
||||
|
||||
def do_qk_norm_mrope_kvcache_update(
|
||||
@@ -456,13 +515,16 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl):
|
||||
# we use direct Q, K, V tensors without caching
|
||||
return
|
||||
key_cache, value_cache = self._split_kv_cache(kv_cache)
|
||||
flash_layout = True
|
||||
|
||||
is_fp8_kv_cache = is_quantized_kv_cache(self.kv_cache_dtype)
|
||||
if is_fp8_kv_cache:
|
||||
key_cache = key_cache.view(self.fp8_dtype)
|
||||
value_cache = value_cache.view(self.fp8_dtype)
|
||||
|
||||
if self._use_shuffle_kv:
|
||||
key_cache, value_cache = self._shuffle_kv_views(key_cache, value_cache)
|
||||
flash_layout = not self._use_shuffle_kv
|
||||
|
||||
rocm_aiter_ops.triton_rope_and_cache(
|
||||
query,
|
||||
key,
|
||||
|
||||
@@ -292,6 +292,8 @@ class RocmAttentionImpl(AttentionImpl):
|
||||
f"num_heads: {num_heads}."
|
||||
)
|
||||
|
||||
self._use_interleaved_v_cache = False
|
||||
|
||||
def _forward_encoder_attention(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
@@ -450,6 +452,7 @@ class RocmAttentionImpl(AttentionImpl):
|
||||
sm_scale=self.scale,
|
||||
output_scale=output_scale,
|
||||
sinks=self.sinks,
|
||||
use_interleaved_v_cache=self._use_interleaved_v_cache,
|
||||
causal=attn_metadata.causal,
|
||||
)
|
||||
|
||||
@@ -475,6 +478,12 @@ class RocmAttentionImpl(AttentionImpl):
|
||||
block_size = value_cache.shape[3]
|
||||
has_native_layout = has_native_kv_cache_layout(key_cache, value_cache)
|
||||
|
||||
# Match the fused writer's interleaved V-cache when fusion is active.
|
||||
interleaved_v = self._use_interleaved_v_cache
|
||||
interleaved_v_pack_factor = (
|
||||
(16 // value_cache.element_size()) if interleaved_v else 0
|
||||
)
|
||||
|
||||
if block_size in (16, 32) and has_native_layout:
|
||||
# Normal 16, 32 with contiguous blocks: use vLLM native HIP C++ logic.
|
||||
PagedAttention.write_to_paged_cache(
|
||||
@@ -486,6 +495,7 @@ class RocmAttentionImpl(AttentionImpl):
|
||||
self.kv_cache_dtype,
|
||||
layer._k_scale,
|
||||
layer._v_scale,
|
||||
interleaved_v,
|
||||
)
|
||||
else:
|
||||
# Non-standard blocks and hybrid attention/Mamba layouts need the
|
||||
@@ -501,8 +511,61 @@ class RocmAttentionImpl(AttentionImpl):
|
||||
self.kv_cache_dtype,
|
||||
layer._k_scale,
|
||||
layer._v_scale,
|
||||
interleaved_v_pack_factor,
|
||||
)
|
||||
|
||||
def fused_qk_norm_rope_kvcache_supported(self):
|
||||
return rocm_aiter_ops.is_enabled()
|
||||
|
||||
def set_interleaved_v_cache(self):
|
||||
# Decode/prefill kernels must read the interleaved V-cache the AITER
|
||||
# fused write emits when use_shuffle_layout=True.
|
||||
self._use_interleaved_v_cache = True
|
||||
|
||||
def do_qk_norm_rope_kvcache_update(
|
||||
self,
|
||||
layer: AttentionLayer,
|
||||
qkv: torch.Tensor,
|
||||
q_out: torch.Tensor,
|
||||
k_out: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
rms_norm_eps: float,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
is_neox: bool,
|
||||
kv_cache: torch.Tensor,
|
||||
layer_slot_mapping: torch.Tensor,
|
||||
):
|
||||
# kv_cache is logical [num_blocks, 2, block_size, num_kv_heads * head_size]
|
||||
# (see forward); the AITER shuffle writer takes one (B, N, H, hs) per side.
|
||||
key_cache, value_cache = kv_cache.unflatten(
|
||||
-1, (self.num_kv_heads, self.head_size)
|
||||
).unbind(1)
|
||||
use_shuffle_layout = self._use_interleaved_v_cache
|
||||
rocm_aiter_ops.do_qk_norm_rope_kvcache_update(
|
||||
qkv=qkv,
|
||||
q_weight=q_weight,
|
||||
k_weight=k_weight,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
positions=positions,
|
||||
num_heads_q=self.num_heads,
|
||||
num_heads_k=self.num_kv_heads,
|
||||
head_dim=self.head_size,
|
||||
is_neox=is_neox,
|
||||
rms_norm_eps=rms_norm_eps,
|
||||
q_out=q_out,
|
||||
k_out=k_out,
|
||||
key_cache=key_cache,
|
||||
value_cache=value_cache,
|
||||
slot_mapping=layer_slot_mapping,
|
||||
k_scale=layer._k_scale_cpu,
|
||||
v_scale=layer._v_scale_cpu,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
use_shuffle_layout=use_shuffle_layout,
|
||||
return_kv=True,
|
||||
)
|
||||
|
||||
def fused_rope_kvcache_supported(self):
|
||||
return rocm_aiter_ops.is_enabled()
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ from typing_extensions import runtime_checkable
|
||||
|
||||
from vllm.config import CacheConfig, VllmConfig, get_layers_from_vllm_config
|
||||
from vllm.config.cache import _layout_from_name
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d, np_to_pinned_tensor
|
||||
@@ -1160,6 +1161,75 @@ def get_dcp_local_seq_lens(
|
||||
return dcp_local_seq_lens
|
||||
|
||||
|
||||
@triton.jit(
|
||||
do_not_specialize_on_alignment=["out_ptr", "block_table_ptr", "seq_lens_ptr"]
|
||||
)
|
||||
def _mamba_align_block_table_kernel(
|
||||
out_ptr,
|
||||
block_table_ptr,
|
||||
block_table_stride_0: tl.int64,
|
||||
block_table_stride_1,
|
||||
seq_lens_ptr,
|
||||
seq_lens_stride,
|
||||
block_size,
|
||||
n_out,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
"""Gather the last ``n_out`` block-table entries of each request.
|
||||
|
||||
One program per request. This replaces a five-statement PyTorch expression
|
||||
that expanded to seven kernels over a ``(num_reqs,)`` tensor -- one element
|
||||
in a single-request decode -- so almost all of its cost was launch
|
||||
overhead rather than work.
|
||||
|
||||
Both block-table strides are passed explicitly. The expression this replaces
|
||||
was ``torch.gather(block_table, 1, ...)``, which honours any layout, and
|
||||
this is a shared utility -- ``gdn_attn``, ``mamba_attn``, ``short_conv_attn``
|
||||
and ``linear_attn`` all call it -- so assuming a column stride of 1 would
|
||||
silently read the wrong slots for a caller holding a non-contiguous view.
|
||||
``kimi_k3/nvidia/kda_metadata.py`` passes both strides for the same reason.
|
||||
|
||||
Two annotations above are load-bearing and easy to drop by accident:
|
||||
|
||||
``block_table_stride_0: tl.int64``
|
||||
Without it ``req_idx * block_table_stride_0`` is computed in i32 --
|
||||
``program_id`` is i32 and Triton specializes a stride below 2^31 to
|
||||
i32 -- so the product wraps once a row offset reaches 2^31 elements
|
||||
(a 4096 x 625000 table faults from row 3436 on MI355X).
|
||||
``torch.gather``, which this replaces, indexes in 64-bit and has no
|
||||
such limit, so narrowing it would be a regression.
|
||||
``mamba_utils.py:43`` annotates its row stride the same way.
|
||||
|
||||
``do_not_specialize_on_alignment`` on the three pointers
|
||||
Triton puts pointer alignment in the specialization key, and the MRV1
|
||||
DBO path hands this kernel ``seq_lens[request_slice]`` /
|
||||
``block_table_tensor[request_slice]`` VIEWS (ubatch_utils.py:278/316).
|
||||
When a ubatch starts at a request index that is not a multiple of 4
|
||||
the view is differently aligned and a fresh variant JIT-compiles
|
||||
mid-step -- 25-28 ms cold, ~1.5 ms warm, and an error under
|
||||
``jit_monitor_mode=error``. The torch expression this replaced never
|
||||
compiled anything, so that recompile would be a regression introduced
|
||||
by this change.
|
||||
"""
|
||||
req_idx = tl.program_id(0)
|
||||
seq_len = tl.load(seq_lens_ptr + req_idx * seq_lens_stride)
|
||||
# A zero-length request in a CUDA graph has an invalid block table; clamp
|
||||
# to 0 so it reads a valid row, exactly as the previous clamp_ did.
|
||||
start = tl.maximum((seq_len - 1) // block_size, 0)
|
||||
offs = tl.arange(0, BLOCK)
|
||||
mask = offs < n_out
|
||||
vals = tl.load(
|
||||
block_table_ptr
|
||||
+ req_idx * block_table_stride_0
|
||||
+ (start + offs) * block_table_stride_1,
|
||||
mask=mask,
|
||||
other=0,
|
||||
)
|
||||
# `out` is freshly allocated by the caller and therefore contiguous, so its
|
||||
# row stride is exactly n_out.
|
||||
tl.store(out_ptr + req_idx * n_out + offs, vals, mask=mask)
|
||||
|
||||
|
||||
def mamba_get_block_table_tensor(
|
||||
block_table: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
@@ -1180,16 +1250,58 @@ def mamba_get_block_table_tensor(
|
||||
return block_table
|
||||
else:
|
||||
assert isinstance(kv_cache_spec, MambaSpec)
|
||||
# NOTE: For 0-length requests in CUDA graph, use a start_index of 0
|
||||
# to handle the invalid block table.
|
||||
start_indices = (seq_lens - 1) // kv_cache_spec.block_size
|
||||
start_indices.clamp_(min=0)
|
||||
# Use int32 for arithmetic to avoid dtype promotion overhead,
|
||||
# then convert to int64 for gather (which requires Long indices)
|
||||
offsets = torch.arange(
|
||||
1 + kv_cache_spec.num_speculative_blocks,
|
||||
device=block_table.device,
|
||||
dtype=torch.int32,
|
||||
# The arithmetic here is trivial -- a divide, a clamp, an arange, an
|
||||
# add, a cast and a gather -- but as PyTorch ops it was seven kernel
|
||||
# launches over a (num_reqs,) tensor, which holds a single element in a
|
||||
# one-request decode. One Triton kernel does the same work in one
|
||||
# launch. Zero-length requests in a CUDA graph still clamp to row 0.
|
||||
num_reqs = block_table.shape[0]
|
||||
n_out = 1 + kv_cache_spec.num_speculative_blocks
|
||||
# The expression this replaced took its row count from `seq_lens` (a
|
||||
# `torch.gather` whose index tensor is built from it), so a mismatch
|
||||
# raised and a cross-device pair raised. Taking `num_reqs` from
|
||||
# `block_table` instead silently changed all of that: with 4 block-table
|
||||
# rows and 2 `seq_lens`, the old code returned 2 correct rows and the new
|
||||
# kernel returns 4, two of them read from memory past the end of
|
||||
# `seq_lens`; a pinned-CPU `seq_lens` with a GPU `block_table` used to
|
||||
# raise and would now be read zero-copy from host memory.
|
||||
#
|
||||
# No in-tree caller does any of that today, but this helper is shared by
|
||||
# gdn_attn, mamba_attn, short_conv_attn and linear_attn. Both are host
|
||||
# metadata, so this costs no synchronisation, and it keeps the CPU
|
||||
# branch below and the GPU kernel agreeing on the same inputs.
|
||||
assert seq_lens.shape[0] == num_reqs, (
|
||||
f"seq_lens has {seq_lens.shape[0]} rows, block_table has {num_reqs}"
|
||||
)
|
||||
indices_to_gather = (start_indices.unsqueeze(1) + offsets).to(torch.int64)
|
||||
return torch.gather(block_table, 1, indices_to_gather)
|
||||
assert seq_lens.device == block_table.device, (
|
||||
f"seq_lens is on {seq_lens.device}, block_table on {block_table.device}"
|
||||
)
|
||||
if not block_table.is_cuda:
|
||||
# Triton runs on the accelerator only, and this helper IS called
|
||||
# with CPU tensors: the GDN and linear-attention metadata builders
|
||||
# are unit-tested on CPU (`DEVICE = torch.device("cpu")`). The
|
||||
# expression this replaced was plain torch and handled that, so
|
||||
# keep it for the CPU path rather than narrowing the contract.
|
||||
# kimi_k3/nvidia/kda_metadata.py asserts `is_cuda` instead; a
|
||||
# shared utility cannot, because its callers legitimately pass CPU.
|
||||
start_indices = (seq_lens - 1) // kv_cache_spec.block_size
|
||||
start_indices = start_indices.clamp(min=0)
|
||||
offsets = torch.arange(n_out, device=block_table.device, dtype=torch.int32)
|
||||
return torch.gather(
|
||||
block_table, 1, (start_indices.unsqueeze(1) + offsets).to(torch.int64)
|
||||
)
|
||||
out = block_table.new_empty((num_reqs, n_out))
|
||||
if num_reqs == 0:
|
||||
return out
|
||||
_mamba_align_block_table_kernel[(num_reqs,)](
|
||||
out,
|
||||
block_table,
|
||||
block_table.stride(0),
|
||||
block_table.stride(1),
|
||||
seq_lens,
|
||||
seq_lens.stride(0),
|
||||
kv_cache_spec.block_size,
|
||||
n_out,
|
||||
BLOCK=triton.next_power_of_2(n_out),
|
||||
)
|
||||
return out
|
||||
|
||||
@@ -84,6 +84,7 @@ def kernel_paged_attention_2d(
|
||||
query_start_len_ptr, # [num_seqs+1]
|
||||
USE_SINKS: tl.constexpr, # bool
|
||||
USE_FP8: tl.constexpr,
|
||||
INTERLEAVED_V_PACK_FACTOR: tl.constexpr,
|
||||
FP8_MIN: tl.constexpr = float8_info.min,
|
||||
FP8_MAX: tl.constexpr = float8_info.max,
|
||||
):
|
||||
@@ -170,13 +171,24 @@ def kernel_paged_attention_2d(
|
||||
+ (offs_d[:, None] % x) * stride_k_cache_4
|
||||
)
|
||||
|
||||
# 4D addressing logic of V (Slot is innermost)
|
||||
v_offset = (
|
||||
p_block_idx[:, None] * stride_v_cache_0
|
||||
+ kv_head_idx * stride_v_cache_1
|
||||
+ offs_d[None, :] * stride_v_cache_2
|
||||
+ internal_offsets[:, None] * stride_v_cache_3
|
||||
)
|
||||
# V addressing: standard 4D or interleaved 5D
|
||||
if INTERLEAVED_V_PACK_FACTOR > 0:
|
||||
v_offset = (
|
||||
p_block_idx[:, None] * stride_v_cache_0
|
||||
+ kv_head_idx * stride_v_cache_1
|
||||
+ (internal_offsets[:, None] // INTERLEAVED_V_PACK_FACTOR)
|
||||
* HEAD_SIZE
|
||||
* INTERLEAVED_V_PACK_FACTOR
|
||||
+ offs_d[None, :] * INTERLEAVED_V_PACK_FACTOR
|
||||
+ (internal_offsets[:, None] % INTERLEAVED_V_PACK_FACTOR)
|
||||
)
|
||||
else:
|
||||
v_offset = (
|
||||
p_block_idx[:, None] * stride_v_cache_0
|
||||
+ kv_head_idx * stride_v_cache_1
|
||||
+ offs_d[None, :] * stride_v_cache_2
|
||||
+ internal_offsets[:, None] * stride_v_cache_3
|
||||
)
|
||||
|
||||
# Only the final tile can straddle seq_len. Slots >= seq_len are
|
||||
# unwritten KV cache that may hold NaN/garbage; they are score-masked
|
||||
@@ -304,6 +316,7 @@ def chunked_prefill_paged_decode(
|
||||
# Optional tensor for sinks
|
||||
sinks=None,
|
||||
is_block_table_ptr: bool = False,
|
||||
use_interleaved_v_cache: bool = False,
|
||||
causal: bool = True,
|
||||
):
|
||||
if sm_scale is None:
|
||||
@@ -314,6 +327,10 @@ def chunked_prefill_paged_decode(
|
||||
if sliding_window is None or sliding_window <= 0:
|
||||
sliding_window = 0
|
||||
|
||||
interleaved_v_pack_factor = (
|
||||
(16 // value_cache.element_size()) if use_interleaved_v_cache else 0
|
||||
)
|
||||
|
||||
if max_query_len > 1:
|
||||
context_attention_fwd(
|
||||
q=query,
|
||||
@@ -336,6 +353,7 @@ def chunked_prefill_paged_decode(
|
||||
skip_decode=True,
|
||||
fp8_out_scale=output_scale,
|
||||
sinks=sinks,
|
||||
interleaved_v_pack_factor=interleaved_v_pack_factor,
|
||||
causal=causal,
|
||||
)
|
||||
|
||||
@@ -428,6 +446,7 @@ def chunked_prefill_paged_decode(
|
||||
k_scale=k_scale,
|
||||
v_scale=v_scale,
|
||||
fp8_out_scale=output_scale,
|
||||
use_interleaved_v_cache=use_interleaved_v_cache,
|
||||
)
|
||||
else:
|
||||
logger.warning_once(
|
||||
@@ -504,4 +523,5 @@ def chunked_prefill_paged_decode(
|
||||
query_start_len_ptr=query_start_loc,
|
||||
USE_SINKS=sinks is not None,
|
||||
USE_FP8=output_scale is not None,
|
||||
INTERLEAVED_V_PACK_FACTOR=interleaved_v_pack_factor,
|
||||
)
|
||||
|
||||
@@ -38,6 +38,7 @@ class PagedAttention:
|
||||
kv_cache_dtype: str,
|
||||
k_scale: torch.Tensor,
|
||||
v_scale: torch.Tensor,
|
||||
interleaved_v: bool,
|
||||
) -> None:
|
||||
ops.reshape_and_cache(
|
||||
key,
|
||||
@@ -48,4 +49,5 @@ class PagedAttention:
|
||||
kv_cache_dtype,
|
||||
k_scale,
|
||||
v_scale,
|
||||
interleaved_v,
|
||||
)
|
||||
|
||||
@@ -56,6 +56,8 @@ def _paged_kv_cache_offsets(
|
||||
stride_v_cache_d,
|
||||
stride_v_cache_bl,
|
||||
PHYSICAL_BLOCK_SIZE: tl.constexpr,
|
||||
BLOCK_DMODEL: tl.constexpr,
|
||||
INTERLEAVED_V_PACK_FACTOR: tl.constexpr = 0,
|
||||
MASK_BLOCK_TABLE: tl.constexpr = False,
|
||||
):
|
||||
"""Compute paged K/V cache element offsets for a tile of token positions.
|
||||
@@ -65,7 +67,9 @@ def _paged_kv_cache_offsets(
|
||||
tiles), then the physical element offsets are built with the same layout the
|
||||
kernel uses everywhere:
|
||||
K cache: [num_blocks, num_kv_heads, head_size // x, block_size, x]
|
||||
V cache: [num_blocks, num_kv_heads, head_size, block_size]
|
||||
V cache: [num_blocks, num_kv_heads, head_size, block_size], or the
|
||||
interleaved layout [num_blocks, num_kv_heads, block_size // x,
|
||||
head_size, x] when `INTERLEAVED_V_PACK_FACTOR` (= x) > 0.
|
||||
Returns `(off_k, off_v)` with shapes [D, N] and [N, D] respectively.
|
||||
|
||||
When `MASK_BLOCK_TABLE` is set, the block-table load is masked with
|
||||
@@ -94,12 +98,25 @@ def _paged_kv_cache_offsets(
|
||||
+ internal[None, :] * stride_k_cache_bl
|
||||
+ (offs_d[:, None] % x) * stride_k_cache_x
|
||||
)
|
||||
off_v = (
|
||||
bn[:, None] * stride_v_cache_bs
|
||||
+ cur_kv_head * stride_v_cache_h
|
||||
+ offs_d[None, :] * stride_v_cache_d
|
||||
+ internal[:, None] * stride_v_cache_bl
|
||||
)
|
||||
if INTERLEAVED_V_PACK_FACTOR > 0:
|
||||
# Interleaved V-cache: [num_blocks, num_kv_heads, block_size/x,
|
||||
# head_size, x] flattened onto the 4D stride layout.
|
||||
off_v = (
|
||||
bn[:, None] * stride_v_cache_bs
|
||||
+ cur_kv_head * stride_v_cache_h
|
||||
+ (internal[:, None] // INTERLEAVED_V_PACK_FACTOR)
|
||||
* BLOCK_DMODEL
|
||||
* INTERLEAVED_V_PACK_FACTOR
|
||||
+ offs_d[None, :] * INTERLEAVED_V_PACK_FACTOR
|
||||
+ (internal[:, None] % INTERLEAVED_V_PACK_FACTOR)
|
||||
)
|
||||
else:
|
||||
off_v = (
|
||||
bn[:, None] * stride_v_cache_bs
|
||||
+ cur_kv_head * stride_v_cache_h
|
||||
+ offs_d[None, :] * stride_v_cache_d
|
||||
+ internal[:, None] * stride_v_cache_bl
|
||||
)
|
||||
return off_k, off_v
|
||||
|
||||
|
||||
@@ -159,6 +176,7 @@ def _fwd_kernel(
|
||||
USE_FP8: tl.constexpr,
|
||||
CAUSAL: tl.constexpr = True,
|
||||
MAX_Q_LEN: tl.constexpr = 0,
|
||||
INTERLEAVED_V_PACK_FACTOR: tl.constexpr = 0,
|
||||
MAX_CTX_LEN: tl.constexpr = 0,
|
||||
FP8_MIN: tl.constexpr = float8_info.min,
|
||||
FP8_MAX: tl.constexpr = float8_info.max,
|
||||
@@ -263,6 +281,8 @@ def _fwd_kernel(
|
||||
stride_v_cache_d,
|
||||
stride_v_cache_bl,
|
||||
PHYSICAL_BLOCK_SIZE,
|
||||
BLOCK_DMODEL,
|
||||
INTERLEAVED_V_PACK_FACTOR,
|
||||
)
|
||||
|
||||
if (
|
||||
@@ -403,6 +423,8 @@ def _fwd_kernel(
|
||||
stride_v_cache_d,
|
||||
stride_v_cache_bl,
|
||||
PHYSICAL_BLOCK_SIZE,
|
||||
BLOCK_DMODEL,
|
||||
INTERLEAVED_V_PACK_FACTOR,
|
||||
MASK_BLOCK_TABLE=True,
|
||||
)
|
||||
k_cur_load = tl.load(
|
||||
@@ -540,6 +562,7 @@ def _fwd_kernel_alibi(
|
||||
BLOCK_DMODEL_PADDED: tl.constexpr, # head size padded to a power of 2
|
||||
BLOCK_N: tl.constexpr,
|
||||
SKIP_DECODE: tl.constexpr,
|
||||
INTERLEAVED_V_PACK_FACTOR: tl.constexpr,
|
||||
):
|
||||
# attn_bias[]
|
||||
cur_batch = tl.program_id(0)
|
||||
@@ -608,12 +631,24 @@ def _fwd_kernel_alibi(
|
||||
+ ((start_n + offs_n[None, :]) % block_size) * stride_k_cache_bl
|
||||
+ (offs_d[:, None] % x) * stride_k_cache_x
|
||||
)
|
||||
off_v = (
|
||||
bn[:, None] * stride_v_cache_bs
|
||||
+ cur_kv_head * stride_v_cache_h
|
||||
+ offs_d[None, :] * stride_v_cache_d
|
||||
+ (start_n + offs_n[:, None]) % block_size * stride_v_cache_bl
|
||||
)
|
||||
if INTERLEAVED_V_PACK_FACTOR > 0:
|
||||
v_internal = (start_n + offs_n[:, None]) % block_size
|
||||
off_v = (
|
||||
bn[:, None] * stride_v_cache_bs
|
||||
+ cur_kv_head * stride_v_cache_h
|
||||
+ (v_internal // INTERLEAVED_V_PACK_FACTOR)
|
||||
* BLOCK_DMODEL
|
||||
* INTERLEAVED_V_PACK_FACTOR
|
||||
+ offs_d[None, :] * INTERLEAVED_V_PACK_FACTOR
|
||||
+ (v_internal % INTERLEAVED_V_PACK_FACTOR)
|
||||
)
|
||||
else:
|
||||
off_v = (
|
||||
bn[:, None] * stride_v_cache_bs
|
||||
+ cur_kv_head * stride_v_cache_h
|
||||
+ offs_d[None, :] * stride_v_cache_d
|
||||
+ (start_n + offs_n[:, None]) % block_size * stride_v_cache_bl
|
||||
)
|
||||
k_load = tl.load(
|
||||
K_cache + off_k,
|
||||
mask=dim_mask[:, None] & ((start_n + offs_n[None, :]) < cur_batch_ctx_len),
|
||||
@@ -796,6 +831,7 @@ def context_attention_fwd(
|
||||
fp8_out_scale=None,
|
||||
sinks=None,
|
||||
is_block_table_ptr: bool = False,
|
||||
interleaved_v_pack_factor: int = 0,
|
||||
causal: bool = True,
|
||||
):
|
||||
q_dtype_is_f32 = q.dtype is torch.float32
|
||||
@@ -937,6 +973,7 @@ def context_attention_fwd(
|
||||
BLOCK_DMODEL_PADDED=Lk_padded,
|
||||
BLOCK_N=BLOCK,
|
||||
SKIP_DECODE=skip_decode,
|
||||
INTERLEAVED_V_PACK_FACTOR=interleaved_v_pack_factor,
|
||||
num_warps=NUM_WARPS,
|
||||
num_stages=1,
|
||||
)
|
||||
@@ -1010,6 +1047,7 @@ def context_attention_fwd(
|
||||
num_warps=4,
|
||||
num_stages=1,
|
||||
USE_SINKS=sinks is not None,
|
||||
INTERLEAVED_V_PACK_FACTOR=interleaved_v_pack_factor,
|
||||
CAUSAL=causal,
|
||||
KV_FROM_CACHE=kv_from_cache,
|
||||
**extra_kargs,
|
||||
|
||||
@@ -52,6 +52,9 @@ def reshape_and_cache_kernel_flash(
|
||||
block_size: tl.constexpr,
|
||||
x: tl.constexpr,
|
||||
USE_HEAD_MAJOR_LAYOUT: tl.constexpr,
|
||||
# >0 writes V interleaved as [Block, Head, Slot//kx, Dim, kx] (head-major
|
||||
# layout only); 0 keeps the standard [Block, Head, Dim, Slot] V layout.
|
||||
INTERLEAVED_V_PACK_FACTOR: tl.constexpr,
|
||||
# FP8 flags
|
||||
FP8_KV_CACHE: tl.constexpr,
|
||||
# tune parameters
|
||||
@@ -76,13 +79,25 @@ def reshape_and_cache_kernel_flash(
|
||||
# Decompose the tile index back into head and dim coordinates.
|
||||
cur_head = tile_pos // head_size
|
||||
cur_dim = tile_pos % head_size
|
||||
# Value addressing (4D): [Block, Head, Dim, Slot]
|
||||
tgt_idx_v = (
|
||||
block_idx * block_stride
|
||||
+ cur_head * head_stride
|
||||
+ cur_dim * dim_stride_v
|
||||
+ block_offset * 1
|
||||
)
|
||||
# Value addressing: standard 4D [Block, Head, Dim, Slot], or the
|
||||
# interleaved 5D [Block, Head, Slot//kx, Dim, kx] the fused AITER
|
||||
# writer emits and the interleaved read path expects.
|
||||
if INTERLEAVED_V_PACK_FACTOR > 0:
|
||||
tgt_idx_v = (
|
||||
block_idx * block_stride
|
||||
+ cur_head * head_stride
|
||||
+ (block_offset // INTERLEAVED_V_PACK_FACTOR)
|
||||
* (head_size * INTERLEAVED_V_PACK_FACTOR)
|
||||
+ cur_dim * INTERLEAVED_V_PACK_FACTOR
|
||||
+ (block_offset % INTERLEAVED_V_PACK_FACTOR)
|
||||
)
|
||||
else:
|
||||
tgt_idx_v = (
|
||||
block_idx * block_stride
|
||||
+ cur_head * head_stride
|
||||
+ cur_dim * dim_stride_v
|
||||
+ block_offset * 1
|
||||
)
|
||||
# Key addressing (5D): [Block, Head, Dim//8, Slot, 8]
|
||||
tgt_idx_k = (
|
||||
block_idx * block_stride
|
||||
@@ -371,11 +386,18 @@ def triton_reshape_and_cache_flash(
|
||||
kv_cache_dtype: str, # "auto", "fp8"
|
||||
k_scale: torch.Tensor, # float32
|
||||
v_scale: torch.Tensor, # float32
|
||||
interleaved_v_pack_factor: int = 0, # >0: interleave V (head-major); 0: standard
|
||||
):
|
||||
num_heads = key.shape[1]
|
||||
head_size = key.shape[2]
|
||||
|
||||
use_head_major_layout = key_cache.ndim == 5
|
||||
if interleaved_v_pack_factor and not use_head_major_layout:
|
||||
raise NotImplementedError(
|
||||
"Interleaved V-cache write is only implemented for the head-major "
|
||||
"(5D) cache layout; got a non-head-major cache with "
|
||||
f"interleaved_v_pack_factor={interleaved_v_pack_factor}."
|
||||
)
|
||||
if use_head_major_layout:
|
||||
block_size = key_cache.shape[3]
|
||||
x = key_cache.shape[4]
|
||||
@@ -452,6 +474,7 @@ def triton_reshape_and_cache_flash(
|
||||
block_size=block_size,
|
||||
x=x,
|
||||
USE_HEAD_MAJOR_LAYOUT=use_head_major_layout,
|
||||
INTERLEAVED_V_PACK_FACTOR=interleaved_v_pack_factor,
|
||||
FP8_KV_CACHE=FP8_KV_CACHE,
|
||||
# autotune parameters
|
||||
TILE_SIZE=TILE_SIZE,
|
||||
|
||||
@@ -759,9 +759,11 @@ class BlockPool:
|
||||
|
||||
"""
|
||||
for block in blocks:
|
||||
if block.is_null:
|
||||
continue
|
||||
# ref_cnt=0 means this block is in the free list (i.e. eviction
|
||||
# candidate), so remove it.
|
||||
if block.ref_cnt == 0 and not block.is_null:
|
||||
if block.ref_cnt == 0:
|
||||
self.free_block_queue.remove(block)
|
||||
block.ref_cnt += 1
|
||||
if self.metrics_collector:
|
||||
@@ -785,11 +787,13 @@ class BlockPool:
|
||||
blocks_to_evict_first = []
|
||||
other_pools: dict[BlockPool, list[KVCacheBlock]] = {}
|
||||
for block in ordered_blocks:
|
||||
if block.is_null: # never pooled; its ref_cnt is not maintained
|
||||
continue
|
||||
if block.pool is not None and block.pool is not self:
|
||||
other_pools.setdefault(block.pool, []).append(block)
|
||||
continue
|
||||
block.ref_cnt -= 1
|
||||
if block.ref_cnt == 0 and not block.is_null:
|
||||
if block.ref_cnt == 0:
|
||||
if block.block_hash is None or not self.enable_caching:
|
||||
# LIFO reuse of non-cached blocks for better GPU locality.
|
||||
blocks_to_evict_first.append(block)
|
||||
|
||||
@@ -14,7 +14,7 @@ from vllm.v1.core.kv_cache_coordinator import (
|
||||
get_kv_cache_coordinator,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlock, KVCacheBlockCopy
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlock, KVCacheBlockCopy, trim_null_head
|
||||
from vllm.v1.core.single_type_kv_cache_manager import MambaManager
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
@@ -117,7 +117,7 @@ class KVCacheBlocks:
|
||||
return [
|
||||
[
|
||||
block.block_id
|
||||
for block in group
|
||||
for block in trim_null_head(group)
|
||||
if block.block_hash is None and not block.is_null
|
||||
]
|
||||
for group in self.blocks
|
||||
|
||||
@@ -8,7 +8,15 @@ import math
|
||||
import os
|
||||
from collections.abc import Callable, Iterable, Iterator, Sequence
|
||||
from dataclasses import dataclass, field, replace
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple, NewType, TypeAlias, overload
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
NamedTuple,
|
||||
NewType,
|
||||
TypeAlias,
|
||||
TypeVar,
|
||||
overload,
|
||||
)
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config import VllmConfig
|
||||
@@ -216,6 +224,32 @@ class KVCacheBlock:
|
||||
)
|
||||
|
||||
|
||||
def null_head_len(blocks: Sequence[KVCacheBlock]) -> int:
|
||||
"""Number of leading null blocks, e.g. a sliding-window table's skipped prefix."""
|
||||
if not blocks or not blocks[0].is_null:
|
||||
return 0
|
||||
lo, hi = 1, len(blocks)
|
||||
while lo < hi:
|
||||
mid = (lo + hi) // 2
|
||||
if blocks[mid].is_null:
|
||||
lo = mid + 1
|
||||
else:
|
||||
hi = mid
|
||||
# All null entries are one shared object, so count() checks the run at C speed.
|
||||
if blocks[:lo].count(blocks[0]) == lo:
|
||||
return lo
|
||||
return next(i for i, b in enumerate(blocks) if not b.is_null)
|
||||
|
||||
|
||||
_BlockSeq = TypeVar("_BlockSeq", list[KVCacheBlock], Sequence[KVCacheBlock])
|
||||
|
||||
|
||||
def trim_null_head(blocks: _BlockSeq) -> _BlockSeq:
|
||||
"""``blocks`` without its leading null blocks; ``blocks`` itself if none."""
|
||||
nhl = null_head_len(blocks)
|
||||
return blocks[nhl:] if nhl else blocks
|
||||
|
||||
|
||||
class KVCacheBlockCopy(NamedTuple):
|
||||
src_block_id: int
|
||||
dst_block_id: int
|
||||
|
||||
@@ -60,14 +60,16 @@ class AsyncScheduler(Scheduler):
|
||||
self, request: Request, new_token_ids: list[int], is_stale: bool = False
|
||||
) -> tuple[list[int], bool]:
|
||||
status_before_update = request.status
|
||||
# Tokens trimmed at a stop resolve their placeholders too.
|
||||
num_new_tokens = len(new_token_ids)
|
||||
new_token_ids, stopped = super()._update_request_with_output(
|
||||
request, new_token_ids
|
||||
request, new_token_ids, is_stale
|
||||
)
|
||||
|
||||
# Placeholders were zeroed at preemption; a stale delivery must not
|
||||
# decrement them (it would underflow).
|
||||
if not is_stale:
|
||||
request.num_output_placeholders -= len(new_token_ids)
|
||||
request.num_output_placeholders -= num_new_tokens
|
||||
assert request.num_output_placeholders >= 0
|
||||
|
||||
# Cache the new tokens. Preempted requests should be skipped.
|
||||
|
||||
@@ -368,6 +368,36 @@ class Scheduler(SchedulerInterface):
|
||||
# prefill batch fully drained the waiting queue. Prefill throttling
|
||||
# is disabled in this case.
|
||||
self.prefill_capacity_bound = False
|
||||
# The DP engine core drives this cadence from a counter shared across
|
||||
# ranks; an engine without one never throttles, so apply the interval
|
||||
# locally, timed from the last prefill-carrying step. Three regimes:
|
||||
# 1. no DP -- local cadence applies;
|
||||
# 2. dense DP rank -- `reconfigure_for_independent_dp_rank` already
|
||||
# set data_parallel_size=1, so each rank gets its own cadence.
|
||||
# Intended, but a behaviour change: this used to be a no-op;
|
||||
# 3. MoE DP rank (size > 1) -- the engine core drives it, no-op here.
|
||||
self.local_prefill_interval = (
|
||||
self.scheduler_config.prefill_schedule_interval
|
||||
if self.parallel_config.data_parallel_size == 1
|
||||
else 1
|
||||
)
|
||||
if self.local_prefill_interval > 1:
|
||||
# Neither the hold nor its release is otherwise visible in logs or
|
||||
# stats, which makes a misapplied cadence hard to diagnose.
|
||||
logger.info_once(
|
||||
"Prefill cadence: applying prefill_schedule_interval=%d "
|
||||
"locally (this engine is not DP-coordinated); prefill is held "
|
||||
"until %d executed steps have passed since the last "
|
||||
"prefill-carrying step.",
|
||||
self.local_prefill_interval,
|
||||
self.local_prefill_interval,
|
||||
)
|
||||
# A full interval back, so the first step is never gated.
|
||||
self.last_prefill_step = -self.local_prefill_interval
|
||||
# Steps that actually scheduled tokens. `current_step` counts every
|
||||
# schedule() call including empty ones, so it would expire the hold
|
||||
# early; it keeps its own meaning (next_decode_eligible_step).
|
||||
self.num_executed_steps = 0
|
||||
self.scheduler_reserve_full_isl = (
|
||||
self.scheduler_config.scheduler_reserve_full_isl
|
||||
)
|
||||
@@ -623,6 +653,9 @@ class Scheduler(SchedulerInterface):
|
||||
scheduled_spec_decode_tokens: dict[str, list[int]] = {}
|
||||
# Whether the running batch contains any prefill requests.
|
||||
prefill_scheduled = False
|
||||
# Whether a waiting request was admitted with prompt left to compute.
|
||||
# Separate from `prefill_scheduled`, which covers only the running loop.
|
||||
prefill_admitted = False
|
||||
# Whether any scheduled request has a synchronous connector KV load.
|
||||
has_sync_kv_loads = False
|
||||
|
||||
@@ -654,6 +687,28 @@ class Scheduler(SchedulerInterface):
|
||||
long_prefill_token_threshold, input_budget // num_eligible_reqs
|
||||
)
|
||||
|
||||
# Local prefill cadence: hold prefill until the interval since the last
|
||||
# prefill-carrying step has elapsed, so decode runs in longer
|
||||
# uninterrupted stretches. Mirrors the DP gate above, including its
|
||||
# capacity escape: once a release step cannot drain the waiting queue,
|
||||
# throttling stops (#44558, test_throttle_capacity_bound_guard_admits).
|
||||
# Without that escape a backed-up engine holds both admissions and
|
||||
# in-flight chunks for interval-1 of every interval steps, capping
|
||||
# prefill at roughly one budget per interval and growing the waiting
|
||||
# queue without bound.
|
||||
if (
|
||||
not defer_prefills
|
||||
and self.local_prefill_interval > 1
|
||||
and not self.prefill_capacity_bound
|
||||
# This call becomes executed step num_executed_steps + 1 if it
|
||||
# schedules anything, and last_prefill_step is stamped on the same
|
||||
# convention below.
|
||||
and (self.num_executed_steps + 1) - self.last_prefill_step
|
||||
< self.local_prefill_interval
|
||||
and any(not r.is_prefill_chunk for r in self.running)
|
||||
):
|
||||
defer_prefills = True
|
||||
|
||||
# First, schedule the RUNNING requests.
|
||||
req_index = 0
|
||||
while req_index < len(self.running) and token_budget > 0:
|
||||
@@ -1318,6 +1373,12 @@ class Scheduler(SchedulerInterface):
|
||||
)
|
||||
|
||||
if load_kv_async:
|
||||
# This path deliberately does not reach `prefill_admitted`
|
||||
# below, so it does not restart the prefill cadence:
|
||||
# `num_new_tokens` was set to 0 above, so the step carries
|
||||
# no prefill compute for this request. The interval is
|
||||
# timed from steps that actually prefilled; refreshing it
|
||||
# here would start the cadence from a step that did none.
|
||||
# If loading async, allocate memory and put request
|
||||
# into the WAITING_FOR_REMOTE_KV state.
|
||||
request.status = RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
@@ -1373,6 +1434,14 @@ class Scheduler(SchedulerInterface):
|
||||
scheduled_spec_decode_tokens[request_id] = [
|
||||
-1
|
||||
] * self.num_spec_tokens
|
||||
# Must use the SAME predicate as the deferral gate in the
|
||||
# waiting loop (`num_computed_tokens < num_tokens - 1`).
|
||||
# Stamping on `< num_prompt_tokens` instead let a request with
|
||||
# exactly one token left pass the gate on a held step and still
|
||||
# restart the interval -- which is every full prefix-cache hit,
|
||||
# and every remote-KV resume, since those set
|
||||
# num_computed_tokens = num_tokens - 1.
|
||||
prefill_admitted |= num_computed_tokens < request.num_tokens - 1
|
||||
# Only track requests that will still be prefilling after this chunk.
|
||||
if num_computed_tokens + num_new_tokens < request.num_tokens:
|
||||
self._inflight_prefills.add(request)
|
||||
@@ -1403,6 +1472,15 @@ class Scheduler(SchedulerInterface):
|
||||
# Check if the scheduling constraints are satisfied.
|
||||
total_num_scheduled_tokens = sum(num_scheduled_tokens.values())
|
||||
assert total_num_scheduled_tokens <= self.max_num_scheduled_tokens
|
||||
# Count this step before stamping the cadence below, so that
|
||||
# last_prefill_step includes the step that carried the prefill.
|
||||
if total_num_scheduled_tokens > 0:
|
||||
self.num_executed_steps += 1
|
||||
|
||||
# Restart the cadence interval from any step that carried prefill,
|
||||
# whether a new admission or a running chunk.
|
||||
if self.local_prefill_interval > 1 and (prefill_scheduled or prefill_admitted):
|
||||
self.last_prefill_step = self.num_executed_steps
|
||||
|
||||
assert token_budget >= 0
|
||||
assert input_budget >= 0
|
||||
@@ -2496,7 +2574,6 @@ class Scheduler(SchedulerInterface):
|
||||
def _update_request_with_output(
|
||||
self, request: Request, new_token_ids: list[int], is_stale: bool = False
|
||||
) -> tuple[list[int], bool]:
|
||||
# is_stale is only used by the AsyncScheduler override.
|
||||
# Append generated tokens and check for stop. Note that if
|
||||
# a request is still being prefilled, we expect the model runner
|
||||
# to return empty token ids for the request.
|
||||
@@ -2508,6 +2585,10 @@ class Scheduler(SchedulerInterface):
|
||||
# This must be called before we make the EngineCoreOutput.
|
||||
stopped = check_stop(request, self.max_model_len)
|
||||
if stopped:
|
||||
# num_computed_tokens covers all new tokens but the last one.
|
||||
# Keep that true after trimming: resumable requests resume from it.
|
||||
if not is_stale:
|
||||
request.num_computed_tokens -= len(new_token_ids) - num_new
|
||||
del new_token_ids[num_new:] # Trim new tokens if needed.
|
||||
break
|
||||
return new_token_ids, stopped
|
||||
|
||||
@@ -16,6 +16,7 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHashWithGroupId,
|
||||
KVCacheBlock,
|
||||
resolve_block_hashes,
|
||||
trim_null_head,
|
||||
)
|
||||
from vllm.v1.hisparse.block_pool import SharedEventQueueBlockPool
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
@@ -569,14 +570,15 @@ class SingleTypeKVCacheManager(ABC):
|
||||
request_id: The request ID.
|
||||
|
||||
Returns:
|
||||
The request's blocks in allocation order.
|
||||
The request's blocks in allocation order, without the leading
|
||||
null blocks (e.g. a sliding-window table's skipped prefix).
|
||||
|
||||
"""
|
||||
# Default to [] in case a request is freed (aborted) before alloc.
|
||||
req_blocks = self.req_to_blocks.pop(request_id, [])
|
||||
self.num_cached_block.pop(request_id, None)
|
||||
self._partial_hit_reqs.pop(request_id, None)
|
||||
return req_blocks
|
||||
return trim_null_head(req_blocks)
|
||||
|
||||
def free(self, request_id: str) -> None:
|
||||
"""Free the blocks for the request.
|
||||
@@ -2547,7 +2549,7 @@ class HiSparseResidentManager(_HiSparseAuxiliaryManager):
|
||||
blocks = self.get_residency_row(request_id)
|
||||
self.residency_changes.pop(request_id, None)
|
||||
super().pop_blocks_for_free(request_id)
|
||||
return blocks
|
||||
return trim_null_head(blocks)
|
||||
|
||||
def adopt_resident_page(
|
||||
self, request_id: str, block_idx: int, block: KVCacheBlock
|
||||
|
||||
@@ -104,6 +104,9 @@ class EngineCoreReadyResponse:
|
||||
supports_draft_weight_updates: bool = False
|
||||
# Full-attention block size in tokens after initialization, or unavailable.
|
||||
effective_attention_block_size: int | None = None
|
||||
coord_store_port: int = 0
|
||||
coordinator_input_address: str | None = None
|
||||
coordinator_output_address: str | None = None
|
||||
|
||||
|
||||
class EngineCoreRequest(
|
||||
|
||||
+75
-11
@@ -22,7 +22,11 @@ from vllm.distributed.weight_transfer.base import (
|
||||
)
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.engine.protocol import EngineClient, StreamingInput
|
||||
from vllm.entrypoints.serve.elastic_ep.middleware import set_scaling_elastic_ep
|
||||
from vllm.entrypoints.serve.elastic_ep.middleware import (
|
||||
get_elastic_ep_rank_retired,
|
||||
set_elastic_ep_rank_retired,
|
||||
set_scaling_elastic_ep,
|
||||
)
|
||||
from vllm.exceptions import (
|
||||
GracefulHTTPError,
|
||||
MaxQueuedTokensError,
|
||||
@@ -53,7 +57,11 @@ from vllm.v1.engine.admission_control import SharedAdmissionStats
|
||||
from vllm.v1.engine.core_client import EngineCoreClient
|
||||
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
|
||||
from vllm.v1.engine.input_processor import InputProcessor
|
||||
from vllm.v1.engine.output_processor import OutputProcessor, RequestOutputCollector
|
||||
from vllm.v1.engine.output_processor import (
|
||||
OutputProcessor,
|
||||
RequestOutputCollector,
|
||||
StreamingInputOutputCollector,
|
||||
)
|
||||
from vllm.v1.engine.parallel_sampling import ParentRequest
|
||||
from vllm.v1.executor import Executor
|
||||
from vllm.v1.fault_tolerance.utils import FaultToleranceRequest, FaultToleranceResult
|
||||
@@ -620,7 +628,9 @@ class AsyncLLM(EngineClient):
|
||||
self.input_processor.assign_request_id(final_req)
|
||||
internal_req_id = final_req.request_id
|
||||
|
||||
queue = RequestOutputCollector(sampling_params.output_kind, internal_req_id)
|
||||
queue = StreamingInputOutputCollector(
|
||||
sampling_params.output_kind, internal_req_id
|
||||
)
|
||||
|
||||
async def handle_inputs():
|
||||
cancelled = False
|
||||
@@ -628,6 +638,13 @@ class AsyncLLM(EngineClient):
|
||||
any_added = False
|
||||
try:
|
||||
async for input_chunk in input_stream:
|
||||
# Stop once the session has finished (aborted or errored).
|
||||
# There is no await between this check and the ADD send,
|
||||
# so any chunk already sent precedes the engine abort.
|
||||
if any_added and not self.output_processor.has_request(
|
||||
internal_req_id
|
||||
):
|
||||
break
|
||||
sp = input_chunk.sampling_params
|
||||
if sp:
|
||||
self._validate_streaming_input_sampling_params(sp)
|
||||
@@ -663,8 +680,10 @@ class AsyncLLM(EngineClient):
|
||||
if not cancelled:
|
||||
if any_added:
|
||||
# Send empty final request to indicate that inputs have
|
||||
# finished. Don't send if cancelled (session was aborted).
|
||||
await self._add_request(final_req, None, None, 0, queue)
|
||||
# finished. Don't send if cancelled or if the session
|
||||
# already finished (aborted or errored).
|
||||
if self.output_processor.has_request(internal_req_id):
|
||||
await self._add_request(final_req, None, None, 0, queue)
|
||||
elif not errored:
|
||||
queue.put(STREAM_FINISHED)
|
||||
|
||||
@@ -903,6 +922,12 @@ class AsyncLLM(EngineClient):
|
||||
mm_cache_stats=renderer.stat_mm_cache(),
|
||||
)
|
||||
except Exception as e:
|
||||
if get_elastic_ep_rank_retired() and isinstance(e, EngineDeadError):
|
||||
logger.info(
|
||||
"[Elastic EP] Retired rank stopped receiving EngineCore outputs"
|
||||
)
|
||||
output_processor.propagate_error(e)
|
||||
return
|
||||
logger.exception("AsyncLLM output_handler failed.")
|
||||
output_processor.propagate_error(e)
|
||||
|
||||
@@ -1088,6 +1113,8 @@ class AsyncLLM(EngineClient):
|
||||
|
||||
async def check_health(self) -> None:
|
||||
logger.debug("Called check_health.")
|
||||
if get_elastic_ep_rank_retired():
|
||||
return
|
||||
if self.errored:
|
||||
raise self.dead_error
|
||||
|
||||
@@ -1237,8 +1264,13 @@ class AsyncLLM(EngineClient):
|
||||
"waiting for requests to drain before scaling"
|
||||
)
|
||||
await self.wait_for_requests_to_drain(drain_timeout)
|
||||
except BaseException:
|
||||
set_scaling_elastic_ep(False)
|
||||
except BaseException as e:
|
||||
if self.vllm_config.parallel_config.data_parallel_external_lb:
|
||||
await self.engine_core.abort_external_elastic_ep(
|
||||
str(e) or type(e).__name__
|
||||
)
|
||||
else:
|
||||
set_scaling_elastic_ep(False)
|
||||
raise
|
||||
|
||||
async def scale_elastic_ep(
|
||||
@@ -1251,7 +1283,8 @@ class AsyncLLM(EngineClient):
|
||||
async def _scale_elastic_ep(
|
||||
self, new_data_parallel_size: int, drain_timeout: int
|
||||
) -> None:
|
||||
old_data_parallel_size = self.vllm_config.parallel_config.data_parallel_size
|
||||
parallel_config = self.vllm_config.parallel_config
|
||||
old_data_parallel_size = parallel_config.data_parallel_size
|
||||
if old_data_parallel_size == new_data_parallel_size:
|
||||
logger.info(
|
||||
"Data parallel size is already %s, skipping scale",
|
||||
@@ -1282,9 +1315,36 @@ class AsyncLLM(EngineClient):
|
||||
if envs.VLLM_ELASTIC_EP_DRAIN_REQUESTS:
|
||||
await self._drain_requests_for_elastic_ep(drain_timeout)
|
||||
|
||||
await self.engine_core.commit_elastic_ep()
|
||||
self.vllm_config.parallel_config.data_parallel_size = new_data_parallel_size
|
||||
set_scaling_elastic_ep(False)
|
||||
rank_will_retire = (
|
||||
parallel_config.data_parallel_external_lb
|
||||
and new_data_parallel_size < old_data_parallel_size
|
||||
and parallel_config.data_parallel_rank >= new_data_parallel_size
|
||||
)
|
||||
if rank_will_retire:
|
||||
set_elastic_ep_rank_retired(True)
|
||||
|
||||
commit_succeeded = False
|
||||
try:
|
||||
await self.engine_core.commit_elastic_ep()
|
||||
parallel_config.data_parallel_size = new_data_parallel_size
|
||||
commit_succeeded = True
|
||||
if rank_will_retire:
|
||||
logger.info(
|
||||
"[Elastic EP] Data-parallel rank %s is retired and awaiting "
|
||||
"external process shutdown",
|
||||
parallel_config.data_parallel_rank,
|
||||
)
|
||||
finally:
|
||||
if rank_will_retire and not commit_succeeded:
|
||||
set_elastic_ep_rank_retired(False)
|
||||
# A failed commit may leave engines partially reconfigured or paused.
|
||||
if commit_succeeded:
|
||||
set_scaling_elastic_ep(False)
|
||||
|
||||
async def get_external_elastic_ep_status(
|
||||
self,
|
||||
) -> dict[str, str | int | None] | None:
|
||||
return await self.engine_core.get_external_elastic_ep_status()
|
||||
|
||||
async def handle_fault(
|
||||
self, fault_tolerance_request: FaultToleranceRequest
|
||||
@@ -1308,6 +1368,10 @@ class AsyncLLM(EngineClient):
|
||||
def errored(self) -> bool:
|
||||
return self.engine_core.resources.engine_dead or not self.is_running
|
||||
|
||||
@property
|
||||
def should_keep_api_server_alive(self) -> bool:
|
||||
return get_elastic_ep_rank_retired()
|
||||
|
||||
@property
|
||||
def dead_error(self) -> BaseException:
|
||||
return EngineDeadError()
|
||||
|
||||
@@ -126,6 +126,7 @@ class DPCoordinator:
|
||||
self.stats_publish_address = front_publish_address
|
||||
self.coord_in_address = back_publish_address
|
||||
self.coord_out_address = back_output_address
|
||||
self._coord_store: object | None = None
|
||||
self._finalizer = weakref.finalize(self, shutdown, [self.proc])
|
||||
|
||||
def get_stats_publish_address(self) -> str:
|
||||
|
||||
+50
-9
@@ -1346,6 +1346,16 @@ class EngineCoreProc(EngineCore):
|
||||
ready_msg["parallel_config_hash"] = (
|
||||
vllm_config.parallel_config.compute_hash()
|
||||
)
|
||||
if vllm_config.parallel_config.enable_elastic_ep:
|
||||
ready_msg["coord_store_port"] = (
|
||||
vllm_config.parallel_config._coord_store_port
|
||||
)
|
||||
ready_msg["elastic_ep_max_dp_size"] = (
|
||||
vllm_config.parallel_config.elastic_ep_max_dp_size
|
||||
)
|
||||
ready_msg["num_redundant_experts"] = (
|
||||
vllm_config.parallel_config.eplb_config.num_redundant_experts
|
||||
)
|
||||
|
||||
handshake_socket.send(msgspec.msgpack.encode(ready_msg))
|
||||
|
||||
@@ -1376,14 +1386,37 @@ class EngineCoreProc(EngineCore):
|
||||
f"minutes"
|
||||
)
|
||||
init_bytes = handshake_socket.recv()
|
||||
init_message: EngineHandshakeMetadata = msgspec.msgpack.decode(
|
||||
init_bytes, type=EngineHandshakeMetadata
|
||||
)
|
||||
num_redundant_experts: int | None = None
|
||||
if (
|
||||
envs.VLLM_ELASTIC_EP_SCALE_UP_LAUNCH
|
||||
and parallel_config is not None
|
||||
and parallel_config.data_parallel_external_lb
|
||||
):
|
||||
from vllm.distributed.elastic_ep.external_elastic_ep import (
|
||||
ExternalElasticEPScaleUpHandshakeMetadata,
|
||||
)
|
||||
|
||||
scale_up_message = msgspec.msgpack.decode(
|
||||
init_bytes, type=ExternalElasticEPScaleUpHandshakeMetadata
|
||||
)
|
||||
init_message = scale_up_message.engine_metadata
|
||||
num_redundant_experts = scale_up_message.num_redundant_experts
|
||||
parallel_config.elastic_ep_max_dp_size = (
|
||||
scale_up_message.elastic_ep_max_dp_size
|
||||
)
|
||||
else:
|
||||
init_message = msgspec.msgpack.decode(
|
||||
init_bytes, type=EngineHandshakeMetadata
|
||||
)
|
||||
logger.debug("Received init message: %s", init_message)
|
||||
|
||||
if parallel_config is not None:
|
||||
for key, value in init_message.parallel_config.items():
|
||||
setattr(parallel_config, key, value)
|
||||
if num_redundant_experts is not None:
|
||||
parallel_config.eplb_config.num_redundant_experts = (
|
||||
num_redundant_experts
|
||||
)
|
||||
|
||||
return init_message.addresses
|
||||
|
||||
@@ -1826,6 +1859,9 @@ class EngineCoreProc(EngineCore):
|
||||
if self.vllm_config.weight_transfer_config is not None
|
||||
else False
|
||||
),
|
||||
coord_store_port=parallel_config._coord_store_port,
|
||||
coordinator_input_address=self.addresses.coordinator_input,
|
||||
coordinator_output_address=self.addresses.coordinator_output,
|
||||
)
|
||||
|
||||
def process_input_sockets(
|
||||
@@ -2342,6 +2378,13 @@ class DPEngineCoreProc(EngineCoreProc):
|
||||
if state.is_complete():
|
||||
if state.worker_type == "removing":
|
||||
raise SystemExit
|
||||
if (
|
||||
state.worker_type == "new"
|
||||
and self.vllm_config.parallel_config.data_parallel_external_lb
|
||||
):
|
||||
# An independently launched rank has no external utility
|
||||
# caller to join the post-commit resume barrier.
|
||||
self.resume_scheduler()
|
||||
self.process_input_queue_block = True
|
||||
self.eep_scaling_state = None
|
||||
elif not state.commit_requested and state.is_ready_for_switch():
|
||||
@@ -2512,12 +2555,10 @@ class DPEngineCoreProc(EngineCoreProc):
|
||||
self.output_queue.put_nowait((0, outputs))
|
||||
else:
|
||||
encoder = MsgpackEncoder()
|
||||
with (
|
||||
zmq.Context() as ctx,
|
||||
make_zmq_socket(
|
||||
ctx, self.addresses.outputs[0], zmq.PUSH, linger=4000
|
||||
) as socket,
|
||||
):
|
||||
ctx = zmq.Context.instance()
|
||||
with make_zmq_socket(
|
||||
ctx, self.addresses.outputs[0], zmq.PUSH, linger=4000
|
||||
) as socket:
|
||||
socket.send_multipart(encoder.encode(outputs))
|
||||
|
||||
def _eep_scale_up_before_kv_init(self):
|
||||
|
||||
+167
-58
@@ -23,6 +23,9 @@ import zmq.asyncio
|
||||
from vllm import envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.kv_events import KVEventsConfig
|
||||
from vllm.distributed.elastic_ep.external_elastic_ep import (
|
||||
ExternalElasticEPScaleCoordinator,
|
||||
)
|
||||
from vllm.envs import VLLM_ENGINE_READY_TIMEOUT_S
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
@@ -271,6 +274,14 @@ class EngineCoreClient(ABC):
|
||||
async def prepare_elastic_ep(self, new_data_parallel_size: int) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def abort_external_elastic_ep(self, error: str) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def get_external_elastic_ep_status(
|
||||
self,
|
||||
) -> dict[str, str | int | None] | None:
|
||||
return None
|
||||
|
||||
async def get_output_async(self) -> EngineCoreOutputs:
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -626,6 +637,8 @@ class MPClient(EngineCoreClient):
|
||||
enable_input_socket_handover = parallel_config.enable_elastic_ep
|
||||
|
||||
self.stats_update_address: str | None = None
|
||||
self.coordinator_input_address: str | None = None
|
||||
self.coordinator_output_address: str | None = None
|
||||
tensor_queue: Queue | None = None
|
||||
if client_addresses:
|
||||
# Engines are managed externally to this client.
|
||||
@@ -702,6 +715,11 @@ class MPClient(EngineCoreClient):
|
||||
coordinator.get_stats_publish_address()
|
||||
)
|
||||
|
||||
self.input_address = self.input_socket.getsockopt_string(zmq.LAST_ENDPOINT)
|
||||
output_socket = self.resources.output_socket
|
||||
assert output_socket is not None
|
||||
self.output_address = output_socket.getsockopt_string(zmq.LAST_ENDPOINT)
|
||||
|
||||
# Serialization setup with tensor queues for multimodal tensor IPC.
|
||||
tensor_ipc_sender: TensorIpcSender | None = None
|
||||
model_config = getattr(vllm_config, "model_config", None)
|
||||
@@ -885,6 +903,25 @@ class MPClient(EngineCoreClient):
|
||||
response.kv_events_config
|
||||
)
|
||||
|
||||
parallel_config = vllm_config.parallel_config
|
||||
if response.coord_store_port:
|
||||
if parallel_config._coord_store_port:
|
||||
assert response.coord_store_port == parallel_config._coord_store_port
|
||||
else:
|
||||
parallel_config._coord_store_port = response.coord_store_port
|
||||
|
||||
for attribute in (
|
||||
"coordinator_input_address",
|
||||
"coordinator_output_address",
|
||||
):
|
||||
address = getattr(response, attribute)
|
||||
if address is not None:
|
||||
existing_address = getattr(self, attribute)
|
||||
if existing_address is None:
|
||||
setattr(self, attribute, address)
|
||||
else:
|
||||
assert address == existing_address
|
||||
|
||||
def get_kv_event_sources(self) -> dict[int, KVEventsConfig]:
|
||||
return dict(self._kv_event_sources)
|
||||
|
||||
@@ -1431,6 +1468,12 @@ class DPAsyncMPClient(AsyncMPClient):
|
||||
renderer: BaseRenderer | None = None,
|
||||
):
|
||||
self.current_wave = 0
|
||||
parallel_config = vllm_config.parallel_config
|
||||
self.external_eep_coordinator: ExternalElasticEPScaleCoordinator | None = None
|
||||
if parallel_config.enable_elastic_ep and (
|
||||
parallel_config.data_parallel_external_lb
|
||||
):
|
||||
self.external_eep_coordinator = ExternalElasticEPScaleCoordinator(self)
|
||||
|
||||
super().__init__(
|
||||
vllm_config,
|
||||
@@ -1514,23 +1557,19 @@ class DPAsyncMPClient(AsyncMPClient):
|
||||
dp_rank = parallel_config.data_parallel_rank
|
||||
assert dp_rank == 0
|
||||
assert dp_size == new_engine_count
|
||||
assert not (
|
||||
parallel_config.data_parallel_hybrid_lb
|
||||
or parallel_config.data_parallel_external_lb
|
||||
)
|
||||
num_ranks = dp_size
|
||||
self.engine_ranks_managed = list(
|
||||
range(dp_rank, dp_rank + num_ranks)
|
||||
)
|
||||
if len(self.lb_engines) < new_engine_count:
|
||||
self.lb_engines = self.lb_engines + [
|
||||
[0, 0, 0.0]
|
||||
for _ in range(
|
||||
new_engine_count - len(self.lb_engines)
|
||||
)
|
||||
]
|
||||
else:
|
||||
self.lb_engines = self.lb_engines[:new_engine_count]
|
||||
assert not parallel_config.data_parallel_hybrid_lb
|
||||
# External LB clients continue to manage only local ranks.
|
||||
if not parallel_config.data_parallel_external_lb:
|
||||
self.engine_ranks_managed = list(range(dp_size))
|
||||
if len(self.lb_engines) < new_engine_count:
|
||||
self.lb_engines += [
|
||||
[0, 0, 0.0]
|
||||
for _ in range(
|
||||
new_engine_count - len(self.lb_engines)
|
||||
)
|
||||
]
|
||||
else:
|
||||
self.lb_engines = self.lb_engines[:new_engine_count]
|
||||
# Send scale up notification to coordinator
|
||||
scale_msg = msgspec.msgpack.encode(
|
||||
("SCALE_ELASTIC_EP", new_engine_count)
|
||||
@@ -1599,6 +1638,93 @@ class DPAsyncMPClient(AsyncMPClient):
|
||||
def get_core_engine_for_request(self, request: EngineCoreRequest):
|
||||
return self.core_engine
|
||||
|
||||
def _eep_commit_pause_mode(self) -> PauseMode:
|
||||
from vllm.distributed.elastic_ep.elastic_execute import (
|
||||
can_reuse_fused_moe_kernel,
|
||||
)
|
||||
|
||||
# MRV2 re-warms through the request pool, so running requests must
|
||||
# finish first while new ones stay queued in the scheduler.
|
||||
parallel_config = self.vllm_config.parallel_config
|
||||
if self.vllm_config.use_v2_model_runner and not can_reuse_fused_moe_kernel(
|
||||
parallel_config
|
||||
):
|
||||
return "wait"
|
||||
return "keep"
|
||||
|
||||
def _setup_elastic_ep_reconfig_bootstrap(self) -> tuple[str, int]:
|
||||
from vllm.distributed.utils import create_tcp_store
|
||||
from vllm.utils.network_utils import get_open_ports_list
|
||||
|
||||
parallel_config = self.vllm_config.parallel_config
|
||||
parallel_config._data_parallel_master_port_list = get_open_ports_list(5)
|
||||
parallel_config.data_parallel_master_port = (
|
||||
parallel_config._data_parallel_master_port_list.pop()
|
||||
)
|
||||
|
||||
ip = parallel_config.data_parallel_master_ip
|
||||
store = create_tcp_store(
|
||||
ip,
|
||||
0,
|
||||
is_master=True,
|
||||
world_size=-1,
|
||||
wait_for_workers=False,
|
||||
)
|
||||
parallel_config._coord_store_port = store.port
|
||||
self._coord_store = store
|
||||
return ip, store.port
|
||||
|
||||
@staticmethod
|
||||
async def eep_process_engine_core_notification(
|
||||
self: "DPAsyncMPClient", notification_data: tuple[str, int]
|
||||
) -> None:
|
||||
if self.external_eep_coordinator is not None:
|
||||
await self.external_eep_coordinator.process_engine_core_notification(
|
||||
notification_data
|
||||
)
|
||||
|
||||
def _get_external_eep_coordinator(self) -> ExternalElasticEPScaleCoordinator:
|
||||
parallel_config = self.vllm_config.parallel_config
|
||||
if not parallel_config.enable_elastic_ep:
|
||||
raise NotImplementedError(
|
||||
"Elastic EP scaling requires enable_elastic_ep=True."
|
||||
)
|
||||
if not parallel_config.data_parallel_external_lb:
|
||||
raise NotImplementedError(
|
||||
"DPAsyncMPClient only supports Elastic EP scaling in external "
|
||||
"load-balancer mode."
|
||||
)
|
||||
coordinator = self.external_eep_coordinator
|
||||
assert coordinator is not None
|
||||
return coordinator
|
||||
|
||||
async def prepare_elastic_ep(self, new_data_parallel_size: int) -> None:
|
||||
cur_data_parallel_size = self.vllm_config.parallel_config.data_parallel_size
|
||||
|
||||
assert new_data_parallel_size != cur_data_parallel_size, (
|
||||
f"new_data_parallel_size {new_data_parallel_size} must be "
|
||||
f"different from cur_data_parallel_size {cur_data_parallel_size}"
|
||||
)
|
||||
await self._get_external_eep_coordinator().prepare(
|
||||
cur_data_parallel_size, new_data_parallel_size
|
||||
)
|
||||
|
||||
async def commit_elastic_ep(self) -> None:
|
||||
await self._get_external_eep_coordinator().commit()
|
||||
|
||||
async def abort_external_elastic_ep(self, error: str) -> None:
|
||||
self._get_external_eep_coordinator().abort(error)
|
||||
|
||||
async def get_external_elastic_ep_status(
|
||||
self,
|
||||
) -> dict[str, str | int | None] | None:
|
||||
coordinator = self.external_eep_coordinator
|
||||
return (
|
||||
None
|
||||
if coordinator is None
|
||||
else await asyncio.to_thread(coordinator.get_status)
|
||||
)
|
||||
|
||||
|
||||
class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
"""Asyncio-compatible client for multi-proc, multi-engine (data parallel)
|
||||
@@ -1618,6 +1744,8 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
|
||||
# To route aborts to the correct engine.
|
||||
self.reqs_in_flight: dict[str, EngineIdentity] = {}
|
||||
# Open streaming-input sessions, whose aborts go to all engines.
|
||||
self.streaming_reqs: set[str] = set()
|
||||
|
||||
# Exact per-engine count of this client's unfinished requests.
|
||||
self.engine_inflight: Counter[EngineIdentity] = Counter()
|
||||
@@ -1640,6 +1768,12 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
) // client_count
|
||||
|
||||
def get_core_engine_for_request(self, request: EngineCoreRequest) -> EngineIdentity:
|
||||
if request.resumable:
|
||||
self.streaming_reqs.add(request.request_id)
|
||||
else:
|
||||
# Also removes a session on its final (end-of-stream) request.
|
||||
self.streaming_reqs.discard(request.request_id)
|
||||
|
||||
if (engine := self.reqs_in_flight.get(request.request_id)) is not None:
|
||||
# This is for streaming-input session affinity.
|
||||
return engine
|
||||
@@ -1711,8 +1845,9 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
|
||||
@staticmethod
|
||||
async def eep_process_engine_core_notification(
|
||||
self: "DPLBAsyncMPClient", notification_data: tuple[str, int]
|
||||
):
|
||||
self: "DPAsyncMPClient", notification_data: tuple[str, int]
|
||||
) -> None:
|
||||
assert isinstance(self, DPLBAsyncMPClient)
|
||||
cache = self.eep_scaling_cache
|
||||
notification_type_str, dp_rank = notification_data
|
||||
try:
|
||||
@@ -1759,15 +1894,24 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
if not request_ids or self.resources.engine_dead:
|
||||
return
|
||||
|
||||
if len(request_ids) == 1:
|
||||
if (
|
||||
len(request_ids) == 1
|
||||
and (req_id := request_ids[0]) not in self.streaming_reqs
|
||||
and (engine := self.reqs_in_flight.get(req_id))
|
||||
):
|
||||
# Fast-path common case.
|
||||
if engine := self.reqs_in_flight.get(request_ids[0]):
|
||||
await self._abort_requests(request_ids, engine)
|
||||
await self._abort_requests(request_ids, engine)
|
||||
return
|
||||
|
||||
by_engine = defaultdict[EngineIdentity, list[str]](list)
|
||||
for req_id in request_ids:
|
||||
if engine := self.reqs_in_flight.get(req_id):
|
||||
if req_id in self.streaming_reqs:
|
||||
# A chunk in flight when the session finished may have
|
||||
# re-created it on any engine.
|
||||
self.streaming_reqs.discard(req_id)
|
||||
for engine in self.core_engines:
|
||||
by_engine[engine].append(req_id)
|
||||
elif engine := self.reqs_in_flight.get(req_id):
|
||||
by_engine[engine].append(req_id)
|
||||
for engine, req_ids in by_engine.items():
|
||||
await self._abort_requests(req_ids, engine)
|
||||
@@ -1864,27 +2008,6 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
new_engine_identities.discard(identity)
|
||||
self._apply_ready_response(payload)
|
||||
|
||||
def _setup_elastic_ep_reconfig_bootstrap(self) -> None:
|
||||
from vllm.distributed.utils import create_tcp_store
|
||||
from vllm.utils.network_utils import get_open_ports_list
|
||||
|
||||
parallel_config = self.vllm_config.parallel_config
|
||||
parallel_config._data_parallel_master_port_list = get_open_ports_list(5)
|
||||
parallel_config.data_parallel_master_port = (
|
||||
parallel_config._data_parallel_master_port_list.pop()
|
||||
)
|
||||
|
||||
ip = parallel_config.data_parallel_master_ip
|
||||
store = create_tcp_store(
|
||||
ip,
|
||||
0,
|
||||
is_master=True,
|
||||
world_size=-1,
|
||||
wait_for_workers=False,
|
||||
)
|
||||
parallel_config._coord_store_port = store.port
|
||||
self._coord_store = store
|
||||
|
||||
def _make_reconfig_request(
|
||||
self,
|
||||
new_data_parallel_size: int,
|
||||
@@ -1939,20 +2062,6 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
await asyncio.to_thread(self._coord_store.wait, ready_keys)
|
||||
logger.info("[Elastic EP] Successfully started new engines")
|
||||
|
||||
def _eep_commit_pause_mode(self) -> PauseMode:
|
||||
from vllm.distributed.elastic_ep.elastic_execute import (
|
||||
can_reuse_fused_moe_kernel,
|
||||
)
|
||||
|
||||
# MRV2 re-warms through the request pool, so running requests must
|
||||
# finish first while new ones stay queued in the scheduler.
|
||||
parallel_config = self.vllm_config.parallel_config
|
||||
if self.vllm_config.use_v2_model_runner and not can_reuse_fused_moe_kernel(
|
||||
parallel_config
|
||||
):
|
||||
return "wait"
|
||||
return "keep"
|
||||
|
||||
async def _commit_scale_up_elastic_ep(self, new_data_parallel_size: int) -> None:
|
||||
new_core_engines = [
|
||||
rank.to_bytes(2, "little")
|
||||
|
||||
@@ -113,6 +113,53 @@ class RequestOutputCollector:
|
||||
self._input_stream_task = None
|
||||
|
||||
|
||||
def _is_chunk_boundary(
|
||||
output: RequestOutput | PoolingRequestOutput | Exception | None,
|
||||
) -> bool:
|
||||
"""Whether the output marks the end of a streaming-input chunk: a
|
||||
finish_reason on a request that has not finished."""
|
||||
return (
|
||||
isinstance(output, RequestOutput)
|
||||
and not output.finished
|
||||
and any(completion.finished() for completion in output.outputs)
|
||||
)
|
||||
|
||||
|
||||
class StreamingInputOutputCollector(RequestOutputCollector):
|
||||
"""Collector for streaming-input requests. Doesn't merge across input
|
||||
chunk boundaries (a finish_reason on a non-finished output)."""
|
||||
|
||||
def __init__(self, output_kind: RequestOutputKind, request_id: str):
|
||||
super().__init__(output_kind, request_id)
|
||||
self._backlog: deque[RequestOutput] = deque()
|
||||
|
||||
def put(self, output: RequestOutput | PoolingRequestOutput | Exception) -> None:
|
||||
if isinstance(output, Exception):
|
||||
self._backlog.clear()
|
||||
elif self._backlog or _is_chunk_boundary(self.output):
|
||||
assert isinstance(output, RequestOutput)
|
||||
if self._backlog and not _is_chunk_boundary(self._backlog[-1]):
|
||||
self._backlog[-1].add(output, aggregate=self.aggregate)
|
||||
else:
|
||||
self._backlog.append(output)
|
||||
return
|
||||
super().put(output)
|
||||
|
||||
async def get(self) -> RequestOutput | PoolingRequestOutput:
|
||||
output = await super().get()
|
||||
if self._backlog:
|
||||
self.output = self._backlog.popleft()
|
||||
self.ready.set()
|
||||
return output
|
||||
|
||||
def get_nowait(self) -> RequestOutput | PoolingRequestOutput | None:
|
||||
output = super().get_nowait()
|
||||
if self._backlog:
|
||||
self.output = self._backlog.popleft()
|
||||
self.ready.set()
|
||||
return output
|
||||
|
||||
|
||||
@dataclass
|
||||
class OutputProcessorOutput:
|
||||
request_outputs: list[RequestOutput | PoolingRequestOutput]
|
||||
@@ -784,6 +831,10 @@ class OutputProcessor:
|
||||
# if required.
|
||||
req_state.logprobs_processor.update_from_output(engine_core_output)
|
||||
|
||||
streaming_chunk_boundary = req_state.streaming_input and (
|
||||
finish_reason not in (FinishReason.ABORT, FinishReason.ERROR)
|
||||
)
|
||||
|
||||
# 4) Create and handle RequestOutput objects.
|
||||
if request_output := req_state.make_request_output(
|
||||
new_token_ids,
|
||||
@@ -794,7 +845,7 @@ class OutputProcessor:
|
||||
ec_transfer_params,
|
||||
error=request_error,
|
||||
):
|
||||
if req_state.streaming_input:
|
||||
if streaming_chunk_boundary:
|
||||
request_output.finished = False
|
||||
|
||||
if req_state.queue is not None:
|
||||
@@ -806,7 +857,7 @@ class OutputProcessor:
|
||||
|
||||
# Free completed requests.
|
||||
if finish_reason is not None:
|
||||
if req_state.streaming_input:
|
||||
if streaming_chunk_boundary:
|
||||
if req_state.input_chunk_queue:
|
||||
update = req_state.input_chunk_queue.popleft()
|
||||
req_state.apply_streaming_update(update)
|
||||
@@ -814,9 +865,10 @@ class OutputProcessor:
|
||||
req_state.input_chunk_queue = None
|
||||
else:
|
||||
self._finish_request(req_state)
|
||||
if not engine_core_output.finished:
|
||||
# If req not finished in EngineCore, but Detokenizer
|
||||
# detected stop string, abort needed in EngineCore.
|
||||
if req_state.streaming_input or not engine_core_output.finished:
|
||||
# EngineCore may still have the request: a stop string
|
||||
# was detected here, or an in-flight streaming-input
|
||||
# chunk may re-create the session.
|
||||
reqs_to_abort.append(req_id)
|
||||
|
||||
# Track per-request stats
|
||||
|
||||
+75
-14
@@ -290,6 +290,7 @@ class CoreEngineProcManager:
|
||||
self._finalizer = weakref.finalize(self, shutdown, self.processes)
|
||||
self.manager_stopped = threading.Event()
|
||||
self.failed_proc_name: str | None = None
|
||||
self._coord_store: object | None = None
|
||||
|
||||
# All ranks share this config object: capture the user-provided
|
||||
# --device-ids list before the per-rank shard overwrites it. Mutating
|
||||
@@ -1139,7 +1140,9 @@ def get_engine_zmq_bind_addresses(
|
||||
offline_mode or local_engines_only or (local_engine_count == dp_size)
|
||||
)
|
||||
# NOTE(yongji): handling scaling from intra-node to inter-node
|
||||
if parallel_config.enable_elastic_ep:
|
||||
if parallel_config.enable_elastic_ep and (
|
||||
not local_engines_only or parallel_config.data_parallel_rank == 0
|
||||
):
|
||||
client_local_only = False
|
||||
|
||||
def _addr() -> str:
|
||||
@@ -1228,6 +1231,32 @@ def launch_core_engines(
|
||||
vllm_config.needs_dp_coordinator and not offline_mode and dp_rank == 0
|
||||
)
|
||||
|
||||
coord_store = None
|
||||
|
||||
create_coord_store = (
|
||||
parallel_config.enable_elastic_ep
|
||||
and parallel_config.data_parallel_backend != "ray"
|
||||
and dp_rank == 0
|
||||
and not parallel_config._coord_store_port
|
||||
)
|
||||
|
||||
if create_coord_store:
|
||||
from vllm.distributed.utils import create_tcp_store
|
||||
|
||||
coord_store = create_tcp_store(
|
||||
host,
|
||||
0,
|
||||
is_master=True,
|
||||
world_size=-1,
|
||||
wait_for_workers=False,
|
||||
)
|
||||
parallel_config._coord_store_port = coord_store.port
|
||||
logger.info(
|
||||
"Created Elastic EP coordination TCPStore on %s:%d",
|
||||
host,
|
||||
coord_store.port,
|
||||
)
|
||||
|
||||
if run_coordinator:
|
||||
coordinator = DPCoordinator(
|
||||
parallel_config,
|
||||
@@ -1243,12 +1272,13 @@ def launch_core_engines(
|
||||
)
|
||||
|
||||
logger.info("Started DP Coordinator process (PID: %d)", coordinator.proc.pid)
|
||||
if coord_store is not None:
|
||||
coordinator._coord_store = coord_store
|
||||
else:
|
||||
coordinator = None
|
||||
|
||||
# Hold a coordination TCPStore (alive for this frame) so engines pick DP
|
||||
# master ports at bind time; pre-allocated ports can be taken before use.
|
||||
coord_store = None
|
||||
if (
|
||||
dp_size > 1
|
||||
and not offline_mode
|
||||
@@ -1344,6 +1374,13 @@ def launch_core_engines(
|
||||
else:
|
||||
local_engine_manager = None
|
||||
|
||||
if (
|
||||
coord_store is not None
|
||||
and coordinator is None
|
||||
and (local_engine_manager is not None)
|
||||
):
|
||||
local_engine_manager._coord_store = coord_store
|
||||
|
||||
launch = CoreEngineLaunch(
|
||||
local_engine_manager, coordinator, addresses, tensor_queue
|
||||
)
|
||||
@@ -1474,21 +1511,33 @@ def wait_for_engine_startup(
|
||||
|
||||
if status == "HELLO" and engine.state == CoreEngineState.NEW:
|
||||
# Send init message with DP config info.
|
||||
parallel_config_payload: dict[str, int | str | list[int]] = {}
|
||||
if coordinated_dp and not (
|
||||
parallel_config.local_engines_only
|
||||
and parallel_config.data_parallel_rank > 0
|
||||
):
|
||||
parallel_config_payload = {
|
||||
k: getattr(parallel_config, k)
|
||||
for k in (
|
||||
"data_parallel_master_ip",
|
||||
"data_parallel_master_port",
|
||||
"_data_parallel_master_port_list",
|
||||
"_coord_store_port",
|
||||
"data_parallel_size",
|
||||
)
|
||||
}
|
||||
if (
|
||||
parallel_config.enable_elastic_ep
|
||||
and not parallel_config._coord_store_port
|
||||
):
|
||||
raise RuntimeError(
|
||||
"Elastic EP startup requires a coordination TCPStore "
|
||||
"port before engine handshake."
|
||||
)
|
||||
init_message = msgspec.msgpack.encode(
|
||||
EngineHandshakeMetadata(
|
||||
addresses=launch.addresses,
|
||||
parallel_config={
|
||||
k: getattr(parallel_config, k)
|
||||
for k in (
|
||||
"data_parallel_master_ip",
|
||||
"data_parallel_master_port",
|
||||
"_data_parallel_master_port_list",
|
||||
"_coord_store_port",
|
||||
"data_parallel_size",
|
||||
)
|
||||
}
|
||||
if coordinated_dp
|
||||
else {},
|
||||
parallel_config=parallel_config_payload,
|
||||
)
|
||||
)
|
||||
handshake_socket.send_multipart((eng_identity, init_message), copy=False)
|
||||
@@ -1496,6 +1545,18 @@ def wait_for_engine_startup(
|
||||
start_pending[0 if local else 1] += 1
|
||||
engine.state = CoreEngineState.CONNECTED
|
||||
elif status == "READY" and engine.state == CoreEngineState.CONNECTED:
|
||||
coord_store_port = msg.get("coord_store_port")
|
||||
if coord_store_port is not None and local:
|
||||
parallel_config._coord_store_port = coord_store_port
|
||||
num_redundant_experts = msg.get("num_redundant_experts")
|
||||
if num_redundant_experts is not None and local:
|
||||
parallel_config.eplb_config.num_redundant_experts = (
|
||||
num_redundant_experts
|
||||
)
|
||||
elastic_ep_max_dp_size = msg.get("elastic_ep_max_dp_size")
|
||||
if elastic_ep_max_dp_size is not None and local:
|
||||
parallel_config.elastic_ep_max_dp_size = elastic_ep_max_dp_size
|
||||
|
||||
# Validate config hash consistency across DP workers for MoE models.
|
||||
if coordinated_dp:
|
||||
worker_config_hash = msg.get("parallel_config_hash")
|
||||
|
||||
@@ -292,7 +292,7 @@ class RayExecutorV2(MultiprocExecutor):
|
||||
# Step 1: Initialize Ray cluster and retrieve placement group
|
||||
if ray is None:
|
||||
raise ImportError("Using Ray backend requires installation of ray.")
|
||||
initialize_ray_cluster(self.parallel_config, require_gpu_on_driver=False)
|
||||
initialize_ray_cluster(self.parallel_config)
|
||||
placement_group = self.parallel_config.placement_group
|
||||
|
||||
tp_size, pp_size, pcp_size = self._get_parallel_sizes()
|
||||
|
||||
@@ -8,7 +8,6 @@ from collections import defaultdict
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_ip
|
||||
|
||||
logger = init_logger(__name__)
|
||||
PG_WAIT_TIMEOUT = 1800
|
||||
@@ -30,14 +29,6 @@ try:
|
||||
from ray.util import placement_group_table
|
||||
from ray.util.placement_group import PlacementGroup
|
||||
|
||||
try:
|
||||
from ray._private.state import available_resources_per_node
|
||||
except ImportError:
|
||||
# Ray 2.9.x doesn't expose `available_resources_per_node`
|
||||
from ray._private.state import state as _state
|
||||
|
||||
available_resources_per_node = _state._available_resources_per_node
|
||||
|
||||
ray_import_err = None
|
||||
|
||||
except ImportError as e:
|
||||
@@ -65,15 +56,8 @@ def _verify_bundles(
|
||||
placement_group: "PlacementGroup",
|
||||
parallel_config: ParallelConfig,
|
||||
device_str: str,
|
||||
require_gpu_on_driver: bool = True,
|
||||
):
|
||||
"""Verify a given placement group has bundles located in the right place.
|
||||
|
||||
There are 2 rules.
|
||||
- Warn if all tensor parallel workers cannot fit in a single node.
|
||||
- Fail if driver node is not included in a placement group
|
||||
(only when require_gpu_on_driver is True).
|
||||
"""
|
||||
"""Warn if all tensor parallel workers cannot fit in a single node."""
|
||||
assert ray.is_initialized(), (
|
||||
"Ray is not initialized although distributed-executor-backend is ray."
|
||||
)
|
||||
@@ -87,17 +71,6 @@ def _verify_bundles(
|
||||
|
||||
for bundle_idx, node_id in bundle_to_node_ids.items():
|
||||
node_id_to_bundle[node_id].append(bundles[bundle_idx])
|
||||
driver_node_id = ray.get_runtime_context().get_node_id()
|
||||
|
||||
if require_gpu_on_driver and driver_node_id not in node_id_to_bundle:
|
||||
raise RuntimeError(
|
||||
f"driver node id {driver_node_id} is not included in a placement "
|
||||
f"group {placement_group.id}. Node id -> bundles "
|
||||
f"{node_id_to_bundle}. "
|
||||
"You don't have enough GPUs available in a current node. Check "
|
||||
"`ray status` and `ray list nodes` to see if you have available "
|
||||
"GPUs in a node `{driver_node_id}` before starting an vLLM engine."
|
||||
)
|
||||
|
||||
for node_id, bundles in node_id_to_bundle.items():
|
||||
if len(bundles) < parallel_config.tensor_parallel_size:
|
||||
@@ -293,7 +266,6 @@ def _wait_until_pg_ready(current_placement_group: "PlacementGroup"):
|
||||
def initialize_ray_cluster(
|
||||
parallel_config: ParallelConfig,
|
||||
ray_address: str | None = None,
|
||||
require_gpu_on_driver: bool = True,
|
||||
):
|
||||
"""Initialize the distributed cluster with Ray.
|
||||
|
||||
@@ -305,10 +277,6 @@ def initialize_ray_cluster(
|
||||
parallel_config: The configurations for parallel execution.
|
||||
ray_address: The address of the Ray cluster. If None, uses
|
||||
the default Ray cluster address.
|
||||
require_gpu_on_driver: If True (default), require at least one GPU
|
||||
on the current (driver) node and pin the first PG bundle to it.
|
||||
Set to False for executors like RayExecutorV2 where all GPU work
|
||||
is delegated to remote Ray actors.
|
||||
|
||||
"""
|
||||
assert_ray_available()
|
||||
@@ -404,25 +372,6 @@ def initialize_ray_cluster(
|
||||
{device_str: 1.0} for _ in range(parallel_config.world_size)
|
||||
]
|
||||
|
||||
# vLLM engine is also a worker to execute model with an accelerator,
|
||||
# so it requires to have the device in a current node. Check if
|
||||
# the current node has at least one device.
|
||||
current_ip = get_ip()
|
||||
current_node_id = ray.get_runtime_context().get_node_id()
|
||||
current_node_resource = available_resources_per_node()[current_node_id]
|
||||
if require_gpu_on_driver:
|
||||
if current_node_resource.get(device_str, 0) < 1:
|
||||
raise ValueError(
|
||||
f"Current node has no {device_str} available. "
|
||||
f"{current_node_resource=}. vLLM engine cannot start "
|
||||
f"without {device_str}. Make sure you have at least 1 "
|
||||
f"{device_str} available in a node "
|
||||
f"{current_node_id=} {current_ip=}."
|
||||
)
|
||||
# This way, at least bundle is required to be created in a
|
||||
# current node.
|
||||
placement_group_specs[0][f"node:{current_ip}"] = 0.001
|
||||
|
||||
# By default, Ray packs resources as much as possible.
|
||||
current_placement_group = ray.util.placement_group(
|
||||
placement_group_specs, strategy="PACK"
|
||||
@@ -430,8 +379,6 @@ def initialize_ray_cluster(
|
||||
_wait_until_pg_ready(current_placement_group)
|
||||
|
||||
assert current_placement_group is not None
|
||||
_verify_bundles(
|
||||
current_placement_group, parallel_config, device_str, require_gpu_on_driver
|
||||
)
|
||||
_verify_bundles(current_placement_group, parallel_config, device_str)
|
||||
# Set the placement group in the parallel config
|
||||
parallel_config.placement_group = current_placement_group
|
||||
|
||||
+11
-29
@@ -17,6 +17,8 @@ from vllm.v1.hisparse.layout import (
|
||||
from vllm.v1.hisparse.runtime import (
|
||||
HiSparseCacheHandle,
|
||||
HiSparseHostPool,
|
||||
drop_hisparse_host_pool,
|
||||
get_hisparse_host_pool,
|
||||
initialize_hisparse_runtime_buffers,
|
||||
release_pinned_state,
|
||||
)
|
||||
@@ -126,7 +128,7 @@ def init_hisparse_kv_cache(
|
||||
block_tables: "BlockTables",
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""Allocate and bind HiSparse caches within the caller's allocation context."""
|
||||
host_pool = HiSparseHostPool(vllm_config, kv_cache_config)
|
||||
host_pool = get_hisparse_host_pool(vllm_config, kv_cache_config)
|
||||
try:
|
||||
kv_caches = allocate_hisparse_kv_caches(
|
||||
kv_cache_config,
|
||||
@@ -150,6 +152,9 @@ def init_hisparse_kv_cache(
|
||||
except Exception:
|
||||
if host_pool.shared_region is not None:
|
||||
host_pool.shared_region.cleanup()
|
||||
elif host_pool.registered is not None:
|
||||
release_pinned_state([], [host_pool.registered])
|
||||
drop_hisparse_host_pool(vllm_config)
|
||||
raise
|
||||
|
||||
|
||||
@@ -163,37 +168,14 @@ def _get_hisparse_cache(
|
||||
|
||||
|
||||
def release_hisparse_profiling_cache(forward_context: dict[str, Any]) -> None:
|
||||
cache_handles = [
|
||||
cache
|
||||
for layer in forward_context.values()
|
||||
if (cache := getattr(layer, "hisparse_cache", None)) is not None
|
||||
]
|
||||
"""Detach the profiling caches but keep the host pool for the real KV cache."""
|
||||
runtimes = {
|
||||
id(cache.runtime): cache.runtime
|
||||
for cache in cache_handles
|
||||
if hasattr(cache.runtime, "_host_cache")
|
||||
for layer in forward_context.values()
|
||||
if (cache := getattr(layer, "hisparse_cache", None)) is not None
|
||||
and hasattr(cache.runtime, "_host_cache")
|
||||
}
|
||||
if not runtimes:
|
||||
return
|
||||
|
||||
shared_regions = {
|
||||
id(runtime.shared_host_region): runtime.shared_host_region
|
||||
for runtime in runtimes.values()
|
||||
if runtime.shared_host_region is not None
|
||||
}
|
||||
assert len(shared_regions) <= 1
|
||||
shared_region = next(iter(shared_regions.values()), None)
|
||||
registered_pools = (
|
||||
[]
|
||||
if shared_region is not None
|
||||
else list(
|
||||
{
|
||||
runtime.registered_host_pool.data_ptr(): runtime.registered_host_pool
|
||||
for runtime in runtimes.values()
|
||||
}.values()
|
||||
)
|
||||
)
|
||||
release_pinned_state(list(runtimes.values()), registered_pools, shared_region)
|
||||
release_pinned_state(list(runtimes.values()), [])
|
||||
|
||||
|
||||
def bind_hisparse_kv_caches(
|
||||
|
||||
@@ -223,7 +223,9 @@ class HiSparseHostPool:
|
||||
self.shared_region: SharedOffloadRegion | None = None
|
||||
|
||||
def allocate(self, size: int) -> torch.Tensor:
|
||||
assert self.backing is None, "HiSparse host pool is already allocated."
|
||||
if self.backing is not None:
|
||||
assert self.backing.numel() == size, "HiSparse host pool size changed."
|
||||
return self.backing
|
||||
config = self.kv_cache_config
|
||||
if config.hisparse_shared_host_pool:
|
||||
tensor_sizes = [
|
||||
@@ -253,6 +255,23 @@ class HiSparseHostPool:
|
||||
return self.backing
|
||||
|
||||
|
||||
_host_pools: dict[str, HiSparseHostPool] = {}
|
||||
|
||||
|
||||
def get_hisparse_host_pool(
|
||||
vllm_config: VllmConfig, kv_cache_config: KVCacheConfig
|
||||
) -> HiSparseHostPool:
|
||||
"""Return the engine's host pool, which outlives the profiling KV cache."""
|
||||
instance_id = vllm_config.instance_id
|
||||
if instance_id not in _host_pools:
|
||||
_host_pools[instance_id] = HiSparseHostPool(vllm_config, kv_cache_config)
|
||||
return _host_pools[instance_id]
|
||||
|
||||
|
||||
def drop_hisparse_host_pool(vllm_config: VllmConfig) -> None:
|
||||
_host_pools.pop(vllm_config.instance_id, None)
|
||||
|
||||
|
||||
def use_shared_hisparse_host_pool(vllm_config: VllmConfig) -> bool:
|
||||
"""Whether replicated MLA host KV can share one local mmap."""
|
||||
parallel = vllm_config.parallel_config
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import contextlib
|
||||
import ctypes
|
||||
import errno
|
||||
import functools
|
||||
import mmap
|
||||
import os
|
||||
import time
|
||||
from collections.abc import Callable, Iterator
|
||||
from ctypes import c_int, c_long, c_size_t, c_void_p
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -21,6 +24,10 @@ logger = init_logger(__name__)
|
||||
|
||||
# MADV_POPULATE_WRITE was added in Linux 5.14 (value 23).
|
||||
_MADV_POPULATE_WRITE = getattr(mmap, "MADV_POPULATE_WRITE", 23)
|
||||
_MAP_FIXED = 0x10
|
||||
# Page-cache inserts are serialized per shm file, capping pre-faulting of one
|
||||
# file near 10 GiB/s; smaller segments are not worth a file.
|
||||
_MIN_SEGMENT_BYTES = 1 << 30
|
||||
|
||||
|
||||
def _wait_for_file_size(fd: int, expected_size: int, timeout: float = 30.0) -> None:
|
||||
@@ -40,6 +47,14 @@ def _wait_for_file_size(fd: int, expected_size: int, timeout: float = 30.0) -> N
|
||||
time.sleep(0.005)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def _libc_mmap() -> Callable[..., int | None]:
|
||||
mmap_fn = ctypes.CDLL(None, use_errno=True).mmap
|
||||
mmap_fn.argtypes = [c_void_p, c_size_t, c_int, c_int, c_int, c_long]
|
||||
mmap_fn.restype = c_void_p
|
||||
return mmap_fn
|
||||
|
||||
|
||||
def _madvise_populate_write(mmap_obj: mmap.mmap, offset: int, length: int) -> None:
|
||||
mmap_obj.madvise(_MADV_POPULATE_WRITE, offset, length)
|
||||
|
||||
@@ -84,6 +99,7 @@ class SharedOffloadRegion:
|
||||
``populate_shard=(index, count)`` pre-faults only this worker's contiguous
|
||||
share, for layouts where every worker maps the same bytes. The barrier
|
||||
also waits for all shards to finish pre-faulting before returning.
|
||||
``num_segments`` splits backing files beneath the contiguous mapping.
|
||||
"""
|
||||
|
||||
BLOCK_SIZE_ALIGNMENT: int = mmap.PAGESIZE
|
||||
@@ -99,6 +115,7 @@ class SharedOffloadRegion:
|
||||
*,
|
||||
creator_memory_check: Callable[[int], None] | None = None,
|
||||
populate_shard: tuple[int, int] | None = None,
|
||||
num_segments: int = 1,
|
||||
) -> None:
|
||||
self.page_size = mmap.PAGESIZE
|
||||
assert kv_bytes_per_chunk % self.page_size == 0
|
||||
@@ -109,6 +126,17 @@ class SharedOffloadRegion:
|
||||
self.cpu_page_size = cpu_page_size
|
||||
|
||||
self.mmap_path = f"/dev/shm/vllm_offload_{engine_id}.mmap"
|
||||
num_segments = min(num_segments, self.total_size_bytes // _MIN_SEGMENT_BYTES)
|
||||
rows = max(1, cdiv(num_chunks, max(1, num_segments)))
|
||||
# (path, offset, length) per file.
|
||||
self._segments = [
|
||||
(
|
||||
f"{self.mmap_path}.{i}" if i else self.mmap_path,
|
||||
row * self._row_stride,
|
||||
min(rows, num_chunks - row) * self._row_stride,
|
||||
)
|
||||
for i, row in enumerate(range(0, max(num_chunks, 1), rows))
|
||||
]
|
||||
self._creator = False # set True only if this worker creates the file
|
||||
self.rank = rank
|
||||
if rank is not None:
|
||||
@@ -126,7 +154,7 @@ class SharedOffloadRegion:
|
||||
# Joiner path — another worker won O_EXCL. Reopen and wait
|
||||
# for the file to reach expected size.
|
||||
self.fd = os.open(self.mmap_path, os.O_RDWR)
|
||||
_wait_for_file_size(self.fd, self.total_size_bytes)
|
||||
_wait_for_file_size(self.fd, self._segments[0][2])
|
||||
logger.info("Opened existing mmap file %s", self.mmap_path)
|
||||
else:
|
||||
# Creator path. We won O_EXCL, so we own the file: any
|
||||
@@ -140,24 +168,32 @@ class SharedOffloadRegion:
|
||||
self.total_size_bytes,
|
||||
allocation_name="CPU KV offload shared region in /dev/shm",
|
||||
)
|
||||
os.ftruncate(self.fd, self.total_size_bytes)
|
||||
for path, _, length in self._segments[1:]:
|
||||
fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600)
|
||||
os.ftruncate(fd, length)
|
||||
os.close(fd)
|
||||
# Joiners wait for the first file, so size it last.
|
||||
os.ftruncate(self.fd, self._segments[0][2])
|
||||
logger.info(
|
||||
"Created mmap file %s (%.2f GB)",
|
||||
self.mmap_path,
|
||||
self.total_size_bytes / 1e9,
|
||||
)
|
||||
|
||||
single = len(self._segments) == 1
|
||||
self.mmap_obj: mmap.mmap | None = mmap.mmap(
|
||||
self.fd,
|
||||
self.fd if single else -1,
|
||||
self.total_size_bytes,
|
||||
flags=mmap.MAP_SHARED,
|
||||
flags=mmap.MAP_SHARED if single else mmap.MAP_PRIVATE,
|
||||
prot=mmap.PROT_READ | mmap.PROT_WRITE,
|
||||
)
|
||||
if not single:
|
||||
self._map_segments()
|
||||
if barrier is not None:
|
||||
barriers_remaining -= 1
|
||||
barrier()
|
||||
if self._creator:
|
||||
os.unlink(self.mmap_path)
|
||||
self._unlink()
|
||||
self._creator = False
|
||||
logger.info("Unlinked mmap file %s", self.mmap_path)
|
||||
if populate_shard is not None:
|
||||
@@ -173,8 +209,7 @@ class SharedOffloadRegion:
|
||||
barrier()
|
||||
except Exception:
|
||||
if self._creator:
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
os.unlink(self.mmap_path)
|
||||
self._unlink()
|
||||
self._creator = False
|
||||
if hasattr(self, "mmap_obj") and self.mmap_obj is not None:
|
||||
# The prefault traceback may retain an exported buffer.
|
||||
@@ -225,11 +260,42 @@ class SharedOffloadRegion:
|
||||
"MADV_POPULATE_WRITE entire region: %.3f s", time.perf_counter() - _t0
|
||||
)
|
||||
|
||||
def _map_segments(self) -> None:
|
||||
"""Replace the anonymous mapping with the segment files, back to back."""
|
||||
assert self.mmap_obj is not None and self.fd is not None
|
||||
base = ctypes.addressof(ctypes.c_char.from_buffer(self.mmap_obj))
|
||||
for path, offset, length in self._segments:
|
||||
fd = self.fd if offset == 0 else os.open(path, os.O_RDWR)
|
||||
address = _libc_mmap()(
|
||||
base + offset,
|
||||
length,
|
||||
mmap.PROT_READ | mmap.PROT_WRITE,
|
||||
mmap.MAP_SHARED | _MAP_FIXED,
|
||||
fd,
|
||||
0,
|
||||
)
|
||||
if fd != self.fd:
|
||||
os.close(fd)
|
||||
if address != base + offset:
|
||||
raise OSError(ctypes.get_errno(), f"Failed to map {path}")
|
||||
|
||||
def _unlink(self) -> None:
|
||||
for path, _, _ in self._segments:
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
os.unlink(path)
|
||||
|
||||
def iter_worker_slot_ranges(self) -> Iterator[tuple[int, int]]:
|
||||
"""Yield this worker's slots as page-aligned (offset, length) ranges."""
|
||||
assert self.rank is not None
|
||||
worker_offset = self.rank * self.cpu_page_size
|
||||
for row in range(self.num_chunks):
|
||||
first = 0
|
||||
if len(self._segments) > 1:
|
||||
# Start each worker on a different file.
|
||||
first = (
|
||||
self.rank * self.num_chunks // (self._row_stride // self.cpu_page_size)
|
||||
)
|
||||
for i in range(self.num_chunks):
|
||||
row = (first + i) % self.num_chunks
|
||||
start = row * self._row_stride + worker_offset
|
||||
aligned = start - start % self.page_size
|
||||
end = round_up(start + self.cpu_page_size, self.page_size)
|
||||
@@ -380,7 +446,7 @@ class SharedOffloadRegion:
|
||||
self.fd = None
|
||||
if self._creator and getattr(self, "mmap_path", None):
|
||||
try:
|
||||
os.unlink(self.mmap_path)
|
||||
self._unlink()
|
||||
logger.info("Removed mmap file %s", self.mmap_path)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
|
||||
@@ -22,6 +22,10 @@ from vllm.v1.kv_offload.cpu.gpu_worker import CPUOffloadingWorker
|
||||
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
|
||||
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
||||
|
||||
# Split the shared region into this many shm files so workers pre-fault them
|
||||
# in parallel (page-cache inserts are serialized per file).
|
||||
_REGION_SEGMENTS = 32
|
||||
|
||||
|
||||
def _all_workers_barrier() -> None:
|
||||
"""Block until every worker rank has reached this point (gloo cpu group).
|
||||
@@ -188,6 +192,7 @@ class CPUOffloadingSpec(OffloadingSpec):
|
||||
kv_bytes_per_chunk=self.kv_bytes_per_chunk,
|
||||
cpu_page_size=self.cpu_page_size_per_worker,
|
||||
barrier=_all_workers_barrier,
|
||||
num_segments=_REGION_SEGMENTS,
|
||||
populate_shard=(
|
||||
(worker_index, world_size) if self.replicated_layout else None
|
||||
),
|
||||
|
||||
@@ -34,11 +34,34 @@ def register_top_k_top_p_warmups() -> None:
|
||||
_topp_split_mask.register_warmup()
|
||||
|
||||
|
||||
def _skip_aiter_sampler_on_gfx1250() -> bool:
|
||||
# Lazy ROCm-only import; keeps arch detection out of import time on CUDA/CPU.
|
||||
from vllm.platforms.rocm import on_gfx1250
|
||||
def _aiter_temp_gumbel_sample(
|
||||
logits: torch.Tensor, generators: dict[int, torch.Generator]
|
||||
) -> torch.Tensor:
|
||||
"""Fused temperature Gumbel-max sampling via aiter.
|
||||
|
||||
return on_gfx1250()
|
||||
Logits are already temperature-scaled by the parent Sampler, so temps=1.
|
||||
Returns int64 token ids of shape (num_tokens,).
|
||||
"""
|
||||
from aiter import mixed_sample_outer_exponential
|
||||
|
||||
n, vocab = logits.shape
|
||||
out = torch.empty(n, dtype=torch.int32, device=logits.device)
|
||||
if not generators:
|
||||
# Common case: shared (1, vocab) exponential noise (cheap RNG).
|
||||
exp = (
|
||||
torch.empty((1, vocab), dtype=torch.float32, device=logits.device)
|
||||
.exponential_()
|
||||
.expand(n, vocab)
|
||||
)
|
||||
else:
|
||||
exp = torch.empty((n, vocab), dtype=torch.float32, device=logits.device)
|
||||
if len(generators) != n:
|
||||
exp.exponential_()
|
||||
for i, g in generators.items():
|
||||
exp[i].exponential_(generator=g)
|
||||
temps = torch.ones(n, dtype=torch.float32, device=logits.device)
|
||||
mixed_sample_outer_exponential(out, logits, exp, temps, eps=1e-10)
|
||||
return out.to(torch.int64)
|
||||
|
||||
|
||||
def _flashinfer_jit_unsupported_reason(capability: DeviceCapability) -> str | None:
|
||||
@@ -246,8 +269,12 @@ class TopKTopPSampler(nn.Module):
|
||||
elif (
|
||||
logprobs_mode not in PROCESSED_LOGPROBS_MODES
|
||||
and rocm_aiter_ops.is_enabled()
|
||||
and not _skip_aiter_sampler_on_gfx1250() # TODO (JPVILLAM): Enable
|
||||
):
|
||||
# Lazy ROCm-only import; keeps arch detection out of import time
|
||||
# on CUDA/CPU.
|
||||
from vllm.platforms.rocm import on_gfx1250
|
||||
|
||||
self.on_gfx1250 = on_gfx1250()
|
||||
self.aiter_ops = None
|
||||
self._aiter_ops_import_failed = False
|
||||
logger.info_once(
|
||||
@@ -375,7 +402,16 @@ class TopKTopPSampler(nn.Module):
|
||||
k: torch.Tensor | None,
|
||||
p: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""Optimized ROCm/aiter path (same structure as forward_cuda)."""
|
||||
"""Optimized ROCm/aiter path."""
|
||||
if self.on_gfx1250:
|
||||
if (
|
||||
k is None
|
||||
and p is None
|
||||
and not self.use_fp64_gumbel
|
||||
and self.logprobs_mode not in PROCESSED_LOGPROBS_MODES
|
||||
):
|
||||
return _aiter_temp_gumbel_sample(logits, generators), None
|
||||
return self.forward_native(logits, generators, k, p)
|
||||
if (k is None and p is None) or generators:
|
||||
if generators:
|
||||
logger.warning_once(
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config.model import LogprobsMode
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
@@ -19,6 +20,61 @@ from vllm.v1.sample.ops.topk_topp_sampler import TopKTopPSampler
|
||||
_SAMPLING_EPS = 1e-5
|
||||
|
||||
|
||||
def mixed_sample_eligible(sm: SamplingMetadata, predict_bonus_token: bool) -> bool:
|
||||
"""Whether the fused aiter sample-from-raw-logits kernel can be used."""
|
||||
if not rocm_aiter_ops.is_enabled():
|
||||
return False
|
||||
|
||||
from vllm.platforms.rocm import on_gfx1250
|
||||
|
||||
if not on_gfx1250():
|
||||
return False
|
||||
if predict_bonus_token:
|
||||
return False
|
||||
if not sm.all_random or sm.temperature is None:
|
||||
return False
|
||||
if sm.top_k is not None or sm.top_p is not None:
|
||||
return False
|
||||
if sm.max_num_logprobs is not None or sm.logprob_token_ids:
|
||||
return False
|
||||
if (
|
||||
not sm.no_penalties
|
||||
or sm.allowed_token_ids_mask is not None
|
||||
or sm.bad_words_token_ids
|
||||
):
|
||||
return False
|
||||
lp = sm.logitsprocs
|
||||
if lp.argmax_invariant or lp.non_argmax_invariant:
|
||||
return False
|
||||
h = sm.thinking_budget_state_holder
|
||||
return h is None or not h.has_tracked_requests()
|
||||
|
||||
|
||||
def _aiter_raw_fused_sample(logits: torch.Tensor, sm: SamplingMetadata) -> torch.Tensor:
|
||||
"""Sample token ids from raw (unscaled) logits via a fused aiter kernel.
|
||||
|
||||
Returns int32 ids of shape (num_tokens, 1).
|
||||
"""
|
||||
from aiter import mixed_sample_outer_exponential
|
||||
|
||||
n, vocab = logits.shape
|
||||
out = torch.empty(n, dtype=torch.int32, device=logits.device)
|
||||
gens = sm.generators
|
||||
if not gens:
|
||||
exp = (
|
||||
torch.empty((1, vocab), dtype=torch.float32, device=logits.device)
|
||||
.exponential_()
|
||||
.expand(n, vocab)
|
||||
)
|
||||
else:
|
||||
exp = torch.empty((n, vocab), dtype=torch.float32, device=logits.device)
|
||||
exp.exponential_()
|
||||
for i, g in gens.items():
|
||||
exp[i].exponential_(generator=g)
|
||||
mixed_sample_outer_exponential(out, logits, exp, sm.temperature, eps=1e-10)
|
||||
return out.unsqueeze(-1)
|
||||
|
||||
|
||||
class Sampler(nn.Module):
|
||||
"""A layer that samples the next tokens from the model's outputs
|
||||
with the following steps in order:
|
||||
@@ -77,6 +133,11 @@ class Sampler(nn.Module):
|
||||
predict_bonus_token: bool = False,
|
||||
logprobs_mode_override: LogprobsMode | None = None,
|
||||
) -> SamplerOutput:
|
||||
if mixed_sample_eligible(sampling_metadata, predict_bonus_token):
|
||||
return SamplerOutput(
|
||||
sampled_token_ids=_aiter_raw_fused_sample(logits, sampling_metadata),
|
||||
logprobs_tensors=None,
|
||||
)
|
||||
logprobs_mode = logprobs_mode_override or self.logprobs_mode
|
||||
# NOTE(woosuk): Use the original logits (before any penalties or
|
||||
# temperature scaling) for the top-k logprobs.
|
||||
|
||||
@@ -5,6 +5,8 @@ from collections.abc import Iterable, Sequence
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from typing import TYPE_CHECKING, NamedTuple
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.parser.engine.adapters import ParserEngineReasoningAdapter
|
||||
@@ -20,7 +22,6 @@ from vllm.v1.structured_output.backend_xgrammar import XgrammarBackend
|
||||
from vllm.v1.structured_output.utils import strip_speculative_padding
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
import torch
|
||||
|
||||
@@ -211,10 +212,16 @@ class StructuredOutputManager:
|
||||
raise
|
||||
|
||||
def _fill_bitmasks(
|
||||
self, batch: Iterable[tuple[StructuredOutputGrammar, int, bool]]
|
||||
self,
|
||||
batch: Iterable[tuple[StructuredOutputGrammar, int, bool, tuple[int, ...]]],
|
||||
) -> None:
|
||||
"""Fill one bitmask row per entry.
|
||||
|
||||
A row the grammar does not constrain allows every token except
|
||||
`masked_ids`, the implicit reasoning ends masked while reasoning.
|
||||
"""
|
||||
assert self._grammar_bitmask is not None
|
||||
for grammar, index, apply_bitmask in batch:
|
||||
for grammar, index, apply_bitmask, masked_ids in batch:
|
||||
if apply_bitmask and not grammar.is_terminated():
|
||||
grammar.fill_bitmask(self._grammar_bitmask, index)
|
||||
else:
|
||||
@@ -222,12 +229,41 @@ class StructuredOutputManager:
|
||||
# reset the relevant part of the bitmask for consequent
|
||||
# requests here.
|
||||
self._grammar_bitmask[index].fill_(self._full_mask)
|
||||
if masked_ids:
|
||||
row = self._grammar_bitmask[index].numpy().view(np.uint32)
|
||||
for token_id in masked_ids:
|
||||
row[token_id >> 5] &= ~np.uint32(1 << (token_id & 31))
|
||||
|
||||
def _async_submit_fill_bitmask(
|
||||
self, batch: list[tuple[StructuredOutputGrammar, int, bool]]
|
||||
self, batch: list[tuple[StructuredOutputGrammar, int, bool, tuple[int, ...]]]
|
||||
) -> Future:
|
||||
return self.executor_for_fillmask.submit(self._fill_bitmasks, batch)
|
||||
|
||||
def _masked_reasoning_ends(self, request: "Request") -> tuple[int, ...]:
|
||||
"""Implicit reasoning-end tokens the grammar rejects as first content.
|
||||
|
||||
Such a token (e.g. a tool-call start) ends reasoning and is also the
|
||||
first content token, so sampling one where the grammar cannot start
|
||||
with it would make the output invalid. These stay masked while the
|
||||
request reasons. The grammar does not advance before reasoning ends,
|
||||
so the set is fixed per request.
|
||||
"""
|
||||
structured_req = request.structured_output_request
|
||||
assert structured_req is not None
|
||||
if structured_req.masked_reasoning_end_ids is None:
|
||||
masked: tuple[int, ...] = ()
|
||||
reasoner = self._get_reasoner(request)
|
||||
if isinstance(reasoner, ParserEngineReasoningAdapter):
|
||||
grammar = structured_req.grammar
|
||||
assert isinstance(grammar, StructuredOutputGrammar)
|
||||
masked = tuple(
|
||||
token_id
|
||||
for token_id in sorted(reasoner.implicit_reasoning_end_token_ids)
|
||||
if not grammar.validate_tokens([token_id])
|
||||
)
|
||||
structured_req.masked_reasoning_end_ids = masked
|
||||
return structured_req.masked_reasoning_end_ids
|
||||
|
||||
def _get_constraint_bounds(
|
||||
self,
|
||||
request: "Request",
|
||||
@@ -377,7 +413,10 @@ class StructuredOutputManager:
|
||||
apply_bitmask = (
|
||||
self._get_constraint_bounds(request, ()).constraint_start == 0
|
||||
)
|
||||
batch.append((grammar, cumulative_index, apply_bitmask))
|
||||
masked_ids = (
|
||||
() if apply_bitmask else self._masked_reasoning_ends(request)
|
||||
)
|
||||
batch.append((grammar, cumulative_index, apply_bitmask, masked_ids))
|
||||
if len(batch) == self.fill_bitmask_parallel_batch_size:
|
||||
promises.append(self._async_submit_fill_bitmask(batch))
|
||||
batch = []
|
||||
@@ -405,6 +444,10 @@ class StructuredOutputManager:
|
||||
grammar_start, constraint_start = self._get_constraint_bounds(
|
||||
request, strip_speculative_padding(req_tokens)
|
||||
)
|
||||
# Rows before constraint_start are still reasoning.
|
||||
masked_ids = (
|
||||
self._masked_reasoning_ends(request) if constraint_start else ()
|
||||
)
|
||||
state_advancements = 0
|
||||
seen_padding = False
|
||||
# Row filled from the last valid grammar state before a draft
|
||||
@@ -417,7 +460,11 @@ class StructuredOutputManager:
|
||||
cumulative_index += 1
|
||||
continue
|
||||
apply_bitmask = not seen_padding and i >= constraint_start
|
||||
self._fill_bitmasks(((grammar, cumulative_index, apply_bitmask),))
|
||||
reasoning = not seen_padding and i < constraint_start
|
||||
row_masked = masked_ids if reasoning else ()
|
||||
self._fill_bitmasks(
|
||||
((grammar, cumulative_index, apply_bitmask, row_masked),)
|
||||
)
|
||||
if token == -1:
|
||||
seen_padding = True
|
||||
elif not seen_padding and i >= grammar_start:
|
||||
@@ -444,7 +491,11 @@ class StructuredOutputManager:
|
||||
bonus_apply = not seen_padding and constraint_start <= len(
|
||||
req_tokens
|
||||
)
|
||||
self._fill_bitmasks(((grammar, cumulative_index, bonus_apply),))
|
||||
bonus_reasoning = not seen_padding and not bonus_apply
|
||||
bonus_masked = masked_ids if bonus_reasoning else ()
|
||||
self._fill_bitmasks(
|
||||
((grammar, cumulative_index, bonus_apply, bonus_masked),)
|
||||
)
|
||||
cumulative_index += 1
|
||||
|
||||
if state_advancements > 0:
|
||||
|
||||
@@ -28,6 +28,9 @@ class StructuredOutputRequest:
|
||||
# Cached per request; do not share reasoning parsers across requests because
|
||||
# their behavior can depend on reasoning_parser_kwargs.
|
||||
reasoner: "ReasoningParser | None" = None
|
||||
# Implicit reasoning-end tokens the grammar rejects as the first content
|
||||
# token; masked while the request reasons. Computed on first use.
|
||||
masked_reasoning_end_ids: tuple[int, ...] | None = None
|
||||
|
||||
@staticmethod
|
||||
def from_sampling_params(
|
||||
|
||||
@@ -14,6 +14,7 @@ from vllm.v1.watermarking.detector import (
|
||||
)
|
||||
from vllm.v1.watermarking.prfs import PhiloxPRF, WatermarkPRF, create_prf
|
||||
from vllm.v1.watermarking.watermarker import (
|
||||
DraftContextState,
|
||||
RandomSampler,
|
||||
SupportsSpeculativeDecoding,
|
||||
Watermarker,
|
||||
@@ -101,6 +102,42 @@ class GumbelWatermarker(Watermarker):
|
||||
)
|
||||
return WatermarkSample(token_ids, logits)
|
||||
|
||||
def _try_sample_block(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
num_steps: int,
|
||||
random_sampler: RandomSampler,
|
||||
state: DraftContextState,
|
||||
) -> torch.Tensor | None:
|
||||
if type(self.prf) is not PhiloxPRF or logits.device.type != "cuda":
|
||||
return None
|
||||
|
||||
from vllm.v1.worker.gpu.sample.watermark import draft_philox_gumbel_sample
|
||||
|
||||
assert random_sampler.logits_cache is not None
|
||||
assert random_sampler.logits_cache_col is not None
|
||||
return draft_philox_gumbel_sample(
|
||||
logits,
|
||||
state.contexts,
|
||||
self.prf.key,
|
||||
num_steps=num_steps,
|
||||
expanded_idx_mapping=random_sampler.expanded_idx_mapping,
|
||||
temperatures=random_sampler.temperatures,
|
||||
seeds=random_sampler.seeds,
|
||||
positions=random_sampler.positions,
|
||||
enabled=state.enabled,
|
||||
logits_cache=random_sampler.logits_cache,
|
||||
logits_cache_col=random_sampler.logits_cache_col,
|
||||
use_fp64=random_sampler.use_fp64,
|
||||
prior_contexts=state.prior_contexts,
|
||||
all_token_ids=state.all_token_ids,
|
||||
prompt_lens=state.prompt_lens,
|
||||
total_lens=state.total_lens,
|
||||
deduplicate=state.deduplicate_contexts != "none",
|
||||
max_history=state.deduplicate_contexts_max_history,
|
||||
include_prompt=state.deduplicate_contexts == "all",
|
||||
)
|
||||
|
||||
|
||||
class DualKeyGumbelWatermarker(Watermarker, SupportsSpeculativeDecoding):
|
||||
def __init__(
|
||||
|
||||
@@ -8,6 +8,7 @@ from vllm.v1.watermarking.factory import create_watermarker
|
||||
from vllm.v1.watermarking.gumbel import GumbelWatermarker
|
||||
from vllm.v1.watermarking.prfs import PhiloxPRF
|
||||
from vllm.v1.watermarking.watermarker import (
|
||||
DraftContextState,
|
||||
RandomSampler,
|
||||
SupportsSpeculativeDecoding,
|
||||
Watermarker,
|
||||
@@ -28,25 +29,25 @@ class DraftWatermarker:
|
||||
) -> None:
|
||||
self.watermarker = watermarker
|
||||
self.num_speculative_steps = num_speculative_steps
|
||||
self.deduplicate_contexts = deduplicate_contexts
|
||||
self.deduplicate_contexts_max_history = deduplicate_contexts_max_history
|
||||
self.contexts = torch.zeros(
|
||||
max_num_reqs,
|
||||
watermarker.context_width,
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
self.state = DraftContextState(
|
||||
contexts=torch.zeros(
|
||||
max_num_reqs,
|
||||
watermarker.context_width,
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
),
|
||||
enabled=torch.zeros(max_num_reqs, dtype=torch.bool, device=device),
|
||||
prior_contexts=torch.zeros(
|
||||
max_num_reqs,
|
||||
num_speculative_steps,
|
||||
watermarker.context_width,
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
),
|
||||
prompt_lens=torch.zeros(max_num_reqs, dtype=torch.int32, device=device),
|
||||
deduplicate_contexts=deduplicate_contexts,
|
||||
deduplicate_contexts_max_history=deduplicate_contexts_max_history,
|
||||
)
|
||||
self.enabled = torch.zeros(max_num_reqs, dtype=torch.bool, device=device)
|
||||
self.prior_contexts = torch.zeros(
|
||||
max_num_reqs,
|
||||
num_speculative_steps,
|
||||
watermarker.context_width,
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
)
|
||||
self.prompt_lens = torch.zeros(max_num_reqs, dtype=torch.int32, device=device)
|
||||
self.all_token_ids: torch.Tensor | None = None
|
||||
self.total_lens: torch.Tensor | None = None
|
||||
|
||||
def prepare(
|
||||
self,
|
||||
@@ -57,11 +58,12 @@ class DraftWatermarker:
|
||||
total_lens: torch.Tensor,
|
||||
) -> None:
|
||||
num_reqs = contexts.shape[0]
|
||||
self.contexts[:num_reqs].copy_(contexts)
|
||||
self.enabled[:num_reqs].copy_(enabled)
|
||||
self.all_token_ids = all_token_ids
|
||||
self.prompt_lens.copy_(prompt_lens)
|
||||
self.total_lens = total_lens
|
||||
state = self.state
|
||||
state.contexts[:num_reqs].copy_(contexts)
|
||||
state.enabled[:num_reqs].copy_(enabled)
|
||||
state.all_token_ids = all_token_ids
|
||||
state.prompt_lens.copy_(prompt_lens)
|
||||
state.total_lens = total_lens
|
||||
|
||||
def _sampling_state(
|
||||
self,
|
||||
@@ -75,22 +77,23 @@ class DraftWatermarker:
|
||||
steps = torch.as_tensor(draft_step, device=logits.device, dtype=torch.int64)
|
||||
if steps.ndim == 0:
|
||||
steps = steps.expand(num_rows)
|
||||
enabled = self.enabled[:num_rows] & (request_temperatures != 0)
|
||||
if self.deduplicate_contexts == "none":
|
||||
state = self.state
|
||||
enabled = state.enabled[:num_rows] & (request_temperatures != 0)
|
||||
if state.deduplicate_contexts == "none":
|
||||
return steps, enabled
|
||||
assert self.all_token_ids is not None
|
||||
assert self.total_lens is not None
|
||||
assert state.all_token_ids is not None
|
||||
assert state.total_lens is not None
|
||||
enabled = draft_watermarking_mask(
|
||||
self.all_token_ids,
|
||||
state.all_token_ids,
|
||||
idx_mapping,
|
||||
self.prompt_lens,
|
||||
self.total_lens,
|
||||
self.prior_contexts,
|
||||
state.prompt_lens,
|
||||
state.total_lens,
|
||||
state.prior_contexts,
|
||||
contexts,
|
||||
steps,
|
||||
enabled,
|
||||
self.deduplicate_contexts_max_history,
|
||||
include_prompt=self.deduplicate_contexts == "all",
|
||||
state.deduplicate_contexts_max_history,
|
||||
include_prompt=state.deduplicate_contexts == "all",
|
||||
)
|
||||
return steps, enabled
|
||||
|
||||
@@ -107,14 +110,26 @@ class DraftWatermarker:
|
||||
logits_cache_col: torch.Tensor | None = None,
|
||||
use_fp64: bool = False,
|
||||
logits_cache_source: torch.Tensor | None = None,
|
||||
num_steps: int = 1,
|
||||
) -> torch.Tensor:
|
||||
if num_steps > 1:
|
||||
return self.sample_block(
|
||||
logits,
|
||||
idx_mapping,
|
||||
temperature,
|
||||
seed,
|
||||
pos,
|
||||
logits_cache=logits_cache,
|
||||
logits_cache_col=logits_cache_col,
|
||||
use_fp64=use_fp64,
|
||||
)
|
||||
assert apply_temperature
|
||||
assert is_drafting
|
||||
assert logits_cache is not None
|
||||
assert logits_cache_col is not None
|
||||
num_rows = logits.shape[0]
|
||||
request_temperatures = temperature[idx_mapping]
|
||||
contexts = self.contexts[:num_rows]
|
||||
contexts = self.state.contexts[:num_rows]
|
||||
steps, enabled = self._sampling_state(
|
||||
logits, idx_mapping, request_temperatures, logits_cache_col, contexts
|
||||
)
|
||||
@@ -145,6 +160,68 @@ class DraftWatermarker:
|
||||
contexts.copy_(torch.cat((contexts[:, 1:], sampled.unsqueeze(-1)), dim=-1))
|
||||
return sampled
|
||||
|
||||
def sample_block(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
idx_mapping: torch.Tensor,
|
||||
temperature: torch.Tensor,
|
||||
seed: torch.Tensor,
|
||||
pos: torch.Tensor,
|
||||
logits_cache: torch.Tensor | None = None,
|
||||
logits_cache_col: torch.Tensor | None = None,
|
||||
use_fp64: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""Sample drafts flattened as (request, step), one step at a time.
|
||||
The target verifies each draft with a context that includes the earlier
|
||||
drafts of its request, so steps must be sampled in order even when
|
||||
their logits were computed in parallel.
|
||||
"""
|
||||
assert logits_cache_col is not None
|
||||
num_steps = self.num_speculative_steps
|
||||
sampled_block = self.watermarker._try_sample_block(
|
||||
logits,
|
||||
num_steps,
|
||||
RandomSampler(
|
||||
expanded_idx_mapping=idx_mapping,
|
||||
temperatures=temperature,
|
||||
seeds=seed,
|
||||
positions=pos,
|
||||
use_fp64=use_fp64,
|
||||
is_drafting=True,
|
||||
logits_cache=logits_cache,
|
||||
logits_cache_col=logits_cache_col,
|
||||
),
|
||||
self.state,
|
||||
)
|
||||
if sampled_block is not None:
|
||||
return sampled_block
|
||||
num_reqs = logits.shape[0] // num_steps
|
||||
|
||||
def by_step(tensor: torch.Tensor) -> torch.Tensor:
|
||||
return tensor.view(num_reqs, num_steps, *tensor.shape[1:])
|
||||
|
||||
logits = by_step(logits)
|
||||
idx_mapping = by_step(idx_mapping)
|
||||
pos = by_step(pos)
|
||||
logits_cache_col = by_step(logits_cache_col)
|
||||
sampled = torch.empty(
|
||||
num_reqs, num_steps, dtype=torch.int64, device=logits.device
|
||||
)
|
||||
for step in range(num_steps):
|
||||
sampled[:, step] = self.sample(
|
||||
logits[:, step],
|
||||
idx_mapping[:, step],
|
||||
temperature,
|
||||
seed,
|
||||
pos[:, step],
|
||||
apply_temperature=True,
|
||||
is_drafting=True,
|
||||
logits_cache=logits_cache,
|
||||
logits_cache_col=logits_cache_col[:, step],
|
||||
use_fp64=use_fp64,
|
||||
)
|
||||
return sampled.view(-1)
|
||||
|
||||
|
||||
def create_speculative_target_watermarker(watermarker: Watermarker) -> Watermarker:
|
||||
if isinstance(watermarker, SupportsSpeculativeDecoding):
|
||||
|
||||
@@ -7,6 +7,7 @@ from typing import Protocol, runtime_checkable
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config.watermarking import WatermarkContextScope
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
|
||||
|
||||
|
||||
@@ -44,6 +45,18 @@ class RandomSampler:
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DraftContextState:
|
||||
contexts: torch.Tensor
|
||||
enabled: torch.Tensor
|
||||
prior_contexts: torch.Tensor
|
||||
prompt_lens: torch.Tensor
|
||||
deduplicate_contexts: WatermarkContextScope
|
||||
deduplicate_contexts_max_history: int | None
|
||||
all_token_ids: torch.Tensor | None = None
|
||||
total_lens: torch.Tensor | None = None
|
||||
|
||||
|
||||
class Watermarker(ABC):
|
||||
@property
|
||||
@abstractmethod
|
||||
@@ -94,6 +107,20 @@ class Watermarker(ABC):
|
||||
) -> WatermarkSample | None:
|
||||
return None
|
||||
|
||||
def _try_sample_block(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
num_steps: int,
|
||||
random_sampler: RandomSampler,
|
||||
state: DraftContextState,
|
||||
) -> torch.Tensor | None:
|
||||
"""Sample a (request, step) draft block in one fused pass.
|
||||
|
||||
Returns None when there is no fused implementation, in which case the
|
||||
caller samples the block one step at a time.
|
||||
"""
|
||||
return None
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsSpeculativeDecoding(Protocol):
|
||||
|
||||
@@ -234,10 +234,24 @@ class CPUWorker(Worker):
|
||||
else:
|
||||
consumed_memory = psutil.Process(os.getpid()).memory_info().rss
|
||||
requested_memory_for_kv = int(self.requested_cpu_memory - consumed_memory)
|
||||
if (
|
||||
requested_memory_for_kv <= 0
|
||||
or requested_memory_for_kv > available_memory
|
||||
):
|
||||
if requested_memory_for_kv <= 0:
|
||||
# The worker's own footprint already fills its reservation, so
|
||||
# freeing memory elsewhere cannot help: only a larger
|
||||
# reservation or an explicit KV cache size can.
|
||||
in_use = math.ceil(consumed_memory / memory_status.total_memory * 100)
|
||||
raise ValueError(
|
||||
f"The worker already uses {format_gib(consumed_memory)} GiB "
|
||||
"(model weights and runtime), more than the "
|
||||
f"{format_gib(self.requested_cpu_memory)} GiB of CPU memory "
|
||||
f"reserved for it on node {cpu_core.numa_node}, leaving none "
|
||||
"for the KV cache. On the CPU backend, the "
|
||||
"`--gpu-memory-utilization` flag controls the fraction of CPU "
|
||||
"memory reserved (despite its name). To resolve: increase "
|
||||
f"`--gpu-memory-utilization` to more than {in_use / 100:.2f}, "
|
||||
"or set the KV cache size explicitly with "
|
||||
"`--kv-cache-memory-bytes` or VLLM_CPU_KVCACHE_SPACE."
|
||||
)
|
||||
if requested_memory_for_kv > available_memory:
|
||||
raise ValueError(
|
||||
f"Available memory on node {cpu_core.numa_node} "
|
||||
f"({format_gib(available_memory)}/"
|
||||
|
||||
@@ -33,6 +33,7 @@ from vllm.distributed.parallel_state import (
|
||||
)
|
||||
from vllm.forward_context import BatchDescriptor, set_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.interfaces import requires_raw_input_tokens
|
||||
from vllm.model_executor.offloader.base import get_offloader
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -690,7 +691,8 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
}
|
||||
if not self.is_first_pp_rank:
|
||||
# Update for non-first PP ranks.
|
||||
model_inputs["input_ids"] = None
|
||||
if not requires_raw_input_tokens(model):
|
||||
model_inputs["input_ids"] = None
|
||||
model_inputs["inputs_embeds"] = None
|
||||
assert intermediate_tensors is not None
|
||||
model_inputs["intermediate_tensors"] = intermediate_tensors[:num_tokens]
|
||||
|
||||
@@ -5,7 +5,6 @@ from __future__ import annotations
|
||||
from dataclasses import dataclass, replace
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
@@ -63,7 +62,6 @@ def sync_cudagraph_and_dp_padding(
|
||||
Returns (synced_batch_desc, sync). `sync` is None when no rank has work.
|
||||
"""
|
||||
assert dp_size > 1, "DP size must be greater than 1"
|
||||
group = get_dp_group().cpu_group
|
||||
tensor = torch.zeros(6, dp_size, dtype=torch.int32, device="cpu")
|
||||
tensor[0][dp_rank] = num_tokens
|
||||
tensor[1][dp_rank] = desired_batch_desc.cg_mode.value
|
||||
@@ -74,7 +72,7 @@ def sync_cudagraph_and_dp_padding(
|
||||
if should_skip_dp_coordination():
|
||||
tensor[:] = tensor[:, dp_rank, None].clone()
|
||||
else:
|
||||
dist.all_reduce(tensor, group=group)
|
||||
get_dp_group().cpu_all_reduce_(tensor)
|
||||
|
||||
num_tokens_across_dp = tensor[0]
|
||||
cg_mode_across_dp = tensor[1]
|
||||
|
||||
@@ -33,7 +33,7 @@ import vllm.envs as envs
|
||||
from vllm.compilation.counter import compilation_counter
|
||||
from vllm.compilation.cuda_graph import CUDAGraphStat
|
||||
from vllm.compilation.wrapper import compile_model_with_stock_torch
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.config.compilation import CompilationMode, CUDAGraphMode
|
||||
from vllm.distributed.aux_output_connector.worker import (
|
||||
AuxOutputWorkerConnector,
|
||||
@@ -46,6 +46,9 @@ from vllm.model_executor.layers.fused_moe.all2all_utils import get_ep_all2all_ma
|
||||
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
|
||||
initialize_mamba_ssu_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.lm_head import (
|
||||
refresh_quantized_lm_heads,
|
||||
)
|
||||
from vllm.model_executor.model_loader import get_model_loader
|
||||
from vllm.model_executor.model_loader.reload.inplace import reload_weights
|
||||
from vllm.model_executor.models.interfaces import requires_raw_input_tokens
|
||||
@@ -569,6 +572,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
return None
|
||||
return speculator.model
|
||||
|
||||
def reset_lora_state(self) -> None:
|
||||
super().reset_lora_state()
|
||||
# Called whenever the target's weights are replaced (reload, weight
|
||||
# transfer), so draft weights derived from them must follow.
|
||||
draft_model = self.get_draft_model()
|
||||
if draft_model is not None:
|
||||
with set_current_vllm_config(self.vllm_config):
|
||||
refresh_quantized_lm_heads(draft_model)
|
||||
|
||||
def reload_weights(
|
||||
self,
|
||||
weights_iterator: Iterable[tuple[str, torch.Tensor]] | None = None,
|
||||
@@ -1970,7 +1982,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
}
|
||||
if not self.is_first_pp_rank:
|
||||
# Update for non-first PP ranks.
|
||||
model_inputs["input_ids"] = None
|
||||
if not requires_raw_input_tokens(self.model):
|
||||
model_inputs["input_ids"] = None
|
||||
model_inputs["inputs_embeds"] = None
|
||||
|
||||
# Prepare the intermediate tensors.
|
||||
|
||||
@@ -70,6 +70,12 @@ def tl_rand32(seed, offset, includes_zero: tl.constexpr):
|
||||
return u
|
||||
|
||||
|
||||
@triton.jit
|
||||
def spec_step_seed(seed, step):
|
||||
"""Mix a speculative step into the high bits of a request seed."""
|
||||
return seed ^ (tl.cast(step, tl.int64) << 32)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _murmur3_rotl32(value, shift: tl.constexpr):
|
||||
return (value << shift) | (value >> (32 - shift))
|
||||
@@ -257,6 +263,8 @@ def gumbel_block_argmax(
|
||||
|
||||
seed = tl.load(seeds_ptr + req_state_idx, mask=is_valid_req, other=0)
|
||||
pos = tl.load(pos_ptr + token_idx)
|
||||
if logits_cache_ptr is not None:
|
||||
seed = spec_step_seed(seed, col)
|
||||
return gumbel_noised_argmax(
|
||||
logits,
|
||||
block,
|
||||
|
||||
@@ -25,7 +25,7 @@ from vllm.v1.watermarking.prfs.philox import (
|
||||
from vllm.v1.watermarking.prfs.philox import (
|
||||
_UINT32_MASK as _UINT32_MASK_VALUE,
|
||||
)
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_noised_argmax
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_noised_argmax, spec_step_seed
|
||||
|
||||
if HAS_TRITON:
|
||||
from triton.language import math as tl_math
|
||||
@@ -530,8 +530,10 @@ def _philox_gumbel_from_logits(logits, word):
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _gumbel_value(logits_ptr, output, mask):
|
||||
def _gumbel_value(logits_ptr, output, mask, temperature):
|
||||
logits = tl.load(logits_ptr, mask=mask, other=float("-inf")).to(tl.float32)
|
||||
if temperature is not None:
|
||||
logits = tl_math.div_rn(logits, temperature)
|
||||
return _philox_gumbel_from_logits(logits, output)
|
||||
|
||||
|
||||
@@ -606,6 +608,7 @@ def _philox_gumbel_kernel(
|
||||
USE_FP64: tl.constexpr,
|
||||
IS_DRAFTING: tl.constexpr,
|
||||
PER_TOKEN_COL: tl.constexpr,
|
||||
APPLY_TEMPERATURE: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
block_index = tl.program_id(1)
|
||||
@@ -619,6 +622,13 @@ def _philox_gumbel_kernel(
|
||||
req_state_idx = tl.load(expanded_idx_mapping_ptr + row).to(tl.int64)
|
||||
valid_req = req_state_idx >= 0
|
||||
|
||||
row_temperature = None
|
||||
if APPLY_TEMPERATURE:
|
||||
row_temperature = tl.load(
|
||||
temp_ptr + req_state_idx, mask=valid_req, other=0.0
|
||||
).to(tl.float32)
|
||||
row_temperature = tl.where(row_temperature == 0, 1.0, row_temperature)
|
||||
|
||||
if logits_cache_ptr is not None:
|
||||
if PER_TOKEN_COL:
|
||||
col = tl.load(logits_cache_col_ptr + row)
|
||||
@@ -644,10 +654,14 @@ def _philox_gumbel_kernel(
|
||||
)
|
||||
seed = tl.load(seeds_ptr + req_state_idx, mask=valid_req, other=0)
|
||||
pos = tl.load(pos_ptr + row)
|
||||
if logits_cache_ptr is not None:
|
||||
seed = spec_step_seed(seed, col)
|
||||
logits_row = logits_ptr + row * logits_stride
|
||||
logits = tl.load(logits_row + candidate, mask=mask, other=float("-inf")).to(
|
||||
tl.float32
|
||||
)
|
||||
if row_temperature is not None:
|
||||
logits = tl_math.div_rn(logits, row_temperature)
|
||||
value, index = gumbel_noised_argmax(
|
||||
logits,
|
||||
candidate,
|
||||
@@ -681,16 +695,16 @@ def _philox_gumbel_kernel(
|
||||
candidate_3 = candidate_0 + 3
|
||||
logits_row = logits_ptr + row * logits_stride
|
||||
value_0 = _gumbel_value(
|
||||
logits_row + candidate_0, output_0, candidate_0 < vocab_size
|
||||
logits_row + candidate_0, output_0, candidate_0 < vocab_size, row_temperature
|
||||
)
|
||||
value_1 = _gumbel_value(
|
||||
logits_row + candidate_1, output_1, candidate_1 < vocab_size
|
||||
logits_row + candidate_1, output_1, candidate_1 < vocab_size, row_temperature
|
||||
)
|
||||
value_2 = _gumbel_value(
|
||||
logits_row + candidate_2, output_2, candidate_2 < vocab_size
|
||||
logits_row + candidate_2, output_2, candidate_2 < vocab_size, row_temperature
|
||||
)
|
||||
value_3 = _gumbel_value(
|
||||
logits_row + candidate_3, output_3, candidate_3 < vocab_size
|
||||
logits_row + candidate_3, output_3, candidate_3 < vocab_size, row_temperature
|
||||
)
|
||||
best_value = value_0
|
||||
best_token = candidate_0
|
||||
@@ -797,6 +811,238 @@ def philox_gumbel_sample(
|
||||
USE_FP64=use_fp64,
|
||||
IS_DRAFTING=is_drafting,
|
||||
PER_TOKEN_COL=logits_cache_col is not None and logits_cache_col.dim() > 0,
|
||||
APPLY_TEMPERATURE=False,
|
||||
)
|
||||
max_block_index = local_max.argmax(dim=-1, keepdim=True)
|
||||
return local_argmax.gather(dim=-1, index=max_block_index).view(-1)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["step"])
|
||||
def _draft_step_kernel(
|
||||
sampled_ptr,
|
||||
sampled_stride,
|
||||
step,
|
||||
local_argmax_ptr,
|
||||
local_max_ptr,
|
||||
num_blocks,
|
||||
contexts_ptr,
|
||||
context_stride,
|
||||
skip_mask_ptr,
|
||||
next_req_indices_ptr,
|
||||
next_steps_ptr,
|
||||
temp_ptr,
|
||||
enabled_ptr,
|
||||
all_token_ids_ptr,
|
||||
all_token_ids_stride,
|
||||
prompt_lens_ptr,
|
||||
total_lens_ptr,
|
||||
prior_contexts_ptr,
|
||||
prior_contexts_stride_0,
|
||||
prior_contexts_stride_1,
|
||||
CONTEXT_WIDTH: tl.constexpr,
|
||||
NUM_SPECULATIVE_STEPS: tl.constexpr,
|
||||
MAX_HISTORY: tl.constexpr,
|
||||
INCLUDE_PROMPT: tl.constexpr,
|
||||
DEDUPLICATE: tl.constexpr,
|
||||
SAMPLE: tl.constexpr,
|
||||
PREPARE_NEXT: tl.constexpr,
|
||||
BLOCKS_POW2: tl.constexpr,
|
||||
):
|
||||
"""Finish draft step `step` (if SAMPLE) and build the next step's skip mask."""
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
contexts_row_ptr = contexts_ptr + row * context_stride
|
||||
if SAMPLE:
|
||||
# Same pick as local_max.argmax(dim=-1): the first NaN, else the first max.
|
||||
blocks = tl.arange(0, BLOCKS_POW2)
|
||||
values = tl.load(
|
||||
local_max_ptr + row * num_blocks + blocks,
|
||||
mask=blocks < num_blocks,
|
||||
other=float("-inf"),
|
||||
)
|
||||
is_nan = values != values
|
||||
best = tl.max(tl.where(is_nan, float("-inf"), values), axis=0)
|
||||
first_nan = tl.min(tl.where(is_nan, blocks, BLOCKS_POW2), axis=0)
|
||||
first_max = tl.min(
|
||||
tl.where((values == best) & (blocks < num_blocks), blocks, BLOCKS_POW2),
|
||||
axis=0,
|
||||
)
|
||||
block = tl.where(first_nan < BLOCKS_POW2, first_nan, first_max)
|
||||
token = tl.load(local_argmax_ptr + row * num_blocks + block)
|
||||
tl.store(sampled_ptr + row * sampled_stride + step, token)
|
||||
|
||||
offsets = tl.arange(0, BLOCKS_POW2)
|
||||
context = tl.load(
|
||||
contexts_row_ptr + offsets + 1,
|
||||
mask=offsets < CONTEXT_WIDTH - 1,
|
||||
other=token,
|
||||
)
|
||||
tl.debug_barrier()
|
||||
tl.store(contexts_row_ptr + offsets, context, mask=offsets < CONTEXT_WIDTH)
|
||||
tl.debug_barrier()
|
||||
|
||||
if PREPARE_NEXT:
|
||||
req_idx = tl.load(next_req_indices_ptr + row).to(tl.int64)
|
||||
temperature = tl.load(temp_ptr + req_idx, mask=req_idx >= 0, other=0.0)
|
||||
active = tl.load(enabled_ptr + row) & (temperature != 0)
|
||||
if DEDUPLICATE:
|
||||
# The dedup kernel body runs in place on the skip mask buffer.
|
||||
tl.store(skip_mask_ptr + row, active)
|
||||
tl.debug_barrier()
|
||||
_repeated_context_mask_kernel(
|
||||
skip_mask_ptr,
|
||||
all_token_ids_ptr,
|
||||
all_token_ids_stride,
|
||||
next_req_indices_ptr,
|
||||
prompt_lens_ptr,
|
||||
total_lens_ptr,
|
||||
next_steps_ptr,
|
||||
1,
|
||||
contexts_ptr,
|
||||
context_stride,
|
||||
None,
|
||||
prior_contexts_ptr,
|
||||
prior_contexts_stride_0,
|
||||
prior_contexts_stride_1,
|
||||
next_steps_ptr,
|
||||
1,
|
||||
skip_mask_ptr,
|
||||
CONTEXT_WIDTH=CONTEXT_WIDTH,
|
||||
NUM_SPECULATIVE_STEPS=NUM_SPECULATIVE_STEPS,
|
||||
MAX_HISTORY=MAX_HISTORY,
|
||||
INCLUDE_PROMPT=INCLUDE_PROMPT,
|
||||
SKIP_PARTIAL_CONTEXT=False,
|
||||
HAS_HISTORY_OFFSETS=True,
|
||||
SPECULATIVE_CONTEXTS=False,
|
||||
DRAFT_CONTEXTS=True,
|
||||
BLOCK=512,
|
||||
)
|
||||
tl.debug_barrier()
|
||||
active = tl.load(skip_mask_ptr + row)
|
||||
tl.store(skip_mask_ptr + row, active == 0)
|
||||
|
||||
|
||||
def draft_philox_gumbel_sample(
|
||||
logits: torch.Tensor,
|
||||
contexts: torch.Tensor,
|
||||
key: int,
|
||||
*,
|
||||
num_steps: int,
|
||||
expanded_idx_mapping: torch.Tensor,
|
||||
temperatures: torch.Tensor,
|
||||
seeds: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
enabled: torch.Tensor,
|
||||
logits_cache: torch.Tensor,
|
||||
logits_cache_col: torch.Tensor,
|
||||
use_fp64: bool,
|
||||
prior_contexts: torch.Tensor,
|
||||
all_token_ids: torch.Tensor | None,
|
||||
prompt_lens: torch.Tensor,
|
||||
total_lens: torch.Tensor | None,
|
||||
deduplicate: bool,
|
||||
max_history: int | None,
|
||||
include_prompt: bool,
|
||||
) -> torch.Tensor:
|
||||
"""Sample `num_steps` draft steps flattened as (request, step), in step order.
|
||||
|
||||
Bit-exact with sampling each step by `DraftWatermarker.sample`, but with two
|
||||
kernel launches per step: the Philox block argmax (with the temperature
|
||||
divide and the logits cache write), then a per-row kernel that picks the
|
||||
token, shifts `contexts` and builds the next step's skip mask. Padding rows
|
||||
(index -1) read temperature 0 and are never watermarked, so they may differ.
|
||||
"""
|
||||
num_rows, vocab_size = logits.shape
|
||||
num_reqs = num_rows // num_steps
|
||||
assert logits.stride(-1) == 1 and contexts.stride(-1) == 1
|
||||
assert logits.dtype == logits_cache.dtype, (
|
||||
"logits cache source and destination must have the same dtype"
|
||||
)
|
||||
assert logits_cache.size(-1) >= vocab_size, (
|
||||
f"draft logits cache vocab dim ({logits_cache.size(-1)}) is narrower "
|
||||
f"than the sampled logits ({vocab_size}). Cached logits would be truncated."
|
||||
)
|
||||
assert not deduplicate or (all_token_ids is not None and total_lens is not None), (
|
||||
"context deduplication requires all_token_ids and total_lens"
|
||||
)
|
||||
logits = logits.view(num_reqs, num_steps, vocab_size)
|
||||
# Step-major copies, so each step reads contiguous per-request slices.
|
||||
expanded_idx_mapping, positions, logits_cache_col = (
|
||||
tensor.view(num_reqs, num_steps).t().contiguous()
|
||||
for tensor in (expanded_idx_mapping, positions, logits_cache_col)
|
||||
)
|
||||
block_size = 1024
|
||||
num_blocks = triton.cdiv(vocab_size, block_size)
|
||||
local_argmax = logits.new_empty(num_reqs, num_blocks, dtype=torch.int64)
|
||||
local_max = logits.new_empty(
|
||||
num_reqs, num_blocks, dtype=torch.float64 if use_fp64 else torch.float32
|
||||
)
|
||||
skip_mask = torch.empty(num_reqs, dtype=torch.bool, device=logits.device)
|
||||
sampled = logits.new_empty(num_reqs, num_steps, dtype=torch.int64)
|
||||
|
||||
def finish_step(step: int) -> None:
|
||||
next_step = min(step + 1, num_steps - 1)
|
||||
_draft_step_kernel[(num_reqs,)](
|
||||
sampled,
|
||||
sampled.stride(0),
|
||||
step,
|
||||
local_argmax,
|
||||
local_max,
|
||||
num_blocks,
|
||||
contexts,
|
||||
contexts.stride(0),
|
||||
skip_mask,
|
||||
expanded_idx_mapping[next_step],
|
||||
logits_cache_col[next_step],
|
||||
temperatures,
|
||||
enabled,
|
||||
all_token_ids,
|
||||
0 if all_token_ids is None else all_token_ids.stride(0),
|
||||
prompt_lens,
|
||||
total_lens,
|
||||
prior_contexts,
|
||||
prior_contexts.stride(0),
|
||||
prior_contexts.stride(1),
|
||||
CONTEXT_WIDTH=contexts.shape[-1],
|
||||
NUM_SPECULATIVE_STEPS=prior_contexts.shape[1],
|
||||
MAX_HISTORY=0 if max_history is None else max_history,
|
||||
INCLUDE_PROMPT=include_prompt,
|
||||
DEDUPLICATE=deduplicate,
|
||||
SAMPLE=step >= 0,
|
||||
PREPARE_NEXT=step + 1 < num_steps,
|
||||
BLOCKS_POW2=triton.next_power_of_2(max(num_blocks, contexts.shape[-1])),
|
||||
)
|
||||
|
||||
finish_step(-1)
|
||||
for step in range(num_steps):
|
||||
_philox_gumbel_kernel[(num_reqs, num_blocks)](
|
||||
local_argmax,
|
||||
local_argmax.stride(0),
|
||||
local_max,
|
||||
local_max.stride(0),
|
||||
logits[:, step],
|
||||
logits.stride(0),
|
||||
contexts,
|
||||
contexts.stride(0),
|
||||
skip_mask,
|
||||
expanded_idx_mapping[step],
|
||||
seeds,
|
||||
positions[step],
|
||||
temperatures,
|
||||
logits_cache,
|
||||
logits_cache.stride(0),
|
||||
logits_cache.stride(1),
|
||||
logits_cache_col[step],
|
||||
logits[:, step],
|
||||
logits.stride(0),
|
||||
key & _UINT32_MASK_VALUE,
|
||||
key >> 32,
|
||||
vocab_size,
|
||||
CONTEXT_WIDTH=contexts.shape[-1],
|
||||
BLOCK_SIZE=block_size,
|
||||
USE_FP64=use_fp64,
|
||||
IS_DRAFTING=True,
|
||||
PER_TOKEN_COL=True,
|
||||
APPLY_TEMPERATURE=True,
|
||||
)
|
||||
finish_step(step)
|
||||
return sampled.view(-1)
|
||||
|
||||
@@ -366,6 +366,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
self.seeds,
|
||||
self.sample_col[:num_sample],
|
||||
self.draft_logits,
|
||||
num_steps=self.num_speculative_steps,
|
||||
)
|
||||
self.draft_tokens[:num_reqs] = draft_tokens.view(
|
||||
num_reqs, self.num_speculative_steps
|
||||
|
||||
@@ -8,7 +8,7 @@ import torch
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_noised_argmax
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_noised_argmax, spec_step_seed
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator
|
||||
|
||||
|
||||
@@ -58,7 +58,7 @@ def _selector_walk_kernel(
|
||||
scores,
|
||||
candidates,
|
||||
mask & valid,
|
||||
seed,
|
||||
spec_step_seed(seed, step),
|
||||
sample_pos,
|
||||
temperature if SAMPLE_PROBABILISTIC else 0.0,
|
||||
IS_DRAFTING=True,
|
||||
|
||||
@@ -6,6 +6,9 @@ import torch.nn as nn
|
||||
from vllm.config import VllmConfig, replace
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.lora.layers.base import BaseLayerWithLoRA
|
||||
from vllm.model_executor.layers.quantization.online.lm_head import (
|
||||
quantized_lm_head_copy,
|
||||
)
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
from vllm.model_executor.model_loader.utils import get_draft_load_config
|
||||
from vllm.model_executor.models.utils import PPMissingLayer
|
||||
@@ -106,9 +109,14 @@ def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mod
|
||||
if target_lm_head is not None and _should_share(
|
||||
eagle_model, "has_own_lm_head", draft_lm_head, target_lm_head
|
||||
):
|
||||
draft_head = target_lm_head
|
||||
if speculative_config.draft_lm_head_quantization is not None:
|
||||
draft_head = quantized_lm_head_copy(
|
||||
target_lm_head, speculative_config.draft_lm_head_quantization
|
||||
)
|
||||
if draft_lm_head is not None:
|
||||
del eagle_model.lm_head
|
||||
eagle_model.lm_head = target_lm_head
|
||||
eagle_model.lm_head = draft_head
|
||||
|
||||
# MTP layers route logits through layer.shared_head.head, not
|
||||
# eagle_model.lm_head, so the per-layer copies need fixing up too.
|
||||
@@ -119,7 +127,11 @@ def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mod
|
||||
sh = getattr(layer, "shared_head", None)
|
||||
if sh is not None and hasattr(sh, "head"):
|
||||
del sh.head
|
||||
sh.head = target_lm_head
|
||||
sh.head = draft_head
|
||||
elif speculative_config.draft_lm_head_quantization is not None:
|
||||
raise ValueError(
|
||||
"draft_lm_head_quantization needs a drafter that shares the target lm_head."
|
||||
)
|
||||
|
||||
# MTP shares topk_indices_buffer with the target model. We update
|
||||
# every module in the draft that holds a buffer reference so that
|
||||
|
||||
@@ -3,7 +3,11 @@
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, tldevice, triton
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_block_argmax, tl_rand32
|
||||
from vllm.v1.worker.gpu.sample.gumbel import (
|
||||
gumbel_block_argmax,
|
||||
spec_step_seed,
|
||||
tl_rand32,
|
||||
)
|
||||
from vllm.v1.worker.gpu.sample.watermark import philox_gumbel_block_argmax
|
||||
|
||||
|
||||
@@ -569,7 +573,8 @@ def _rejection_kernel(
|
||||
verifying &= is_valid_draft
|
||||
if verifying:
|
||||
pos = tl.load(pos_ptr + logit_idx)
|
||||
u = tl_rand32(seed, pos, includes_zero=False)
|
||||
step_seed = spec_step_seed(seed, i) if USE_BLOCK_VERIFICATION else seed
|
||||
u = tl_rand32(step_seed, pos, includes_zero=False)
|
||||
if is_greedy:
|
||||
# Greedy sampling. Accept IFF draft matches target argmax.
|
||||
# NOTE: Target argmax is stored directly so that resampling
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import replace
|
||||
from functools import partial
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import numpy as np
|
||||
@@ -446,6 +447,7 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
draft_step: torch.Tensor,
|
||||
draft_logits: torch.Tensor | None,
|
||||
spec_step_idx: int = 0,
|
||||
num_steps: int = 1,
|
||||
) -> torch.Tensor:
|
||||
if draft_logits is None and self.use_local_argmax_reduction:
|
||||
return self.get_draft_top_tokens(hidden_states, spec_step_idx)
|
||||
@@ -455,7 +457,7 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
sampler = (
|
||||
gumbel_sample
|
||||
if self.draft_watermarker is None
|
||||
else self.draft_watermarker.sample
|
||||
else partial(self.draft_watermarker.sample, num_steps=num_steps)
|
||||
)
|
||||
sampled = sampler(
|
||||
logits,
|
||||
|
||||
@@ -62,6 +62,10 @@ from vllm.distributed.weight_transfer import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.model_executor.layers.quantization.online.lm_head import (
|
||||
restore_quantized_lm_heads,
|
||||
share_source_lm_heads,
|
||||
)
|
||||
from vllm.model_executor.model_loader import get_model_loader
|
||||
from vllm.model_executor.model_loader.weight_checksum import (
|
||||
compute_tensor_digests,
|
||||
@@ -1544,12 +1548,19 @@ class Worker(WorkerBase):
|
||||
"active. Call finish_weight_update first."
|
||||
)
|
||||
|
||||
draft_model = self.get_draft_model() if is_draft else None
|
||||
try:
|
||||
if is_draft:
|
||||
self._set_draft_weight_update_target()
|
||||
if draft_model is not None:
|
||||
# Load through the unquantized target head, as without
|
||||
# draft_lm_head_quantization; finish re-derives the copies.
|
||||
share_source_lm_heads(draft_model)
|
||||
self.weight_transfer_engine.start_weight_update()
|
||||
except BaseException:
|
||||
self.weight_transfer_engine.reset_weight_update_target()
|
||||
if draft_model is not None:
|
||||
restore_quantized_lm_heads(draft_model, refresh=False)
|
||||
raise
|
||||
self._weight_update_active = True
|
||||
self._weight_update_is_draft = is_draft
|
||||
@@ -1589,6 +1600,10 @@ class Worker(WorkerBase):
|
||||
except BaseException:
|
||||
self._weight_update_active = False
|
||||
self.weight_transfer_engine.reset_weight_update_target()
|
||||
if self._weight_update_is_draft and (
|
||||
(draft_model := self.get_draft_model()) is not None
|
||||
):
|
||||
restore_quantized_lm_heads(draft_model, refresh=False)
|
||||
raise
|
||||
|
||||
def finish_weight_update(self) -> None:
|
||||
@@ -1605,6 +1620,10 @@ class Worker(WorkerBase):
|
||||
self.weight_transfer_engine.finish_weight_update()
|
||||
self.weight_transfer_engine.reset_weight_update_target()
|
||||
self._weight_update_active = False
|
||||
if self._weight_update_is_draft and (
|
||||
(draft_model := self.get_draft_model()) is not None
|
||||
):
|
||||
restore_quantized_lm_heads(draft_model)
|
||||
|
||||
# Weight transfer bypasses GPUModelRunner.reload_weights().
|
||||
if not self._weight_update_is_draft:
|
||||
|
||||
Reference in New Issue
Block a user