Merge branch 'main' into patches

This commit is contained in:
2026-10-17 08:06:56 +00:00
222 changed files with 14823 additions and 3606 deletions
+19 -2
View File
@@ -6,7 +6,7 @@ from typing import Any
import torch
from vllm.config import VllmConfig
from vllm.config import VllmConfig, get_current_vllm_config_or_none
from vllm.model_executor.layers.mamba.checkpoint import (
MambaPrefillCheckpointBuilder,
MambaPrefillCheckpointMetadata,
@@ -15,12 +15,13 @@ from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backend import (
AttentionBackend,
CommonAttentionMetadata,
max_decode_query_len,
)
from vllm.v1.attention.backends.mamba_attn import (
BaseMambaAttentionMetadata,
BaseMambaAttentionMetadataBuilder,
)
from vllm.v1.kv_cache_interface import MambaSpec
from vllm.v1.kv_cache_interface import KVCacheSpec, MambaSpec
def compute_varlen_chunk_metadata(
@@ -104,6 +105,14 @@ class Mamba2AttentionBackend(AttentionBackend):
def is_ssm(cls) -> bool:
return True
@classmethod
def supports_device_cpu_query_lens_mismatch(cls) -> bool:
# Decode kernels take device offsets; ReplaySSM plans off CPU counts.
vllm_config = get_current_vllm_config_or_none()
if vllm_config is None:
return True
return not vllm_config.cache_config.use_replayssm
@dataclass
class Mamba2AttentionMetadata(BaseMambaAttentionMetadata):
@@ -124,6 +133,14 @@ class Mamba2AttentionMetadataBuilder(
):
metadata_cls = Mamba2AttentionMetadata
@classmethod
def get_varlen_cudagraph_max_query_len(
cls,
vllm_config: VllmConfig,
kv_cache_spec: KVCacheSpec,
) -> int | None:
return max_decode_query_len(vllm_config)
def __init__(
self,
kv_cache_spec: MambaSpec,
@@ -22,6 +22,17 @@ if TYPE_CHECKING:
from vllm.platforms.interface import DeviceCapability
def is_cute_dsl_mla_prefill_supported(
qk_head_dim: int, v_head_dim: int, query_dtype: torch.dtype
) -> bool:
# FlashInfer has no public compatibility API; replace these checks when available.
return (
query_dtype in (torch.float16, torch.bfloat16, torch.float8_e4m3fn)
and (qk_head_dim, v_head_dim) in ((128, 128), (192, 128))
and (qk_head_dim != 192 or query_dtype == torch.float8_e4m3fn)
)
class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
"""TRT-LLM Ragged backend for MLA prefill."""
@@ -79,6 +90,7 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
v_head_dim=v_head_dim,
vllm_config=vllm_config,
)
self._context_backend = vllm_config.attention_config.trtllm_ragged_backend
self._workspace_spec = (
(envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE,),
torch.uint8,
@@ -198,6 +210,16 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
"TRTLLM ragged context prefill contains an empty query or KV row"
)
if self._context_backend == "cute-dsl":
if not is_cute_dsl_mla_prefill_supported(q.shape[-1], v.shape[-1], q.dtype):
raise ValueError(
"CuTe DSL context prefill does not support "
f"Q/K head dimension {q.shape[-1]}, "
f"V head dimension {v.shape[-1]}, and query dtype {q.dtype}."
)
# Generic MLA supplies V as a projection slice; CuTe requires packed V.
v = v.contiguous()
if out is None:
out = torch.empty(
q.shape[0],
@@ -225,6 +247,7 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
cum_seq_lens_kv=chunk.cu_seq_lens,
enable_pdl=False,
is_causal=False,
backend=self._context_backend,
return_lse=True,
out=out,
skip_all_rows_active_check=True,
+145 -10
View File
@@ -35,6 +35,7 @@ from vllm.v1.attention.backend import (
AttentionLayer,
CommonAttentionMetadata,
MultipleOf,
max_decode_query_len,
)
from vllm.v1.attention.backends.mla.triton_mla import _compute_num_kv_splits
from vllm.v1.attention.backends.utils import (
@@ -49,7 +50,11 @@ from vllm.v1.attention.ops.rocm_aiter_mla_prefill import (
gather_compressed_context,
)
from vllm.v1.attention.ops.triton_decode_attention import decode_attention_fwd
from vllm.v1.kv_cache_interface import AttentionSpec, is_quantized_kv_cache
from vllm.v1.kv_cache_interface import (
AttentionSpec,
KVCacheSpec,
is_quantized_kv_cache,
)
from vllm.v1.worker.workspace import (
current_workspace_manager,
is_workspace_manager_initialized,
@@ -457,6 +462,16 @@ def _aiter_mla_small_head_mode() -> str:
return mode
def _adaptive_varlen_verify_enabled(vllm_config: VllmConfig) -> bool:
speculative_config = vllm_config.speculative_config
return (
speculative_config is not None
and bool(getattr(speculative_config, "enable_adaptive_verification", False))
and vllm_config.parallel_config.decode_context_parallel_size == 1
and max_decode_query_len(vllm_config) > 1
)
class AiterMLABackend(MLACommonBackend):
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16]
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
@@ -620,6 +635,8 @@ class AiterMLADecodeMetadata(MLACommonDecodeMetadata):
use_gluon_verify: bool = False
# Whether persistent MLA metadata was computed
has_persistent_metadata: bool = False
varlen_q_rows: torch.Tensor | None = None
varlen_o_rows: torch.Tensor | None = None
@dataclass
@@ -720,6 +737,21 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
return uniform_qo_len
@classmethod
def get_varlen_cudagraph_max_query_len(
cls: type["AiterMLAMetadataBuilder"],
vllm_config: VllmConfig,
kv_cache_spec: KVCacheSpec,
) -> int | None:
if not _adaptive_varlen_verify_enabled(vllm_config):
return None
if (
cls.get_cudagraph_support(vllm_config, kv_cache_spec)
!= AttentionCGSupport.UNIFORM_BATCH
):
return None
return max_decode_query_len(vllm_config)
def __init__(
self,
kv_cache_spec: AttentionSpec,
@@ -823,6 +855,26 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
# persistent gate below and makes aiter raise a KeyError mid-run.
self._mtp_decode_qlen = self.reorder_batch_threshold or 1
self._varlen_decode_qlen = 0
if _adaptive_varlen_verify_enabled(vllm_config):
assert self._mtp_decode_qlen == max_decode_query_len(vllm_config), (
self._mtp_decode_qlen,
max_decode_query_len(vllm_config),
)
self._varlen_decode_qlen = self._mtp_decode_qlen
self.query_len_support = QueryLenSupport.VARLEN # type: ignore[misc]
max_rows = vllm_config.scheduler_config.max_num_seqs * (
self._varlen_decode_qlen
)
row_ids = torch.arange(max_rows, dtype=torch.int64, device=device)
self._varlen_row_req = row_ids // self._varlen_decode_qlen
self._varlen_row_pos = row_ids % self._varlen_decode_qlen
self._varlen_row_ids = row_ids
self._varlen_q_rows = torch.zeros_like(row_ids)
self._varlen_o_rows = torch.zeros(
max_rows + 1, dtype=torch.int64, device=device
)
# Store the kernel block size from the spec. When kernel_block_size=1
# (no spec-dec), behavior is identical to the original. When > 1
# (e.g. 16 with Eagle3), we expand block-level indices into per-token
@@ -850,6 +902,12 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
max_num_pages_per_req = vllm_config.model_config.max_model_len
max_num_reqs = vllm_config.scheduler_config.max_num_seqs
max_num_pages = max_num_reqs * max_num_pages_per_req
speculative_config = vllm_config.speculative_config
if _adaptive_varlen_verify_enabled(vllm_config):
assert speculative_config is not None
max_num_pages += max_num_reqs * (
speculative_config.num_speculative_tokens or 0
)
# Preparing persistent buffers
# TODO: we can disambiguate between decode and mixed-prefill decode here
@@ -1437,13 +1495,18 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
device = self.device
num_reqs = seq_lens_device.size(0)
qo_len = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1]
max_qo_len = qo_len.max().item()
padded_mtp_qo_len = self._uniform_padded_mtp_qo_len(
qo_len, max_qo_len, num_decode_tokens
)
if padded_mtp_qo_len > 0:
max_qo_len = padded_mtp_qo_len
pad_uniform_mtp = padded_mtp_qo_len > 0
varlen_decode = self._varlen_decode_qlen > 0 and causal
if varlen_decode:
max_qo_len = self._varlen_decode_qlen
pad_uniform_mtp = True
else:
max_qo_len = qo_len.max().item()
padded_mtp_qo_len = self._uniform_padded_mtp_qo_len(
qo_len, max_qo_len, num_decode_tokens
)
if padded_mtp_qo_len > 0:
max_qo_len = padded_mtp_qo_len
pad_uniform_mtp = padded_mtp_qo_len > 0
seq_lens_for_kernel = seq_lens_device
# the global lengths need the same dummy-row
@@ -1477,6 +1540,13 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
g_tot_seq_lens.new_full((), max_qo_len * self.dcp_world_size),
)
varlen_q_rows = varlen_o_rows = None
if varlen_decode:
seq_lens_for_kernel = seq_lens_device + (max_qo_len - qo_lens_device)
varlen_q_rows, varlen_o_rows = self._fill_varlen_rows(
query_start_loc_device, qo_lens_device, num_reqs
)
if self._graph_seq_lens is not None:
self._graph_seq_lens[:num_kernel_reqs].copy_(
seq_lens_for_kernel, non_blocking=True
@@ -1537,6 +1607,8 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
# device synchronisation. Programs whose chunk lies past
# num_tokens mask out entirely.
max_tokens_per_req = block_table_tensor.shape[1] * self.kernel_block_size
if varlen_decode:
max_tokens_per_req += max_qo_len - 1
num_chunks = max(1, cdiv(max_tokens_per_req, 1024))
_expand_page_indices_kernel[(num_reqs, num_chunks)](
self.paged_kv_indices,
@@ -1544,8 +1616,10 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
block_table_tensor.stride(0),
block_table_tensor.stride(1),
paged_kv_indptr,
seq_lens_device,
KERNEL_BLOCK_SIZE=self.kernel_block_size,
BLOCK_SIZE=1024,
CLAMP_TAIL=varlen_decode,
)
paged_kv_indices = self.paged_kv_indices
@@ -1740,10 +1814,38 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
use_gluon_verify=use_gluon_verify,
attn_out_dtype=self.decode_attn_out_dtype,
has_persistent_metadata=has_persistent_metadata,
varlen_q_rows=varlen_q_rows,
varlen_o_rows=varlen_o_rows,
)
return attn_metadata
def _fill_varlen_rows(
self,
query_start_loc_device: torch.Tensor,
qo_lens_device: torch.Tensor,
num_reqs: int,
) -> tuple[torch.Tensor, torch.Tensor]:
width = self._varlen_decode_qlen
num_rows = num_reqs * width
row_req = self._varlen_row_req[:num_rows]
row_pos = self._varlen_row_pos[:num_rows]
row_qlen = qo_lens_device.to(torch.int64).index_select(0, row_req)
row_start = query_start_loc_device.to(torch.int64).index_select(0, row_req)
is_real = row_pos < row_qlen
packed_row = row_start + row_pos
q_rows = self._varlen_q_rows[:num_rows]
q_rows.copy_(torch.where(is_real, packed_row, 0))
o_rows = self._varlen_o_rows
o_rows.zero_()
dump_slot = o_rows.size(0) - 1
o_rows.scatter_(
0,
torch.where(is_real, packed_row, dump_slot),
self._varlen_row_ids[:num_rows],
)
return q_rows, o_rows
def build(
self,
common_prefix_len: int,
@@ -1795,8 +1897,10 @@ def _expand_page_indices_kernel(
block_table_stride_0,
block_table_stride_1,
cu_num_tokens,
num_real_tokens,
KERNEL_BLOCK_SIZE: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
CLAMP_TAIL: tl.constexpr = False,
):
"""Expand block table entries into per-token flat page indices.
@@ -1810,6 +1914,10 @@ def _expand_page_indices_kernel(
When KERNEL_BLOCK_SIZE=K: block table entry b (covering K tokens)
is expanded to flat indices b*K, b*K+1, ..., b*K+(K-1).
With CLAMP_TAIL, entries past num_real_tokens[req] (padding a request's
page list out to its kernel length) repeat the request's first page, so
they never index past its block-table row.
"""
# One program per (request, token-chunk). Parallelising over requests
# alone degenerates at low concurrency: with num_reqs == 1 a single
@@ -1819,6 +1927,8 @@ def _expand_page_indices_kernel(
row_ptr = block_table + req_idx * block_table_stride_0
start_idx = tl.load(cu_num_tokens + req_idx)
num_tokens = tl.load(cu_num_tokens + req_idx + 1) - start_idx
if CLAMP_TAIL:
real_tokens = tl.load(num_real_tokens + req_idx)
# The grid is sized from the block table width, an upper bound over all
# requests, so a ragged batch launches chunks past a short request's end.
@@ -1830,11 +1940,14 @@ def _expand_page_indices_kernel(
token_offsets = chunk_start + tl.arange(0, BLOCK_SIZE)
mask = token_offsets < num_tokens
source_offsets = token_offsets
if CLAMP_TAIL:
source_offsets = tl.where(token_offsets < real_tokens, token_offsets, 0)
# Which block in the block table does this token belong to?
block_idx = token_offsets // KERNEL_BLOCK_SIZE
block_idx = source_offsets // KERNEL_BLOCK_SIZE
# Offset within that block
offset_in_block = token_offsets % KERNEL_BLOCK_SIZE
offset_in_block = source_offsets % KERNEL_BLOCK_SIZE
# Load the block ID from the block table
# Both strides are taken from the caller: the block table is a view owned
@@ -2600,6 +2713,28 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
kv_c_and_k_pe_cache: torch.Tensor,
attn_metadata: AiterMLAMetadata,
layer: AttentionLayer,
) -> tuple[torch.Tensor, torch.Tensor | None]:
decode = attn_metadata.decode
if decode is None or decode.varlen_q_rows is None:
return self._forward_mqa(q, kv_c_and_k_pe_cache, attn_metadata, layer)
assert decode.varlen_o_rows is not None
q_rows = decode.varlen_q_rows
if isinstance(q, tuple):
num_tokens = q[0].shape[0]
q = (q[0].index_select(0, q_rows), q[1].index_select(0, q_rows))
else:
num_tokens = q.shape[0]
q = q.index_select(0, q_rows)
o, lse = self._forward_mqa(q, kv_c_and_k_pe_cache, attn_metadata, layer)
assert lse is None
return o.index_select(0, decode.varlen_o_rows[:num_tokens]), None
def _forward_mqa(
self,
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
kv_c_and_k_pe_cache: torch.Tensor,
attn_metadata: AiterMLAMetadata,
layer: AttentionLayer,
) -> tuple[torch.Tensor, torch.Tensor | None]:
assert kv_c_and_k_pe_cache.numel() > 0
assert attn_metadata.decode is not None
+103
View File
@@ -0,0 +1,103 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Interface and resolution for pluggable KV cache config builders."""
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING
from vllm.utils.import_utils import resolve_obj_by_qualname
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
KVCacheSpec,
)
class KVCacheConfigBuilder(ABC):
"""Interface for platform KV cache placement.
.. note::
This interface is experimental and subject to change. The exact hooks
and the ``num_blocks`` contract may evolve before it stabilizes.
Subclasses normally inherit from ``DefaultKVCacheConfigBuilder`` and
override only the hooks they need. The cross-worker planning flow
(:func:`vllm.v1.core.kv_cache_planning.get_kv_cache_configs`) is Core-owned;
this builder supplies the placement decisions: how layers are split into
groups and how a group list is materialized into tensors for a given block
count. Grouping coordination lives in ``DefaultKVCacheConfigBuilder`` (a
model implementing :class:`~vllm.model_executor.models.interfaces.
SupportsKVCacheGrouping` owns its grouping there); a platform that replaces
the builder fully owns its own grouping.
"""
@abstractmethod
def get_kv_cache_groups(
self,
vllm_config: "VllmConfig",
kv_cache_spec: dict[str, "KVCacheSpec"],
) -> list["KVCacheGroupSpec"]:
"""Split the given layers into KV cache groups.
Planning passes all workers' layers; profiling passes one worker's.
Core rejects a result that does not put each layer in exactly one group.
The default builder honors a model's
:class:`~vllm.model_executor.models.interfaces.SupportsKVCacheGrouping`
declaration here before falling back to the default grouping; a
platform-specific builder decides its own grouping.
"""
raise NotImplementedError
@abstractmethod
def get_kv_cache_config_from_groups(
self,
vllm_config: "VllmConfig",
kv_cache_groups: list["KVCacheGroupSpec"],
num_blocks: int,
) -> "KVCacheConfig":
"""Materialize the groups for exactly ``num_blocks`` blocks.
Each device backing's size is ``bytes_per_block * num_blocks``. All
device tensors share one backing allocation unless the builder
overrides ``_get_pool_bytes_per_block``. Host-resident tensors, such as
HiSparse source storage, are excluded from the device byte accounting
and sized independently.
"""
raise NotImplementedError
def _get_pool_bytes_per_block(
self,
vllm_config: "VllmConfig",
kv_cache_groups: list["KVCacheGroupSpec"],
) -> int | None:
"""Return the device bytes one block ID consumes, or ``None``.
Private and transitional; expected to go away with worker-driven
allocation (RFC #42449). With ``None`` (the default), Core derives the
value from a one-block placement with one device backing. Override it
only for placements with several device allocations.
"""
return None
def get_kv_cache_config_builder(
vllm_config: "VllmConfig",
) -> KVCacheConfigBuilder:
"""Resolve the builder selected by the current platform.
Resolution priority is owned by the platform hook
(:meth:`vllm.platforms.interface.Platform.get_kv_cache_config_builder_cls`).
Builders are stateless (every method takes ``vllm_config``), so a fresh
instance is returned per call and engines with different configs in the
same process always get the correct builder.
"""
from vllm.platforms import current_platform
builder_cls = resolve_obj_by_qualname(
current_platform.get_kv_cache_config_builder_cls(vllm_config)
)
return builder_cls()
+190 -24
View File
@@ -14,6 +14,7 @@ from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv, round_up
from vllm.utils.mem_utils import format_gib
from vllm.utils.torch_utils import get_dtype_size
from vllm.v1.core.kv_cache_config_builder import KVCacheConfigBuilder
from vllm.v1.core.kv_cache_utils import _kv_cache_device_name
from vllm.v1.hisparse.layout import (
get_hisparse_host_pool_bytes,
@@ -261,6 +262,48 @@ def _pool_bytes_per_block(
)
def _pool_bytes_per_block_from_builder(
builder: KVCacheConfigBuilder,
vllm_config: VllmConfig,
kv_cache_groups: list[KVCacheGroupSpec],
) -> int:
"""Bytes consumed by one block in the worker's device KV cache pool.
Core places one block with the builder and reads the size of its single
device backing. A placement with several device allocations declares the
value through ``KVCacheConfigBuilder._get_pool_bytes_per_block``. HiSparse
keeps the pure computation, since probing its placement would check the host
pool against ``max_model_len`` before auto-fit shortens it.
``num_gpu_blocks_override`` is suppressed so it cannot inflate the probe.
"""
if not kv_cache_groups:
return 0
if vllm_config.attention_config.hisparse_config is not None:
return _pool_bytes_per_block(vllm_config, kv_cache_groups)
pool_bytes = builder._get_pool_bytes_per_block(vllm_config, kv_cache_groups)
if pool_bytes is not None:
return pool_bytes
saved_override = vllm_config.cache_config.num_gpu_blocks_override
vllm_config.cache_config.num_gpu_blocks_override = None
try:
probe = builder.get_kv_cache_config_from_groups(
vllm_config, kv_cache_groups, num_blocks=1
)
finally:
vllm_config.cache_config.num_gpu_blocks_override = saved_override
backing_sizes = {
tensor.size for tensor in probe.kv_cache_tensors if not tensor.host_resident
}
if len(backing_sizes) != 1:
raise ValueError(
"KV cache tensors must share one device backing allocation, but got "
f"sizes {sorted(backing_sizes)}. Override "
"KVCacheConfigBuilder._get_pool_bytes_per_block() for placements "
"with several device allocations."
)
return backing_sizes.pop()
def get_uniform_page_size(kv_cache_specs: Iterable[KVCacheSpec]) -> int:
"""Get the page size of the KV cache."""
page_sizes = {layer.page_size_bytes for layer in kv_cache_specs}
@@ -1264,6 +1307,36 @@ def get_kv_cache_groups(
return groups
def get_kv_cache_groups_from_builder(
builder: KVCacheConfigBuilder,
vllm_config: VllmConfig,
kv_cache_spec: dict[str, KVCacheSpec],
) -> list[KVCacheGroupSpec]:
"""Group the layers with the builder and check the result.
Core checks every grouping, whether it comes from the default grouping, a
model's ``SupportsKVCacheGrouping`` or a platform builder.
"""
groups = builder.get_kv_cache_groups(vllm_config, kv_cache_spec)
_check_exact_partition(groups, kv_cache_spec)
return groups
def _check_exact_partition(
groups: list[KVCacheGroupSpec],
kv_cache_spec: dict[str, KVCacheSpec],
) -> None:
"""Ensure the groups partition every input layer exactly once."""
grouped_layers: set[str] = set()
for group in groups:
assert not set(group.layer_names) & grouped_layers, (
f"KV cache group overlaps: {set(group.layer_names) & grouped_layers}."
)
grouped_layers |= set(group.layer_names)
missing = set(kv_cache_spec) - grouped_layers
assert not missing, f"KV cache groups miss layers: {sorted(missing)}."
def _layer_tp_replicas(spec: KVCacheSpec, tp_size: int, dcp_size: int) -> int:
if not isinstance(spec, AttentionSpec) or spec.max_tp_shards is None:
return 1
@@ -1285,6 +1358,7 @@ def kv_cache_groups_tp_replicas(
def _max_memory_usage_bytes_from_groups(
builder: KVCacheConfigBuilder,
vllm_config: VllmConfig,
kv_cache_groups: list[KVCacheGroupSpec],
) -> int:
@@ -1301,7 +1375,9 @@ def _max_memory_usage_bytes_from_groups(
if not kv_cache_groups:
return 0
bytes_per_block = _pool_bytes_per_block(vllm_config, kv_cache_groups)
bytes_per_block = _pool_bytes_per_block_from_builder(
builder, vllm_config, kv_cache_groups
)
total_blocks = 0
for group in kv_cache_groups:
spec = group.kv_cache_spec
@@ -1317,6 +1393,7 @@ def _max_memory_usage_bytes_from_groups(
def _estimate_max_model_len_from_groups(
builder: KVCacheConfigBuilder,
vllm_config: VllmConfig,
kv_cache_groups: list[KVCacheGroupSpec],
available_memory: int,
@@ -1334,13 +1411,16 @@ def _estimate_max_model_len_from_groups(
vllm_config.model_config.max_model_len = model_len
if hisparse_enabled:
try:
get_kv_cache_config_from_groups(
vllm_config, kv_cache_groups, available_memory
num_blocks = available_memory // _pool_bytes_per_block_from_builder(
builder, vllm_config, kv_cache_groups
)
builder.get_kv_cache_config_from_groups(
vllm_config, kv_cache_groups, num_blocks
)
except ValueError:
return False
return (
_max_memory_usage_bytes_from_groups(vllm_config, kv_cache_groups)
_max_memory_usage_bytes_from_groups(builder, vllm_config, kv_cache_groups)
<= available_memory
)
@@ -1362,6 +1442,7 @@ def _estimate_max_model_len_from_groups(
def _auto_fit_max_model_len(
builder: KVCacheConfigBuilder,
vllm_config: VllmConfig,
projected_groups_per_worker: list[list[KVCacheGroupSpec]],
available_memory: list[int],
@@ -1372,6 +1453,7 @@ def _auto_fit_max_model_len(
workers.
Args:
builder: The builder whose hooks drive block accounting.
vllm_config: The global VllmConfig (will be modified in-place)
projected_groups_per_worker: KV cache groups projected to each worker.
available_memory: Memory available for KV cache in bytes for each
@@ -1395,7 +1477,9 @@ def _auto_fit_max_model_len(
for groups, avail_mem in zip(projected_groups_per_worker, available_memory):
if not groups:
continue
worker_max = _estimate_max_model_len_from_groups(vllm_config, groups, avail_mem)
worker_max = _estimate_max_model_len_from_groups(
builder, vllm_config, groups, avail_mem
)
if worker_max < auto_fit_max:
auto_fit_max = worker_max
limiting_worker_mem = avail_mem
@@ -1473,6 +1557,9 @@ def _project_kv_cache_groups_to_worker(
worker_layer_names,
group_spec,
is_eagle_group=group.is_eagle_group and bool(worker_layer_names),
host_resident=group.host_resident,
enable_kv_transfer=group.enable_kv_transfer,
role=group.role,
)
)
return projected_groups
@@ -1512,6 +1599,10 @@ def get_kv_cache_configs(
The generated KVCacheConfigs for each worker.
"""
from vllm.v1.core.kv_cache_config_builder import get_kv_cache_config_builder
builder = get_kv_cache_config_builder(vllm_config)
# Merge the KV cache specs of all workers. Different PP stages may have
# different layer names, and different TP ranks of the same PP stage should
# have the same KV cache spec.
@@ -1542,7 +1633,9 @@ def get_kv_cache_configs(
# Get global KV cache groups. This also handles spec unification for
# hybrid models when disable_hybrid_kv_cache_manager is enabled.
# After this call, merged_kv_cache_specs may be modified in-place.
global_kv_cache_groups = get_kv_cache_groups(vllm_config, merged_kv_cache_specs)
global_kv_cache_groups = get_kv_cache_groups_from_builder(
builder, vllm_config, merged_kv_cache_specs
)
# If original_max_model_len was -1, automatically
# determine the maximum model length that fits in available GPU memory.
@@ -1553,11 +1646,10 @@ def get_kv_cache_configs(
]
# If `num_gpu_blocks_override` is set, the cache size that will actually
# be allocated is decoupled from the profiled `available_memory`:
# `may_override_num_blocks` in `get_kv_cache_config_from_groups` clamps
# `num_blocks` to the override. Reflect that in `available_memory` here so
# auto-fit, the admission check, and the per-worker config builder all
# plan against the same effective capacity.
# be allocated is decoupled from the profiled `available_memory`. Reflect
# that in `available_memory` here so auto-fit, the admission check, and
# the per-worker config builder all plan against the same effective
# capacity.
override = vllm_config.cache_config.num_gpu_blocks_override
if override is not None:
adjusted_memory: list[int] = []
@@ -1565,7 +1657,9 @@ def get_kv_cache_configs(
if not groups:
adjusted_memory.append(avail_mem)
continue
bytes_per_block = _pool_bytes_per_block(vllm_config, groups)
bytes_per_block = _pool_bytes_per_block_from_builder(
builder, vllm_config, groups
)
logger.info(
"Overriding num_gpu_blocks=%d with num_gpu_blocks_override=%d",
avail_mem // bytes_per_block,
@@ -1578,12 +1672,18 @@ def get_kv_cache_configs(
# the capacity check both plan against usable blocks. Allocation below
# still uses the full memory.
check_memory = [
avail_mem - _pool_bytes_per_block(vllm_config, groups) if groups else avail_mem
(
avail_mem - _pool_bytes_per_block_from_builder(builder, vllm_config, groups)
if groups
else avail_mem
)
for groups, avail_mem in zip(projected_groups_per_worker, available_memory)
]
if vllm_config.model_config.original_max_model_len == -1:
_auto_fit_max_model_len(vllm_config, projected_groups_per_worker, check_memory)
_auto_fit_max_model_len(
builder, vllm_config, projected_groups_per_worker, check_memory
)
# Check if the available memory is enough per worker.
for groups, avail_mem in zip(projected_groups_per_worker, check_memory):
@@ -1591,9 +1691,9 @@ def get_kv_cache_configs(
continue
_check_enough_kv_cache_memory(
avail_mem,
partial(_max_memory_usage_bytes_from_groups, vllm_config, groups),
partial(_max_memory_usage_bytes_from_groups, builder, vllm_config, groups),
vllm_config.model_config.max_model_len,
partial(_estimate_max_model_len_from_groups, vllm_config, groups),
partial(_estimate_max_model_len_from_groups, builder, vllm_config, groups),
)
kv_cache_configs: list[KVCacheConfig] = []
@@ -1603,9 +1703,15 @@ def get_kv_cache_configs(
assert sum(len(group.layer_names) for group in projected_groups) == len(
kv_cache_spec_one_worker
), "Some layers are not assigned to any group."
bytes_per_block = _pool_bytes_per_block_from_builder(
builder, vllm_config, projected_groups
)
num_blocks = (
available_memory_one_worker // bytes_per_block if projected_groups else 1
)
kv_cache_configs.append(
get_kv_cache_config_from_groups(
vllm_config, projected_groups, available_memory_one_worker
builder.get_kv_cache_config_from_groups(
vllm_config, projected_groups, num_blocks
)
)
@@ -1618,13 +1724,12 @@ def get_kv_cache_configs(
for i, kv_cache_config in enumerate(kv_cache_configs):
if kv_cache_config.num_blocks == min_num_blocks:
continue
# Re-plan with exactly the memory the smallest rank can afford, so
# strides and offsets stay consistent with the shrunken allocation.
groups = kv_cache_config.kv_cache_groups
kv_cache_configs[i] = get_kv_cache_config_from_groups(
# Re-plan with exactly the smallest rank's block count, so strides and
# offsets stay consistent with the shrunken allocation.
kv_cache_configs[i] = builder.get_kv_cache_config_from_groups(
vllm_config,
groups,
min_num_blocks * _pool_bytes_per_block(vllm_config, groups),
kv_cache_config.kv_cache_groups,
min_num_blocks,
)
for kv_cache_config in kv_cache_configs:
@@ -1635,3 +1740,64 @@ def get_kv_cache_configs(
)
return kv_cache_configs
class DefaultKVCacheConfigBuilder(KVCacheConfigBuilder):
"""Default grouping and placement, backed by the module-level functions here.
.. note::
This class is part of the experimental ``KVCacheConfigBuilder``
interface and is subject to change.
Subclasses override only the hooks they need. The planning flow and its
invariants (``num_gpu_blocks_override``, null-block reservation, auto-fit,
admission checks, cross-rank convergence) stay in the Core-owned
:func:`get_kv_cache_configs`.
"""
def get_kv_cache_groups(
self,
vllm_config: VllmConfig,
kv_cache_spec: dict[str, KVCacheSpec],
) -> list[KVCacheGroupSpec]:
"""See ``KVCacheConfigBuilder.get_kv_cache_groups``.
A model implementing ``SupportsKVCacheGrouping`` owns the grouping
decision; otherwise the default grouping is used.
"""
if not vllm_config.model_config.supports_kv_cache_grouping:
return get_kv_cache_groups(vllm_config, kv_cache_spec)
if vllm_config.attention_config.hisparse_config is not None:
raise ValueError(
"A model-declared get_kv_cache_groups cannot be combined "
"with HiSparse, which owns its own KV cache grouping. "
"Remove one of them."
)
model_cls, _ = vllm_config.model_config.registry.resolve_model_cls(
vllm_config.model_config.architectures,
vllm_config.model_config,
)
return model_cls.get_kv_cache_groups(vllm_config, kv_cache_spec)
def get_kv_cache_config_from_groups(
self,
vllm_config: VllmConfig,
kv_cache_groups: list[KVCacheGroupSpec],
num_blocks: int,
) -> KVCacheConfig:
"""See ``KVCacheConfigBuilder.get_kv_cache_config_from_groups``."""
if len(kv_cache_groups) == 0:
# Attention free models do not have KV cache.
# Return num_blocks=1 as BlockPool always needs a null_block.
return KVCacheConfig(
num_blocks=1,
kv_cache_tensors=[],
kv_cache_groups=kv_cache_groups,
prefix_cache_retention_interval=(
vllm_config.cache_config.prefix_cache_retention_interval
),
)
bytes_per_block = _pool_bytes_per_block(vllm_config, kv_cache_groups)
return get_kv_cache_config_from_groups(
vllm_config, kv_cache_groups, num_blocks * bytes_per_block
)
+10 -32
View File
@@ -41,12 +41,9 @@ class DPCoordinator:
DPEngineCoreProc._has_global_unfinished_reqs method.
* Broadcasts the START_DP_WAVE message to engines to move them from paused
to running state when one engine receives a new request. This can happen
in two cases:
1) A front-end sending a new request while the engines are paused will
concurrently notify the coordinator.
2) An engine receiving a request for a stale request wave while in paused
state will notify the coordinator.
to running state when an idle engine receives a new request and notifies
the coordinator. Only the engines start waves: a front-end's view of the
running state can be stale, and a paused engine discards START_DP_WAVE.
Engines will move into running state when receiving a new request or
START_DP_WAVE message.
@@ -353,27 +350,9 @@ class DPCoordinatorProc:
)
continue # Skip normal engine notification processing
# Wave coordination: handle new-request messages from front-end.
# Only process these when wave coordination is enabled
if self.enable_wave_coordination:
# We received a message on the front-end XPUB socket,
# from an API server sending a new request while the
# engines are paused, so that we can wake the other
# engines.
engine_to_exclude, wave = decoded
if not engines_running:
if wave < current_wave:
# If the wave number is stale, ensure the message
# is handled by all the engines.
engine_to_exclude = None
# engines_running is only set from the engines'
# own notifications; a paused engine discards
# START_DP_WAVE, so sending it is not evidence
# that the engines are running.
self._send_start_wave(
publish_back, current_wave, engine_to_exclude
)
logger.error(
"DP Coordinator received unexpected message from front-end"
)
if output_back in events:
# We received a message from one of the engines.
@@ -448,13 +427,12 @@ class DPCoordinatorProc:
wave > current_wave
or (wave == current_wave and not engines_running)
):
# 3. The engine received request for a non-current wave
# so we must ensure that other engines progress to the
# next wave (race condition handling).
# 3. An engine started a wave (e.g. an idle engine
# received a request), so the others must join it.
logger.debug(
"Starting wave %d after notification of "
"stale wave request from engine.",
"Starting wave %d on notification from engine %d.",
wave,
eng_index,
)
current_wave = wave
engines_running = True
+3 -4
View File
@@ -2269,15 +2269,14 @@ class DPEngineCoreProc(EngineCoreProc):
def add_request(self, request: Request, request_wave: int = 0):
super().add_request(request, request_wave)
if self.has_coordinator and request_wave != self.current_wave:
if self.has_coordinator:
if request_wave > self.current_wave:
self.current_wave = request_wave
elif (
if (
not self.engines_running
and self.scheduler.pause_state == PauseState.UNPAUSED
):
# Request received for an already-completed wave, notify
# front-end that we need to start the next one.
# Engines alone start waves: have the coordinator wake the others.
self.engines_running = True
self.output_queue.put_nowait(
(-1, EngineCoreOutputs(start_wave=self.current_wave))
+43 -75
View File
@@ -506,8 +506,8 @@ class BackgroundResources:
coordinator: DPCoordinator | None = None
output_socket: zmq.Socket | zmq.asyncio.Socket | None = None
input_socket: zmq.Socket | zmq.asyncio.Socket | None = None
first_req_send_socket: zmq.asyncio.Socket | None = None
first_req_rcv_socket: zmq.asyncio.Socket | None = None
scale_send_socket: zmq.asyncio.Socket | None = None
scale_rcv_socket: zmq.asyncio.Socket | None = None
stats_update_socket: zmq.asyncio.Socket | None = None
output_queue_task: asyncio.Task | None = None
stats_update_task: asyncio.Task | None = None
@@ -535,8 +535,8 @@ class BackgroundResources:
sockets = (
self.output_socket,
self.input_socket,
self.first_req_send_socket,
self.first_req_rcv_socket,
self.scale_send_socket,
self.scale_rcv_socket,
self.stats_update_socket,
)
@@ -1493,9 +1493,9 @@ class DPAsyncMPClient(AsyncMPClient):
self.eep_scaling_cache: ElasticScalingCache | None = None
self.first_req_sock_addr = get_open_zmq_inproc_path()
self.first_req_send_socket = self.resources.first_req_send_socket = (
make_zmq_socket(self.ctx, self.first_req_sock_addr, zmq.PAIR, bind=True)
self.scale_sock_addr = get_open_zmq_inproc_path()
self.scale_send_socket = self.resources.scale_send_socket = make_zmq_socket(
self.ctx, self.scale_sock_addr, zmq.PAIR, bind=True
)
try:
# If we are running in an asyncio event loop, start the stats task.
@@ -1518,75 +1518,48 @@ class DPAsyncMPClient(AsyncMPClient):
with (
make_zmq_socket(self.ctx, stats_addr, zmq.XSUB, linger=0) as socket,
make_zmq_socket(
self.ctx, self.first_req_sock_addr, zmq.PAIR, bind=False, linger=0
) as first_req_rcv_socket,
self.ctx, self.scale_sock_addr, zmq.PAIR, bind=False, linger=0
) as scale_rcv_socket,
):
assert isinstance(socket, zmq.asyncio.Socket)
assert isinstance(first_req_rcv_socket, zmq.asyncio.Socket)
assert isinstance(scale_rcv_socket, zmq.asyncio.Socket)
self.resources.stats_update_socket = socket
self.resources.first_req_rcv_socket = first_req_rcv_socket
self.resources.scale_rcv_socket = scale_rcv_socket
# Send subscription message.
await socket.send(b"\x01")
poller = zmq.asyncio.Poller()
poller.register(socket, zmq.POLLIN)
poller.register(first_req_rcv_socket, zmq.POLLIN)
poller.register(scale_rcv_socket, zmq.POLLIN)
while True:
events = await poller.poll()
if (
not self.engines_running
and len(events) == 2
or (events[0][0] == first_req_rcv_socket)
):
# Check if this is a regular request notification or
# scale up notification
buf = first_req_rcv_socket.recv(flags=zmq.NOBLOCK).result()
decoded = msgspec.msgpack.decode(buf)
if (
isinstance(decoded, (list, tuple))
and len(decoded) == 2
and decoded[0] == "SCALE_ELASTIC_EP"
):
# Extract new engine count from the decoded message
new_engine_count = decoded[1]
# Update engine_ranks_managed and count_slice
parallel_config = self.vllm_config.parallel_config
dp_size = parallel_config.data_parallel_size
dp_rank = parallel_config.data_parallel_rank
assert dp_rank == 0
assert dp_size == new_engine_count
assert not parallel_config.data_parallel_hybrid_lb
# External LB clients continue to manage only local ranks.
if not parallel_config.data_parallel_external_lb:
self.engine_ranks_managed = list(range(dp_size))
if len(self.lb_engines) < new_engine_count:
self.lb_engines += [
[0, 0, 0.0]
for _ in range(
new_engine_count - len(self.lb_engines)
)
]
else:
self.lb_engines = self.lb_engines[:new_engine_count]
# Send scale up notification to coordinator
scale_msg = msgspec.msgpack.encode(
("SCALE_ELASTIC_EP", new_engine_count)
)
await socket.send(scale_msg)
continue
# we're sending a request while the engines are
# paused, so that it can wake the others up
# (to run dummy EP loop).
assert decoded[0] == "FIRST_REQ"
target_eng_index = decoded[1]
self.engines_running = True
msg = msgspec.msgpack.encode(
(target_eng_index, self.current_wave)
)
await socket.send(msg)
events = dict(await poller.poll())
if scale_rcv_socket in events:
buf = scale_rcv_socket.recv(flags=zmq.NOBLOCK).result()
msg_type, new_engine_count = msgspec.msgpack.decode(buf)
assert msg_type == "SCALE_ELASTIC_EP"
# Update engine_ranks_managed and count_slice
parallel_config = self.vllm_config.parallel_config
dp_size = parallel_config.data_parallel_size
dp_rank = parallel_config.data_parallel_rank
assert dp_rank == 0
assert dp_size == new_engine_count
assert not parallel_config.data_parallel_hybrid_lb
# External LB clients continue to manage only local ranks.
if not parallel_config.data_parallel_external_lb:
self.engine_ranks_managed = list(range(dp_size))
if len(self.lb_engines) < new_engine_count:
self.lb_engines += [
[0, 0, 0.0]
for _ in range(
new_engine_count - len(self.lb_engines)
)
]
else:
self.lb_engines = self.lb_engines[:new_engine_count]
# Forward the scale notification to the coordinator.
await socket.send(buf)
continue
buf = None
while True:
@@ -1624,14 +1597,9 @@ class DPAsyncMPClient(AsyncMPClient):
request.current_wave = self.current_wave
request.client_index = self.client_index
# The engine that receives it wakes the other ranks if they are idle.
chosen_engine = self.get_core_engine_for_request(request)
to_await = self._send_input(EngineCoreRequestType.ADD, request, chosen_engine)
if not self.engines_running:
# Notify coordinator that we're sending a request
req_msg = msgspec.msgpack.encode(("FIRST_REQ", chosen_engine))
await self.first_req_send_socket.send(req_msg)
await to_await
await self._send_input(EngineCoreRequestType.ADD, request, chosen_engine)
self._ensure_output_queue_task()
@@ -2100,7 +2068,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
scale_up_marker = msgspec.msgpack.encode(
("SCALE_ELASTIC_EP", new_data_parallel_size)
)
await self.first_req_send_socket.send(scale_up_marker)
await self.scale_send_socket.send(scale_up_marker)
logger.info(
"[Elastic EP] Scale up completed, new data parallel size: %s",
@@ -2176,7 +2144,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
scale_down_marker = msgspec.msgpack.encode(
("SCALE_ELASTIC_EP", new_data_parallel_size)
)
await self.first_req_send_socket.send(scale_down_marker)
await self.scale_send_socket.send(scale_down_marker)
await wait_future
await self.resume_scheduler_async()
except Exception:
+12 -12
View File
@@ -483,24 +483,24 @@ class TopKTopPSampler(nn.Module):
return xpu_sample(logits, k, p, self.logprobs_mode)
# Note: this is a workaround for
# https://github.com/pytorch/pytorch/pull/151218
@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend)
def compiled_random_sample(logits: torch.Tensor) -> torch.Tensor:
probs = logits.softmax(dim=-1, dtype=torch.float32)
q = torch.empty_like(probs)
q.exponential_()
return probs.div(q).argmax(dim=-1).view(-1)
def apply_top_k_top_p(
logits: torch.Tensor, k: torch.Tensor | None, p: torch.Tensor | None
logits: torch.Tensor,
k: torch.Tensor | None,
p: torch.Tensor | None,
has_topp_only_rows: bool = True,
) -> torch.Tensor:
"""Apply top-k and/or top-p masking to logits.
has_topp_only_rows must be True if any row has p < 1 and
k >= logits.shape[-1] (no active top-k).
"""
if p is None and k is None:
return logits
if HAS_TRITON:
return apply_top_k_top_p_triton(logits, k, p)
return apply_top_k_top_p_triton(
logits, k, p, has_topp_only_rows=has_topp_only_rows
)
is_cpu = current_platform.is_cpu()
return apply_top_k_top_p_pytorch(logits, k, p, allow_cpu_sync=is_cpu)
+17 -6
View File
@@ -987,13 +987,19 @@ _SPLIT_MAX_BATCH = 64
_SPLIT_MAX_SPLITS = 32 # power of two
_SPLIT_FANOUT = 8
_SPLIT_ROUNDS = 5
# The step kernels are latency-bound at one program per SM, so keep splitting
# past that while slices span at least this many entries; shorter slices no
# longer amortize each program's row stats and combine replay.
_SPLIT_MIN_SLICE = 7168
_TRITON_SPLIT_CACHE: dict[torch.device, dict[str, torch.Tensor]] = {}
def _topp_split_count(batch_size: int, num_sm: int) -> int:
def _topp_split_count(batch_size: int, vocab_size: int, num_sm: int) -> int:
splits = 1
while splits < _SPLIT_MAX_SPLITS and splits * 2 * batch_size <= num_sm:
while splits < _SPLIT_MAX_SPLITS and (
splits * 2 * batch_size <= num_sm or splits * 2 * _SPLIT_MIN_SLICE <= vocab_size
):
splits *= 2
return splits
@@ -1079,7 +1085,7 @@ def _topp_split_stats(
num_sm: int,
) -> DispatchSpec:
batch_size, vocab_size = logits.shape
splits = _topp_split_count(batch_size, num_sm)
splits = _topp_split_count(batch_size, vocab_size, num_sm)
return (logits.shape[0] * splits,), dict(
K=k if k is not None else logits,
HAS_K=k is not None,
@@ -1373,7 +1379,7 @@ def _topp_split_step(
num_sm: int,
) -> DispatchSpec:
batch_size, vocab_size = logits.shape
splits = _topp_split_count(batch_size, num_sm)
splits = _topp_split_count(batch_size, vocab_size, num_sm)
return (logits.shape[0] * splits,), dict(
K=k if k is not None else logits,
HAS_K=k is not None,
@@ -1513,6 +1519,7 @@ def _topp_sb_mask_kernel(
cum = tl.cumsum(dmask.to(tl.int32)).to(tl.float32) + num_preceding
keep_dup = dmask & (cum <= numkeep)
keep = keep & (~dmask | keep_dup)
num_preceding += tl.sum(dmask.to(tl.float32))
tl.store(ROW + i + offs, tl.where(keep, x, MASK_VALUE), mask=mask_n)
@@ -1554,7 +1561,7 @@ def _topp_split_mask(
num_sm: int,
) -> DispatchSpec:
batch_size, vocab_size = logits.shape
splits = _topp_split_count(batch_size, num_sm)
splits = _topp_split_count(batch_size, vocab_size, num_sm)
return (logits.shape[0] * splits,), dict(
K=k if k is not None else logits,
HAS_K=k is not None,
@@ -1618,6 +1625,7 @@ def apply_top_k_top_p_triton(
k: torch.Tensor | None,
p: torch.Tensor | None,
mask_value: float = float("-inf"),
has_topp_only_rows: bool = True,
) -> torch.Tensor:
"""Apply combined top-k and top-p masking using Triton.
@@ -1632,6 +1640,9 @@ def apply_top_k_top_p_triton(
p: [batch_size] float32 tensor of top-p values per row (0 to 1),
or None to disable top-p
mask_value: Value for masked positions (default: -inf)
has_topp_only_rows: Whether any row has p < 1 and k >= vocab_size.
False lets small batches skip the split top-p pipeline; pass it
only if no such row exists, as such a row would get no top-p.
Returns:
The masked logits tensor. It may or may not be modified in-place.
@@ -1714,7 +1725,7 @@ def apply_top_k_top_p_triton(
mask_value,
num_sm,
)
if use_split:
if use_split and has_topp_only_rows:
_apply_topp_split(logits, k_ptr, p_ptr, mask_value, num_sm)
return logits
+15 -3
View File
@@ -151,6 +151,7 @@ class SimpleCPUOffloadScheduler:
lazy_offload: bool = False,
disk_capacity_bytes: int = 0,
use_page_cache: bool = False,
num_cpu_blocks: int | None = None,
):
self.vllm_config = vllm_config
self.kv_cache_config = kv_cache_config
@@ -170,7 +171,7 @@ class SimpleCPUOffloadScheduler:
# Derive a CPU KVCacheConfig from the GPU config and build a coordinator
assert kv_cache_config is not None
self.cpu_kv_cache_config = self._derive_cpu_config(
kv_cache_config, offload_capacity
kv_cache_config, offload_capacity, num_cpu_blocks
)
self.num_cpu_blocks = self.cpu_kv_cache_config.num_blocks
self.prefix_cacheable_group_ids = (
@@ -278,7 +279,9 @@ class SimpleCPUOffloadScheduler:
@staticmethod
def _derive_cpu_config(
gpu_config: "KVCacheConfig", cpu_capacity_bytes: int
gpu_config: "KVCacheConfig",
cpu_capacity_bytes: int,
num_cpu_blocks: int | None = None,
) -> "KVCacheConfig":
"""Derive a CPU KVCacheConfig from the GPU config.
Same kv_cache_groups, num_blocks scaled by CPU/GPU memory ratio."""
@@ -291,7 +294,16 @@ class SimpleCPUOffloadScheduler:
# so its size is the total GPU KV cache size.
gpu_total_bytes = gpu_config.kv_cache_tensors[0].size
num_gpu_blocks = gpu_config.num_blocks
num_cpu_blocks = max(1, num_gpu_blocks * cpu_capacity_bytes // gpu_total_bytes)
local_capacity = num_gpu_blocks * cpu_capacity_bytes // gpu_total_bytes
if num_cpu_blocks is None:
num_cpu_blocks = max(1, local_capacity)
elif (
type(num_cpu_blocks) is not int or not 0 < num_cpu_blocks <= local_capacity
):
raise ValueError(
f"num_cpu_blocks={num_cpu_blocks!r} must be between 1 and "
f"local offload capacity {local_capacity}"
)
# Create CPU kv_cache_tensors mirroring GPU by scaling size proportionally.
cpu_tensors = [
KVCacheTensor(
+21 -2
View File
@@ -36,10 +36,12 @@ class SimpleCPUOffloadWorker:
disk_capacity_bytes: int = 0,
disk_buffer_slots: int = 2,
use_page_cache: bool = False,
num_cpu_blocks: int | None = None,
):
self.vllm_config = vllm_config
self.kv_cache_config = kv_cache_config
self.cpu_capacity_bytes = cpu_capacity_bytes
self._num_cpu_blocks_override = num_cpu_blocks
self.disk_path = disk_path
self.disk_capacity_bytes = disk_capacity_bytes
self.disk_buffer_slots = disk_buffer_slots
@@ -159,7 +161,10 @@ class SimpleCPUOffloadWorker:
]
total_bytes_per_block = sum(per_tensor_bpb)
self.num_cpu_blocks = max(1, self.cpu_capacity_bytes // total_bytes_per_block)
self.num_cpu_blocks = self._resolve_num_cpu_blocks(
self.disk_capacity_bytes if self.disk_mode else self.cpu_capacity_bytes,
total_bytes_per_block,
)
# Use lowest priority so KV cache I/O yields to compute streams.
low_pri, _ = torch.cuda.Stream.priority_range()
@@ -173,13 +178,27 @@ class SimpleCPUOffloadWorker:
else:
self._init_cpu_mode(unique_gpu_caches, total_bytes_per_block, self.device)
def _resolve_num_cpu_blocks(self, capacity_bytes: int, bytes_per_block: int) -> int:
local_capacity = capacity_bytes // bytes_per_block
requested = self._num_cpu_blocks_override
if requested is None:
return max(1, local_capacity)
if type(requested) is not int or not 0 < requested <= local_capacity:
raise ValueError(
f"num_cpu_blocks={requested!r} must be between 1 and "
f"local offload capacity {local_capacity}"
)
return requested
def _init_disk_mode(
self,
unique_gpu_caches: dict[str, torch.Tensor],
total_bytes_per_block: int,
device: torch.device,
) -> None:
num_disk_slots = max(1, self.disk_capacity_bytes // total_bytes_per_block)
num_disk_slots = self._resolve_num_cpu_blocks(
self.disk_capacity_bytes, total_bytes_per_block
)
self.num_cpu_blocks = num_disk_slots
logger.info(
+5 -1
View File
@@ -58,6 +58,7 @@ class GPUWatermarkSampler(Sampler):
pos: torch.Tensor,
top_k: torch.Tensor | None,
top_p: torch.Tensor | None,
has_topp_only_rows: bool,
use_fused_sampler: bool,
) -> tuple[torch.Tensor, torch.Tensor]:
enabled = self.watermarking.np[idx_mapping_np] & (
@@ -71,10 +72,13 @@ class GPUWatermarkSampler(Sampler):
pos,
top_k,
top_p,
has_topp_only_rows,
use_fused_sampler,
)
processed_logits = apply_top_k_top_p(processed_logits, top_k, top_p)
processed_logits = apply_top_k_top_p(
processed_logits, top_k, top_p, has_topp_only_rows=has_topp_only_rows
)
contexts = self._get_contexts(expanded_idx_mapping)
repeated_contexts = None
if self.deduplicate_contexts != "none":
+9 -13
View File
@@ -1036,26 +1036,22 @@ def _extrapolate_full_graph_memory(mem_samples: list[int], total_graphs: int) ->
def _init_minimal_kv_cache_for_profiling(runner: "GPUModelRunner") -> None:
"""Allocate the smallest KV cache that still lets every graph be captured."""
from vllm.v1.core.kv_cache_planning import (
get_kv_cache_config_from_groups,
get_kv_cache_groups,
)
from vllm.v1.core.kv_cache_config_builder import get_kv_cache_config_builder
from vllm.v1.core.kv_cache_planning import get_kv_cache_groups_from_builder
kv_cache_spec = runner.get_kv_cache_spec()
kv_cache_groups = get_kv_cache_groups(runner.vllm_config, kv_cache_spec)
builder = get_kv_cache_config_builder(runner.vllm_config)
kv_cache_groups = get_kv_cache_groups_from_builder(
builder, runner.vllm_config, kv_cache_spec
)
# At least one block per sequence is required to capture the graphs.
min_blocks = (
min(runner.max_num_reqs, runner.compilation_config.max_cudagraph_capture_size)
or 1
)
saved_override = runner.cache_config.num_gpu_blocks_override
runner.cache_config.num_gpu_blocks_override = min_blocks
try:
minimal_config = get_kv_cache_config_from_groups(
runner.vllm_config, kv_cache_groups, available_memory=0
)
finally:
runner.cache_config.num_gpu_blocks_override = saved_override
minimal_config = builder.get_kv_cache_config_from_groups(
runner.vllm_config, kv_cache_groups, num_blocks=min_blocks
)
runner.initialize_kv_cache(minimal_config, is_profiling=True)
runner.cache_config.num_gpu_blocks = minimal_config.num_blocks
+7 -3
View File
@@ -298,8 +298,8 @@ class Sampler:
seq_lens_upper_bound_np,
skip_top_k_top_p=True,
)
top_k, top_p = self.sampling_states.get_top_k_top_p(
expanded_idx_mapping, idx_mapping_np
top_k, top_p, has_topp_only_rows = self.sampling_states.get_top_k_top_p(
expanded_idx_mapping, idx_mapping_np, logits.shape[-1]
)
# Don't use a fused sampler if no requests use top_k/top_p, if there are
# any greedy requests or per-request seeds, or if post-processed
@@ -321,6 +321,7 @@ class Sampler:
pos,
top_k,
top_p,
has_topp_only_rows,
use_fused_sampler,
)
@@ -332,6 +333,7 @@ class Sampler:
pos: torch.Tensor,
top_k: torch.Tensor | None,
top_p: torch.Tensor | None,
has_topp_only_rows: bool,
use_fused_sampler: bool,
) -> tuple[torch.Tensor, torch.Tensor]:
if use_fused_sampler:
@@ -342,7 +344,9 @@ class Sampler:
else: # Use XPU sampler
sampled, _ = xpu_sample(processed_logits, top_k, top_p)
else:
processed_logits = apply_top_k_top_p(processed_logits, top_k, top_p)
processed_logits = apply_top_k_top_p(
processed_logits, top_k, top_p, has_topp_only_rows=has_topp_only_rows
)
sampled = gumbel_sample(
processed_logits,
expanded_idx_mapping,
+20 -7
View File
@@ -103,13 +103,22 @@ class SamplingStates:
apply_min_p(logits, expanded_idx_mapping, self.min_p.gpu)
def get_top_k_top_p(
self, expanded_idx_mapping: torch.Tensor, idx_mapping_np: np.ndarray
) -> tuple[torch.Tensor | None, torch.Tensor | None]:
do_top_k = np.any(self.top_k.np[idx_mapping_np] != self.vocab_size)
do_top_p = np.any(self.top_p.np[idx_mapping_np] != 1.0)
self,
expanded_idx_mapping: torch.Tensor,
idx_mapping_np: np.ndarray,
vocab_size: int,
) -> tuple[torch.Tensor | None, torch.Tensor | None, bool]:
"""Returns (top_k, top_p, has_topp_only_rows)."""
top_k_np = self.top_k.np[idx_mapping_np]
top_p_np = self.top_p.np[idx_mapping_np]
do_top_k = np.any(top_k_np != self.vocab_size)
do_top_p = np.any(top_p_np != 1.0)
top_k = self.top_k.gpu[expanded_idx_mapping] if do_top_k else None
top_p = self.top_p.gpu[expanded_idx_mapping] if do_top_p else None
return top_k, top_p
has_topp_only_rows = bool(
do_top_p and np.any((top_k_np >= vocab_size) & (top_p_np < 1.0))
)
return top_k, top_p, has_topp_only_rows
def apply_top_k_top_p(
self,
@@ -117,10 +126,14 @@ class SamplingStates:
expanded_idx_mapping: torch.Tensor,
idx_mapping_np: np.ndarray,
) -> torch.Tensor:
top_k, top_p = self.get_top_k_top_p(expanded_idx_mapping, idx_mapping_np)
top_k, top_p, has_topp_only_rows = self.get_top_k_top_p(
expanded_idx_mapping, idx_mapping_np, logits.shape[-1]
)
if top_k is None and top_p is None:
return logits
return apply_top_k_top_p(logits, top_k, top_p)
return apply_top_k_top_p(
logits, top_k, top_p, has_topp_only_rows=has_topp_only_rows
)
def any_greedy(self, idx_mapping_np: np.ndarray) -> bool:
return bool(np.any(self.temperature.np[idx_mapping_np] == 0.0))
@@ -194,6 +194,16 @@ def _compute_global_logprobs_and_logsumexp(
return target_log_prob, draft_log_prob, target_lse, draft_lse
@triton.jit
def _is_bonus_logit(expanded_local_pos_ptr, logit_idx, num_logits):
next_local_pos = tl.load(
expanded_local_pos_ptr + logit_idx + 1,
mask=logit_idx + 1 < num_logits,
other=0,
)
return next_local_pos == 0
@triton.jit
def _compute_local_logits_stats_kernel(
# [num_logits, num_blocks]
@@ -225,17 +235,17 @@ def _compute_local_logits_stats_kernel(
# [max_num_reqs]
temp_ptr,
vocab_size,
num_speculative_steps,
num_logits,
BLOCK_SIZE: tl.constexpr,
HAS_DRAFT_LOGITS: tl.constexpr,
):
logit_idx = tl.program_id(0).to(tl.int64)
draft_step_idx = tl.load(expanded_local_pos_ptr + logit_idx)
if draft_step_idx >= num_speculative_steps:
# Bonus token. Max/argmax and summed exponentials are not needed.
if _is_bonus_logit(expanded_local_pos_ptr, logit_idx, num_logits):
# Max/argmax and summed exponentials are not needed.
return
draft_step_idx = tl.load(expanded_local_pos_ptr + logit_idx)
req_state_idx = tl.load(expanded_idx_mapping_ptr + logit_idx).to(tl.int64)
temp = tl.load(temp_ptr + req_state_idx).to(tl.float32)
@@ -423,17 +433,17 @@ def _compute_local_residual_mass_kernel(
# [max_num_reqs]
temp_ptr,
vocab_size,
num_speculative_steps,
num_logits,
vocab_num_blocks,
BLOCK_SIZE: tl.constexpr,
PADDED_VOCAB_NUM_BLOCKS: tl.constexpr,
):
logit_idx = tl.program_id(0).to(tl.int64)
# Adaptive verification can end before num_speculative_steps.
if logit_idx + 1 >= tl.num_programs(0):
return
draft_step_idx = tl.load(expanded_local_pos_ptr + logit_idx)
if draft_step_idx == 0 or draft_step_idx >= num_speculative_steps:
if draft_step_idx == 0 or _is_bonus_logit(
expanded_local_pos_ptr, logit_idx, num_logits
):
# The acceptance threshold, h, looks one position ahead and sums
# over: max(p_i * M_b(x|x_{<i}) - M_s(x|x_{<i}), 0). Tokens at the
# first and last (bonus) positions aren't needed for this computation.
@@ -1138,7 +1148,7 @@ def rejection_sample(
expanded_local_pos,
temperature,
vocab_size,
num_speculative_steps,
num_logits,
BLOCK_SIZE=VOCAB_BLOCK_SIZE,
HAS_DRAFT_LOGITS=has_draft_logits,
)
@@ -1209,7 +1219,7 @@ def rejection_sample(
expanded_local_pos,
temperature,
vocab_size,
num_speculative_steps,
num_logits,
vocab_num_blocks,
BLOCK_SIZE=VOCAB_BLOCK_SIZE,
PADDED_VOCAB_NUM_BLOCKS=padded_vocab_num_blocks,
+8 -12
View File
@@ -6174,27 +6174,23 @@ class GPUModelRunner(
gc.collect()
def _init_minimal_kv_cache_for_profiling(self) -> None:
from vllm.v1.core.kv_cache_planning import (
get_kv_cache_config_from_groups,
get_kv_cache_groups,
)
from vllm.v1.core.kv_cache_config_builder import get_kv_cache_config_builder
from vllm.v1.core.kv_cache_planning import get_kv_cache_groups_from_builder
kv_cache_spec = self.get_kv_cache_spec()
KVCacheSpecRegistry.check_kv_cache_spec_registry(kv_cache_spec)
kv_cache_groups = get_kv_cache_groups(self.vllm_config, kv_cache_spec)
builder = get_kv_cache_config_builder(self.vllm_config)
kv_cache_groups = get_kv_cache_groups_from_builder(
builder, self.vllm_config, kv_cache_spec
)
# the minimum number of blocks required is 1 block *per sequence*
min_blocks = (
min(self.max_num_reqs, self.compilation_config.max_cudagraph_capture_size)
or 1
)
# Temporarily change num_gpu_blocks_override to allocate a minimal KV cache
saved_override = self.cache_config.num_gpu_blocks_override
self.cache_config.num_gpu_blocks_override = min_blocks
minimal_config = get_kv_cache_config_from_groups(
self.vllm_config, kv_cache_groups, available_memory=0
minimal_config = builder.get_kv_cache_config_from_groups(
self.vllm_config, kv_cache_groups, num_blocks=min_blocks
)
self.cache_config.num_gpu_blocks_override = saved_override
self.initialize_kv_cache(minimal_config, is_profiling=True)
self.cache_config.num_gpu_blocks = minimal_config.num_blocks