Merge branch 'main' into patches
This commit is contained in:
@@ -6,7 +6,7 @@ from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config import VllmConfig, get_current_vllm_config_or_none
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointBuilder,
|
||||
MambaPrefillCheckpointMetadata,
|
||||
@@ -15,12 +15,13 @@ from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
CommonAttentionMetadata,
|
||||
max_decode_query_len,
|
||||
)
|
||||
from vllm.v1.attention.backends.mamba_attn import (
|
||||
BaseMambaAttentionMetadata,
|
||||
BaseMambaAttentionMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
from vllm.v1.kv_cache_interface import KVCacheSpec, MambaSpec
|
||||
|
||||
|
||||
def compute_varlen_chunk_metadata(
|
||||
@@ -104,6 +105,14 @@ class Mamba2AttentionBackend(AttentionBackend):
|
||||
def is_ssm(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def supports_device_cpu_query_lens_mismatch(cls) -> bool:
|
||||
# Decode kernels take device offsets; ReplaySSM plans off CPU counts.
|
||||
vllm_config = get_current_vllm_config_or_none()
|
||||
if vllm_config is None:
|
||||
return True
|
||||
return not vllm_config.cache_config.use_replayssm
|
||||
|
||||
|
||||
@dataclass
|
||||
class Mamba2AttentionMetadata(BaseMambaAttentionMetadata):
|
||||
@@ -124,6 +133,14 @@ class Mamba2AttentionMetadataBuilder(
|
||||
):
|
||||
metadata_cls = Mamba2AttentionMetadata
|
||||
|
||||
@classmethod
|
||||
def get_varlen_cudagraph_max_query_len(
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: KVCacheSpec,
|
||||
) -> int | None:
|
||||
return max_decode_query_len(vllm_config)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_spec: MambaSpec,
|
||||
|
||||
@@ -22,6 +22,17 @@ if TYPE_CHECKING:
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
|
||||
|
||||
def is_cute_dsl_mla_prefill_supported(
|
||||
qk_head_dim: int, v_head_dim: int, query_dtype: torch.dtype
|
||||
) -> bool:
|
||||
# FlashInfer has no public compatibility API; replace these checks when available.
|
||||
return (
|
||||
query_dtype in (torch.float16, torch.bfloat16, torch.float8_e4m3fn)
|
||||
and (qk_head_dim, v_head_dim) in ((128, 128), (192, 128))
|
||||
and (qk_head_dim != 192 or query_dtype == torch.float8_e4m3fn)
|
||||
)
|
||||
|
||||
|
||||
class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
|
||||
"""TRT-LLM Ragged backend for MLA prefill."""
|
||||
|
||||
@@ -79,6 +90,7 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
|
||||
v_head_dim=v_head_dim,
|
||||
vllm_config=vllm_config,
|
||||
)
|
||||
self._context_backend = vllm_config.attention_config.trtllm_ragged_backend
|
||||
self._workspace_spec = (
|
||||
(envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE,),
|
||||
torch.uint8,
|
||||
@@ -198,6 +210,16 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
|
||||
"TRTLLM ragged context prefill contains an empty query or KV row"
|
||||
)
|
||||
|
||||
if self._context_backend == "cute-dsl":
|
||||
if not is_cute_dsl_mla_prefill_supported(q.shape[-1], v.shape[-1], q.dtype):
|
||||
raise ValueError(
|
||||
"CuTe DSL context prefill does not support "
|
||||
f"Q/K head dimension {q.shape[-1]}, "
|
||||
f"V head dimension {v.shape[-1]}, and query dtype {q.dtype}."
|
||||
)
|
||||
# Generic MLA supplies V as a projection slice; CuTe requires packed V.
|
||||
v = v.contiguous()
|
||||
|
||||
if out is None:
|
||||
out = torch.empty(
|
||||
q.shape[0],
|
||||
@@ -225,6 +247,7 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
|
||||
cum_seq_lens_kv=chunk.cu_seq_lens,
|
||||
enable_pdl=False,
|
||||
is_causal=False,
|
||||
backend=self._context_backend,
|
||||
return_lse=True,
|
||||
out=out,
|
||||
skip_all_rows_active_check=True,
|
||||
|
||||
@@ -35,6 +35,7 @@ from vllm.v1.attention.backend import (
|
||||
AttentionLayer,
|
||||
CommonAttentionMetadata,
|
||||
MultipleOf,
|
||||
max_decode_query_len,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.triton_mla import _compute_num_kv_splits
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
@@ -49,7 +50,11 @@ from vllm.v1.attention.ops.rocm_aiter_mla_prefill import (
|
||||
gather_compressed_context,
|
||||
)
|
||||
from vllm.v1.attention.ops.triton_decode_attention import decode_attention_fwd
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec, is_quantized_kv_cache
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
KVCacheSpec,
|
||||
is_quantized_kv_cache,
|
||||
)
|
||||
from vllm.v1.worker.workspace import (
|
||||
current_workspace_manager,
|
||||
is_workspace_manager_initialized,
|
||||
@@ -457,6 +462,16 @@ def _aiter_mla_small_head_mode() -> str:
|
||||
return mode
|
||||
|
||||
|
||||
def _adaptive_varlen_verify_enabled(vllm_config: VllmConfig) -> bool:
|
||||
speculative_config = vllm_config.speculative_config
|
||||
return (
|
||||
speculative_config is not None
|
||||
and bool(getattr(speculative_config, "enable_adaptive_verification", False))
|
||||
and vllm_config.parallel_config.decode_context_parallel_size == 1
|
||||
and max_decode_query_len(vllm_config) > 1
|
||||
)
|
||||
|
||||
|
||||
class AiterMLABackend(MLACommonBackend):
|
||||
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16]
|
||||
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
|
||||
@@ -620,6 +635,8 @@ class AiterMLADecodeMetadata(MLACommonDecodeMetadata):
|
||||
use_gluon_verify: bool = False
|
||||
# Whether persistent MLA metadata was computed
|
||||
has_persistent_metadata: bool = False
|
||||
varlen_q_rows: torch.Tensor | None = None
|
||||
varlen_o_rows: torch.Tensor | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -720,6 +737,21 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
|
||||
return uniform_qo_len
|
||||
|
||||
@classmethod
|
||||
def get_varlen_cudagraph_max_query_len(
|
||||
cls: type["AiterMLAMetadataBuilder"],
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: KVCacheSpec,
|
||||
) -> int | None:
|
||||
if not _adaptive_varlen_verify_enabled(vllm_config):
|
||||
return None
|
||||
if (
|
||||
cls.get_cudagraph_support(vllm_config, kv_cache_spec)
|
||||
!= AttentionCGSupport.UNIFORM_BATCH
|
||||
):
|
||||
return None
|
||||
return max_decode_query_len(vllm_config)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_spec: AttentionSpec,
|
||||
@@ -823,6 +855,26 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
# persistent gate below and makes aiter raise a KeyError mid-run.
|
||||
self._mtp_decode_qlen = self.reorder_batch_threshold or 1
|
||||
|
||||
self._varlen_decode_qlen = 0
|
||||
if _adaptive_varlen_verify_enabled(vllm_config):
|
||||
assert self._mtp_decode_qlen == max_decode_query_len(vllm_config), (
|
||||
self._mtp_decode_qlen,
|
||||
max_decode_query_len(vllm_config),
|
||||
)
|
||||
self._varlen_decode_qlen = self._mtp_decode_qlen
|
||||
self.query_len_support = QueryLenSupport.VARLEN # type: ignore[misc]
|
||||
max_rows = vllm_config.scheduler_config.max_num_seqs * (
|
||||
self._varlen_decode_qlen
|
||||
)
|
||||
row_ids = torch.arange(max_rows, dtype=torch.int64, device=device)
|
||||
self._varlen_row_req = row_ids // self._varlen_decode_qlen
|
||||
self._varlen_row_pos = row_ids % self._varlen_decode_qlen
|
||||
self._varlen_row_ids = row_ids
|
||||
self._varlen_q_rows = torch.zeros_like(row_ids)
|
||||
self._varlen_o_rows = torch.zeros(
|
||||
max_rows + 1, dtype=torch.int64, device=device
|
||||
)
|
||||
|
||||
# Store the kernel block size from the spec. When kernel_block_size=1
|
||||
# (no spec-dec), behavior is identical to the original. When > 1
|
||||
# (e.g. 16 with Eagle3), we expand block-level indices into per-token
|
||||
@@ -850,6 +902,12 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
max_num_pages_per_req = vllm_config.model_config.max_model_len
|
||||
max_num_reqs = vllm_config.scheduler_config.max_num_seqs
|
||||
max_num_pages = max_num_reqs * max_num_pages_per_req
|
||||
speculative_config = vllm_config.speculative_config
|
||||
if _adaptive_varlen_verify_enabled(vllm_config):
|
||||
assert speculative_config is not None
|
||||
max_num_pages += max_num_reqs * (
|
||||
speculative_config.num_speculative_tokens or 0
|
||||
)
|
||||
|
||||
# Preparing persistent buffers
|
||||
# TODO: we can disambiguate between decode and mixed-prefill decode here
|
||||
@@ -1437,13 +1495,18 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
device = self.device
|
||||
num_reqs = seq_lens_device.size(0)
|
||||
qo_len = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1]
|
||||
max_qo_len = qo_len.max().item()
|
||||
padded_mtp_qo_len = self._uniform_padded_mtp_qo_len(
|
||||
qo_len, max_qo_len, num_decode_tokens
|
||||
)
|
||||
if padded_mtp_qo_len > 0:
|
||||
max_qo_len = padded_mtp_qo_len
|
||||
pad_uniform_mtp = padded_mtp_qo_len > 0
|
||||
varlen_decode = self._varlen_decode_qlen > 0 and causal
|
||||
if varlen_decode:
|
||||
max_qo_len = self._varlen_decode_qlen
|
||||
pad_uniform_mtp = True
|
||||
else:
|
||||
max_qo_len = qo_len.max().item()
|
||||
padded_mtp_qo_len = self._uniform_padded_mtp_qo_len(
|
||||
qo_len, max_qo_len, num_decode_tokens
|
||||
)
|
||||
if padded_mtp_qo_len > 0:
|
||||
max_qo_len = padded_mtp_qo_len
|
||||
pad_uniform_mtp = padded_mtp_qo_len > 0
|
||||
|
||||
seq_lens_for_kernel = seq_lens_device
|
||||
# the global lengths need the same dummy-row
|
||||
@@ -1477,6 +1540,13 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
g_tot_seq_lens.new_full((), max_qo_len * self.dcp_world_size),
|
||||
)
|
||||
|
||||
varlen_q_rows = varlen_o_rows = None
|
||||
if varlen_decode:
|
||||
seq_lens_for_kernel = seq_lens_device + (max_qo_len - qo_lens_device)
|
||||
varlen_q_rows, varlen_o_rows = self._fill_varlen_rows(
|
||||
query_start_loc_device, qo_lens_device, num_reqs
|
||||
)
|
||||
|
||||
if self._graph_seq_lens is not None:
|
||||
self._graph_seq_lens[:num_kernel_reqs].copy_(
|
||||
seq_lens_for_kernel, non_blocking=True
|
||||
@@ -1537,6 +1607,8 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
# device synchronisation. Programs whose chunk lies past
|
||||
# num_tokens mask out entirely.
|
||||
max_tokens_per_req = block_table_tensor.shape[1] * self.kernel_block_size
|
||||
if varlen_decode:
|
||||
max_tokens_per_req += max_qo_len - 1
|
||||
num_chunks = max(1, cdiv(max_tokens_per_req, 1024))
|
||||
_expand_page_indices_kernel[(num_reqs, num_chunks)](
|
||||
self.paged_kv_indices,
|
||||
@@ -1544,8 +1616,10 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
block_table_tensor.stride(0),
|
||||
block_table_tensor.stride(1),
|
||||
paged_kv_indptr,
|
||||
seq_lens_device,
|
||||
KERNEL_BLOCK_SIZE=self.kernel_block_size,
|
||||
BLOCK_SIZE=1024,
|
||||
CLAMP_TAIL=varlen_decode,
|
||||
)
|
||||
paged_kv_indices = self.paged_kv_indices
|
||||
|
||||
@@ -1740,10 +1814,38 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
use_gluon_verify=use_gluon_verify,
|
||||
attn_out_dtype=self.decode_attn_out_dtype,
|
||||
has_persistent_metadata=has_persistent_metadata,
|
||||
varlen_q_rows=varlen_q_rows,
|
||||
varlen_o_rows=varlen_o_rows,
|
||||
)
|
||||
|
||||
return attn_metadata
|
||||
|
||||
def _fill_varlen_rows(
|
||||
self,
|
||||
query_start_loc_device: torch.Tensor,
|
||||
qo_lens_device: torch.Tensor,
|
||||
num_reqs: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
width = self._varlen_decode_qlen
|
||||
num_rows = num_reqs * width
|
||||
row_req = self._varlen_row_req[:num_rows]
|
||||
row_pos = self._varlen_row_pos[:num_rows]
|
||||
row_qlen = qo_lens_device.to(torch.int64).index_select(0, row_req)
|
||||
row_start = query_start_loc_device.to(torch.int64).index_select(0, row_req)
|
||||
is_real = row_pos < row_qlen
|
||||
packed_row = row_start + row_pos
|
||||
q_rows = self._varlen_q_rows[:num_rows]
|
||||
q_rows.copy_(torch.where(is_real, packed_row, 0))
|
||||
o_rows = self._varlen_o_rows
|
||||
o_rows.zero_()
|
||||
dump_slot = o_rows.size(0) - 1
|
||||
o_rows.scatter_(
|
||||
0,
|
||||
torch.where(is_real, packed_row, dump_slot),
|
||||
self._varlen_row_ids[:num_rows],
|
||||
)
|
||||
return q_rows, o_rows
|
||||
|
||||
def build(
|
||||
self,
|
||||
common_prefix_len: int,
|
||||
@@ -1795,8 +1897,10 @@ def _expand_page_indices_kernel(
|
||||
block_table_stride_0,
|
||||
block_table_stride_1,
|
||||
cu_num_tokens,
|
||||
num_real_tokens,
|
||||
KERNEL_BLOCK_SIZE: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
CLAMP_TAIL: tl.constexpr = False,
|
||||
):
|
||||
"""Expand block table entries into per-token flat page indices.
|
||||
|
||||
@@ -1810,6 +1914,10 @@ def _expand_page_indices_kernel(
|
||||
|
||||
When KERNEL_BLOCK_SIZE=K: block table entry b (covering K tokens)
|
||||
is expanded to flat indices b*K, b*K+1, ..., b*K+(K-1).
|
||||
|
||||
With CLAMP_TAIL, entries past num_real_tokens[req] (padding a request's
|
||||
page list out to its kernel length) repeat the request's first page, so
|
||||
they never index past its block-table row.
|
||||
"""
|
||||
# One program per (request, token-chunk). Parallelising over requests
|
||||
# alone degenerates at low concurrency: with num_reqs == 1 a single
|
||||
@@ -1819,6 +1927,8 @@ def _expand_page_indices_kernel(
|
||||
row_ptr = block_table + req_idx * block_table_stride_0
|
||||
start_idx = tl.load(cu_num_tokens + req_idx)
|
||||
num_tokens = tl.load(cu_num_tokens + req_idx + 1) - start_idx
|
||||
if CLAMP_TAIL:
|
||||
real_tokens = tl.load(num_real_tokens + req_idx)
|
||||
|
||||
# The grid is sized from the block table width, an upper bound over all
|
||||
# requests, so a ragged batch launches chunks past a short request's end.
|
||||
@@ -1830,11 +1940,14 @@ def _expand_page_indices_kernel(
|
||||
|
||||
token_offsets = chunk_start + tl.arange(0, BLOCK_SIZE)
|
||||
mask = token_offsets < num_tokens
|
||||
source_offsets = token_offsets
|
||||
if CLAMP_TAIL:
|
||||
source_offsets = tl.where(token_offsets < real_tokens, token_offsets, 0)
|
||||
|
||||
# Which block in the block table does this token belong to?
|
||||
block_idx = token_offsets // KERNEL_BLOCK_SIZE
|
||||
block_idx = source_offsets // KERNEL_BLOCK_SIZE
|
||||
# Offset within that block
|
||||
offset_in_block = token_offsets % KERNEL_BLOCK_SIZE
|
||||
offset_in_block = source_offsets % KERNEL_BLOCK_SIZE
|
||||
|
||||
# Load the block ID from the block table
|
||||
# Both strides are taken from the caller: the block table is a view owned
|
||||
@@ -2600,6 +2713,28 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
|
||||
kv_c_and_k_pe_cache: torch.Tensor,
|
||||
attn_metadata: AiterMLAMetadata,
|
||||
layer: AttentionLayer,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
decode = attn_metadata.decode
|
||||
if decode is None or decode.varlen_q_rows is None:
|
||||
return self._forward_mqa(q, kv_c_and_k_pe_cache, attn_metadata, layer)
|
||||
assert decode.varlen_o_rows is not None
|
||||
q_rows = decode.varlen_q_rows
|
||||
if isinstance(q, tuple):
|
||||
num_tokens = q[0].shape[0]
|
||||
q = (q[0].index_select(0, q_rows), q[1].index_select(0, q_rows))
|
||||
else:
|
||||
num_tokens = q.shape[0]
|
||||
q = q.index_select(0, q_rows)
|
||||
o, lse = self._forward_mqa(q, kv_c_and_k_pe_cache, attn_metadata, layer)
|
||||
assert lse is None
|
||||
return o.index_select(0, decode.varlen_o_rows[:num_tokens]), None
|
||||
|
||||
def _forward_mqa(
|
||||
self,
|
||||
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
kv_c_and_k_pe_cache: torch.Tensor,
|
||||
attn_metadata: AiterMLAMetadata,
|
||||
layer: AttentionLayer,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
assert kv_c_and_k_pe_cache.numel() > 0
|
||||
assert attn_metadata.decode is not None
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Interface and resolution for pluggable KV cache config builders."""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.utils.import_utils import resolve_obj_by_qualname
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheSpec,
|
||||
)
|
||||
|
||||
|
||||
class KVCacheConfigBuilder(ABC):
|
||||
"""Interface for platform KV cache placement.
|
||||
|
||||
.. note::
|
||||
This interface is experimental and subject to change. The exact hooks
|
||||
and the ``num_blocks`` contract may evolve before it stabilizes.
|
||||
|
||||
Subclasses normally inherit from ``DefaultKVCacheConfigBuilder`` and
|
||||
override only the hooks they need. The cross-worker planning flow
|
||||
(:func:`vllm.v1.core.kv_cache_planning.get_kv_cache_configs`) is Core-owned;
|
||||
this builder supplies the placement decisions: how layers are split into
|
||||
groups and how a group list is materialized into tensors for a given block
|
||||
count. Grouping coordination lives in ``DefaultKVCacheConfigBuilder`` (a
|
||||
model implementing :class:`~vllm.model_executor.models.interfaces.
|
||||
SupportsKVCacheGrouping` owns its grouping there); a platform that replaces
|
||||
the builder fully owns its own grouping.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def get_kv_cache_groups(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
kv_cache_spec: dict[str, "KVCacheSpec"],
|
||||
) -> list["KVCacheGroupSpec"]:
|
||||
"""Split the given layers into KV cache groups.
|
||||
|
||||
Planning passes all workers' layers; profiling passes one worker's.
|
||||
Core rejects a result that does not put each layer in exactly one group.
|
||||
|
||||
The default builder honors a model's
|
||||
:class:`~vllm.model_executor.models.interfaces.SupportsKVCacheGrouping`
|
||||
declaration here before falling back to the default grouping; a
|
||||
platform-specific builder decides its own grouping.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def get_kv_cache_config_from_groups(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
kv_cache_groups: list["KVCacheGroupSpec"],
|
||||
num_blocks: int,
|
||||
) -> "KVCacheConfig":
|
||||
"""Materialize the groups for exactly ``num_blocks`` blocks.
|
||||
|
||||
Each device backing's size is ``bytes_per_block * num_blocks``. All
|
||||
device tensors share one backing allocation unless the builder
|
||||
overrides ``_get_pool_bytes_per_block``. Host-resident tensors, such as
|
||||
HiSparse source storage, are excluded from the device byte accounting
|
||||
and sized independently.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def _get_pool_bytes_per_block(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
kv_cache_groups: list["KVCacheGroupSpec"],
|
||||
) -> int | None:
|
||||
"""Return the device bytes one block ID consumes, or ``None``.
|
||||
|
||||
Private and transitional; expected to go away with worker-driven
|
||||
allocation (RFC #42449). With ``None`` (the default), Core derives the
|
||||
value from a one-block placement with one device backing. Override it
|
||||
only for placements with several device allocations.
|
||||
"""
|
||||
return None
|
||||
|
||||
|
||||
def get_kv_cache_config_builder(
|
||||
vllm_config: "VllmConfig",
|
||||
) -> KVCacheConfigBuilder:
|
||||
"""Resolve the builder selected by the current platform.
|
||||
|
||||
Resolution priority is owned by the platform hook
|
||||
(:meth:`vllm.platforms.interface.Platform.get_kv_cache_config_builder_cls`).
|
||||
Builders are stateless (every method takes ``vllm_config``), so a fresh
|
||||
instance is returned per call and engines with different configs in the
|
||||
same process always get the correct builder.
|
||||
"""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
builder_cls = resolve_obj_by_qualname(
|
||||
current_platform.get_kv_cache_config_builder_cls(vllm_config)
|
||||
)
|
||||
return builder_cls()
|
||||
@@ -14,6 +14,7 @@ from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv, round_up
|
||||
from vllm.utils.mem_utils import format_gib
|
||||
from vllm.utils.torch_utils import get_dtype_size
|
||||
from vllm.v1.core.kv_cache_config_builder import KVCacheConfigBuilder
|
||||
from vllm.v1.core.kv_cache_utils import _kv_cache_device_name
|
||||
from vllm.v1.hisparse.layout import (
|
||||
get_hisparse_host_pool_bytes,
|
||||
@@ -261,6 +262,48 @@ def _pool_bytes_per_block(
|
||||
)
|
||||
|
||||
|
||||
def _pool_bytes_per_block_from_builder(
|
||||
builder: KVCacheConfigBuilder,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
) -> int:
|
||||
"""Bytes consumed by one block in the worker's device KV cache pool.
|
||||
|
||||
Core places one block with the builder and reads the size of its single
|
||||
device backing. A placement with several device allocations declares the
|
||||
value through ``KVCacheConfigBuilder._get_pool_bytes_per_block``. HiSparse
|
||||
keeps the pure computation, since probing its placement would check the host
|
||||
pool against ``max_model_len`` before auto-fit shortens it.
|
||||
``num_gpu_blocks_override`` is suppressed so it cannot inflate the probe.
|
||||
"""
|
||||
if not kv_cache_groups:
|
||||
return 0
|
||||
if vllm_config.attention_config.hisparse_config is not None:
|
||||
return _pool_bytes_per_block(vllm_config, kv_cache_groups)
|
||||
pool_bytes = builder._get_pool_bytes_per_block(vllm_config, kv_cache_groups)
|
||||
if pool_bytes is not None:
|
||||
return pool_bytes
|
||||
saved_override = vllm_config.cache_config.num_gpu_blocks_override
|
||||
vllm_config.cache_config.num_gpu_blocks_override = None
|
||||
try:
|
||||
probe = builder.get_kv_cache_config_from_groups(
|
||||
vllm_config, kv_cache_groups, num_blocks=1
|
||||
)
|
||||
finally:
|
||||
vllm_config.cache_config.num_gpu_blocks_override = saved_override
|
||||
backing_sizes = {
|
||||
tensor.size for tensor in probe.kv_cache_tensors if not tensor.host_resident
|
||||
}
|
||||
if len(backing_sizes) != 1:
|
||||
raise ValueError(
|
||||
"KV cache tensors must share one device backing allocation, but got "
|
||||
f"sizes {sorted(backing_sizes)}. Override "
|
||||
"KVCacheConfigBuilder._get_pool_bytes_per_block() for placements "
|
||||
"with several device allocations."
|
||||
)
|
||||
return backing_sizes.pop()
|
||||
|
||||
|
||||
def get_uniform_page_size(kv_cache_specs: Iterable[KVCacheSpec]) -> int:
|
||||
"""Get the page size of the KV cache."""
|
||||
page_sizes = {layer.page_size_bytes for layer in kv_cache_specs}
|
||||
@@ -1264,6 +1307,36 @@ def get_kv_cache_groups(
|
||||
return groups
|
||||
|
||||
|
||||
def get_kv_cache_groups_from_builder(
|
||||
builder: KVCacheConfigBuilder,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: dict[str, KVCacheSpec],
|
||||
) -> list[KVCacheGroupSpec]:
|
||||
"""Group the layers with the builder and check the result.
|
||||
|
||||
Core checks every grouping, whether it comes from the default grouping, a
|
||||
model's ``SupportsKVCacheGrouping`` or a platform builder.
|
||||
"""
|
||||
groups = builder.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
_check_exact_partition(groups, kv_cache_spec)
|
||||
return groups
|
||||
|
||||
|
||||
def _check_exact_partition(
|
||||
groups: list[KVCacheGroupSpec],
|
||||
kv_cache_spec: dict[str, KVCacheSpec],
|
||||
) -> None:
|
||||
"""Ensure the groups partition every input layer exactly once."""
|
||||
grouped_layers: set[str] = set()
|
||||
for group in groups:
|
||||
assert not set(group.layer_names) & grouped_layers, (
|
||||
f"KV cache group overlaps: {set(group.layer_names) & grouped_layers}."
|
||||
)
|
||||
grouped_layers |= set(group.layer_names)
|
||||
missing = set(kv_cache_spec) - grouped_layers
|
||||
assert not missing, f"KV cache groups miss layers: {sorted(missing)}."
|
||||
|
||||
|
||||
def _layer_tp_replicas(spec: KVCacheSpec, tp_size: int, dcp_size: int) -> int:
|
||||
if not isinstance(spec, AttentionSpec) or spec.max_tp_shards is None:
|
||||
return 1
|
||||
@@ -1285,6 +1358,7 @@ def kv_cache_groups_tp_replicas(
|
||||
|
||||
|
||||
def _max_memory_usage_bytes_from_groups(
|
||||
builder: KVCacheConfigBuilder,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
) -> int:
|
||||
@@ -1301,7 +1375,9 @@ def _max_memory_usage_bytes_from_groups(
|
||||
if not kv_cache_groups:
|
||||
return 0
|
||||
|
||||
bytes_per_block = _pool_bytes_per_block(vllm_config, kv_cache_groups)
|
||||
bytes_per_block = _pool_bytes_per_block_from_builder(
|
||||
builder, vllm_config, kv_cache_groups
|
||||
)
|
||||
total_blocks = 0
|
||||
for group in kv_cache_groups:
|
||||
spec = group.kv_cache_spec
|
||||
@@ -1317,6 +1393,7 @@ def _max_memory_usage_bytes_from_groups(
|
||||
|
||||
|
||||
def _estimate_max_model_len_from_groups(
|
||||
builder: KVCacheConfigBuilder,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
available_memory: int,
|
||||
@@ -1334,13 +1411,16 @@ def _estimate_max_model_len_from_groups(
|
||||
vllm_config.model_config.max_model_len = model_len
|
||||
if hisparse_enabled:
|
||||
try:
|
||||
get_kv_cache_config_from_groups(
|
||||
vllm_config, kv_cache_groups, available_memory
|
||||
num_blocks = available_memory // _pool_bytes_per_block_from_builder(
|
||||
builder, vllm_config, kv_cache_groups
|
||||
)
|
||||
builder.get_kv_cache_config_from_groups(
|
||||
vllm_config, kv_cache_groups, num_blocks
|
||||
)
|
||||
except ValueError:
|
||||
return False
|
||||
return (
|
||||
_max_memory_usage_bytes_from_groups(vllm_config, kv_cache_groups)
|
||||
_max_memory_usage_bytes_from_groups(builder, vllm_config, kv_cache_groups)
|
||||
<= available_memory
|
||||
)
|
||||
|
||||
@@ -1362,6 +1442,7 @@ def _estimate_max_model_len_from_groups(
|
||||
|
||||
|
||||
def _auto_fit_max_model_len(
|
||||
builder: KVCacheConfigBuilder,
|
||||
vllm_config: VllmConfig,
|
||||
projected_groups_per_worker: list[list[KVCacheGroupSpec]],
|
||||
available_memory: list[int],
|
||||
@@ -1372,6 +1453,7 @@ def _auto_fit_max_model_len(
|
||||
workers.
|
||||
|
||||
Args:
|
||||
builder: The builder whose hooks drive block accounting.
|
||||
vllm_config: The global VllmConfig (will be modified in-place)
|
||||
projected_groups_per_worker: KV cache groups projected to each worker.
|
||||
available_memory: Memory available for KV cache in bytes for each
|
||||
@@ -1395,7 +1477,9 @@ def _auto_fit_max_model_len(
|
||||
for groups, avail_mem in zip(projected_groups_per_worker, available_memory):
|
||||
if not groups:
|
||||
continue
|
||||
worker_max = _estimate_max_model_len_from_groups(vllm_config, groups, avail_mem)
|
||||
worker_max = _estimate_max_model_len_from_groups(
|
||||
builder, vllm_config, groups, avail_mem
|
||||
)
|
||||
if worker_max < auto_fit_max:
|
||||
auto_fit_max = worker_max
|
||||
limiting_worker_mem = avail_mem
|
||||
@@ -1473,6 +1557,9 @@ def _project_kv_cache_groups_to_worker(
|
||||
worker_layer_names,
|
||||
group_spec,
|
||||
is_eagle_group=group.is_eagle_group and bool(worker_layer_names),
|
||||
host_resident=group.host_resident,
|
||||
enable_kv_transfer=group.enable_kv_transfer,
|
||||
role=group.role,
|
||||
)
|
||||
)
|
||||
return projected_groups
|
||||
@@ -1512,6 +1599,10 @@ def get_kv_cache_configs(
|
||||
The generated KVCacheConfigs for each worker.
|
||||
|
||||
"""
|
||||
from vllm.v1.core.kv_cache_config_builder import get_kv_cache_config_builder
|
||||
|
||||
builder = get_kv_cache_config_builder(vllm_config)
|
||||
|
||||
# Merge the KV cache specs of all workers. Different PP stages may have
|
||||
# different layer names, and different TP ranks of the same PP stage should
|
||||
# have the same KV cache spec.
|
||||
@@ -1542,7 +1633,9 @@ def get_kv_cache_configs(
|
||||
# Get global KV cache groups. This also handles spec unification for
|
||||
# hybrid models when disable_hybrid_kv_cache_manager is enabled.
|
||||
# After this call, merged_kv_cache_specs may be modified in-place.
|
||||
global_kv_cache_groups = get_kv_cache_groups(vllm_config, merged_kv_cache_specs)
|
||||
global_kv_cache_groups = get_kv_cache_groups_from_builder(
|
||||
builder, vllm_config, merged_kv_cache_specs
|
||||
)
|
||||
|
||||
# If original_max_model_len was -1, automatically
|
||||
# determine the maximum model length that fits in available GPU memory.
|
||||
@@ -1553,11 +1646,10 @@ def get_kv_cache_configs(
|
||||
]
|
||||
|
||||
# If `num_gpu_blocks_override` is set, the cache size that will actually
|
||||
# be allocated is decoupled from the profiled `available_memory`:
|
||||
# `may_override_num_blocks` in `get_kv_cache_config_from_groups` clamps
|
||||
# `num_blocks` to the override. Reflect that in `available_memory` here so
|
||||
# auto-fit, the admission check, and the per-worker config builder all
|
||||
# plan against the same effective capacity.
|
||||
# be allocated is decoupled from the profiled `available_memory`. Reflect
|
||||
# that in `available_memory` here so auto-fit, the admission check, and
|
||||
# the per-worker config builder all plan against the same effective
|
||||
# capacity.
|
||||
override = vllm_config.cache_config.num_gpu_blocks_override
|
||||
if override is not None:
|
||||
adjusted_memory: list[int] = []
|
||||
@@ -1565,7 +1657,9 @@ def get_kv_cache_configs(
|
||||
if not groups:
|
||||
adjusted_memory.append(avail_mem)
|
||||
continue
|
||||
bytes_per_block = _pool_bytes_per_block(vllm_config, groups)
|
||||
bytes_per_block = _pool_bytes_per_block_from_builder(
|
||||
builder, vllm_config, groups
|
||||
)
|
||||
logger.info(
|
||||
"Overriding num_gpu_blocks=%d with num_gpu_blocks_override=%d",
|
||||
avail_mem // bytes_per_block,
|
||||
@@ -1578,12 +1672,18 @@ def get_kv_cache_configs(
|
||||
# the capacity check both plan against usable blocks. Allocation below
|
||||
# still uses the full memory.
|
||||
check_memory = [
|
||||
avail_mem - _pool_bytes_per_block(vllm_config, groups) if groups else avail_mem
|
||||
(
|
||||
avail_mem - _pool_bytes_per_block_from_builder(builder, vllm_config, groups)
|
||||
if groups
|
||||
else avail_mem
|
||||
)
|
||||
for groups, avail_mem in zip(projected_groups_per_worker, available_memory)
|
||||
]
|
||||
|
||||
if vllm_config.model_config.original_max_model_len == -1:
|
||||
_auto_fit_max_model_len(vllm_config, projected_groups_per_worker, check_memory)
|
||||
_auto_fit_max_model_len(
|
||||
builder, vllm_config, projected_groups_per_worker, check_memory
|
||||
)
|
||||
|
||||
# Check if the available memory is enough per worker.
|
||||
for groups, avail_mem in zip(projected_groups_per_worker, check_memory):
|
||||
@@ -1591,9 +1691,9 @@ def get_kv_cache_configs(
|
||||
continue
|
||||
_check_enough_kv_cache_memory(
|
||||
avail_mem,
|
||||
partial(_max_memory_usage_bytes_from_groups, vllm_config, groups),
|
||||
partial(_max_memory_usage_bytes_from_groups, builder, vllm_config, groups),
|
||||
vllm_config.model_config.max_model_len,
|
||||
partial(_estimate_max_model_len_from_groups, vllm_config, groups),
|
||||
partial(_estimate_max_model_len_from_groups, builder, vllm_config, groups),
|
||||
)
|
||||
|
||||
kv_cache_configs: list[KVCacheConfig] = []
|
||||
@@ -1603,9 +1703,15 @@ def get_kv_cache_configs(
|
||||
assert sum(len(group.layer_names) for group in projected_groups) == len(
|
||||
kv_cache_spec_one_worker
|
||||
), "Some layers are not assigned to any group."
|
||||
bytes_per_block = _pool_bytes_per_block_from_builder(
|
||||
builder, vllm_config, projected_groups
|
||||
)
|
||||
num_blocks = (
|
||||
available_memory_one_worker // bytes_per_block if projected_groups else 1
|
||||
)
|
||||
kv_cache_configs.append(
|
||||
get_kv_cache_config_from_groups(
|
||||
vllm_config, projected_groups, available_memory_one_worker
|
||||
builder.get_kv_cache_config_from_groups(
|
||||
vllm_config, projected_groups, num_blocks
|
||||
)
|
||||
)
|
||||
|
||||
@@ -1618,13 +1724,12 @@ def get_kv_cache_configs(
|
||||
for i, kv_cache_config in enumerate(kv_cache_configs):
|
||||
if kv_cache_config.num_blocks == min_num_blocks:
|
||||
continue
|
||||
# Re-plan with exactly the memory the smallest rank can afford, so
|
||||
# strides and offsets stay consistent with the shrunken allocation.
|
||||
groups = kv_cache_config.kv_cache_groups
|
||||
kv_cache_configs[i] = get_kv_cache_config_from_groups(
|
||||
# Re-plan with exactly the smallest rank's block count, so strides and
|
||||
# offsets stay consistent with the shrunken allocation.
|
||||
kv_cache_configs[i] = builder.get_kv_cache_config_from_groups(
|
||||
vllm_config,
|
||||
groups,
|
||||
min_num_blocks * _pool_bytes_per_block(vllm_config, groups),
|
||||
kv_cache_config.kv_cache_groups,
|
||||
min_num_blocks,
|
||||
)
|
||||
|
||||
for kv_cache_config in kv_cache_configs:
|
||||
@@ -1635,3 +1740,64 @@ def get_kv_cache_configs(
|
||||
)
|
||||
|
||||
return kv_cache_configs
|
||||
|
||||
|
||||
class DefaultKVCacheConfigBuilder(KVCacheConfigBuilder):
|
||||
"""Default grouping and placement, backed by the module-level functions here.
|
||||
|
||||
.. note::
|
||||
This class is part of the experimental ``KVCacheConfigBuilder``
|
||||
interface and is subject to change.
|
||||
|
||||
Subclasses override only the hooks they need. The planning flow and its
|
||||
invariants (``num_gpu_blocks_override``, null-block reservation, auto-fit,
|
||||
admission checks, cross-rank convergence) stay in the Core-owned
|
||||
:func:`get_kv_cache_configs`.
|
||||
"""
|
||||
|
||||
def get_kv_cache_groups(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: dict[str, KVCacheSpec],
|
||||
) -> list[KVCacheGroupSpec]:
|
||||
"""See ``KVCacheConfigBuilder.get_kv_cache_groups``.
|
||||
|
||||
A model implementing ``SupportsKVCacheGrouping`` owns the grouping
|
||||
decision; otherwise the default grouping is used.
|
||||
"""
|
||||
if not vllm_config.model_config.supports_kv_cache_grouping:
|
||||
return get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
if vllm_config.attention_config.hisparse_config is not None:
|
||||
raise ValueError(
|
||||
"A model-declared get_kv_cache_groups cannot be combined "
|
||||
"with HiSparse, which owns its own KV cache grouping. "
|
||||
"Remove one of them."
|
||||
)
|
||||
model_cls, _ = vllm_config.model_config.registry.resolve_model_cls(
|
||||
vllm_config.model_config.architectures,
|
||||
vllm_config.model_config,
|
||||
)
|
||||
return model_cls.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
|
||||
def get_kv_cache_config_from_groups(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
num_blocks: int,
|
||||
) -> KVCacheConfig:
|
||||
"""See ``KVCacheConfigBuilder.get_kv_cache_config_from_groups``."""
|
||||
if len(kv_cache_groups) == 0:
|
||||
# Attention free models do not have KV cache.
|
||||
# Return num_blocks=1 as BlockPool always needs a null_block.
|
||||
return KVCacheConfig(
|
||||
num_blocks=1,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=kv_cache_groups,
|
||||
prefix_cache_retention_interval=(
|
||||
vllm_config.cache_config.prefix_cache_retention_interval
|
||||
),
|
||||
)
|
||||
bytes_per_block = _pool_bytes_per_block(vllm_config, kv_cache_groups)
|
||||
return get_kv_cache_config_from_groups(
|
||||
vllm_config, kv_cache_groups, num_blocks * bytes_per_block
|
||||
)
|
||||
|
||||
@@ -41,12 +41,9 @@ class DPCoordinator:
|
||||
DPEngineCoreProc._has_global_unfinished_reqs method.
|
||||
|
||||
* Broadcasts the START_DP_WAVE message to engines to move them from paused
|
||||
to running state when one engine receives a new request. This can happen
|
||||
in two cases:
|
||||
1) A front-end sending a new request while the engines are paused will
|
||||
concurrently notify the coordinator.
|
||||
2) An engine receiving a request for a stale request wave while in paused
|
||||
state will notify the coordinator.
|
||||
to running state when an idle engine receives a new request and notifies
|
||||
the coordinator. Only the engines start waves: a front-end's view of the
|
||||
running state can be stale, and a paused engine discards START_DP_WAVE.
|
||||
|
||||
Engines will move into running state when receiving a new request or
|
||||
START_DP_WAVE message.
|
||||
@@ -353,27 +350,9 @@ class DPCoordinatorProc:
|
||||
)
|
||||
continue # Skip normal engine notification processing
|
||||
|
||||
# Wave coordination: handle new-request messages from front-end.
|
||||
# Only process these when wave coordination is enabled
|
||||
if self.enable_wave_coordination:
|
||||
# We received a message on the front-end XPUB socket,
|
||||
# from an API server sending a new request while the
|
||||
# engines are paused, so that we can wake the other
|
||||
# engines.
|
||||
engine_to_exclude, wave = decoded
|
||||
if not engines_running:
|
||||
if wave < current_wave:
|
||||
# If the wave number is stale, ensure the message
|
||||
# is handled by all the engines.
|
||||
engine_to_exclude = None
|
||||
|
||||
# engines_running is only set from the engines'
|
||||
# own notifications; a paused engine discards
|
||||
# START_DP_WAVE, so sending it is not evidence
|
||||
# that the engines are running.
|
||||
self._send_start_wave(
|
||||
publish_back, current_wave, engine_to_exclude
|
||||
)
|
||||
logger.error(
|
||||
"DP Coordinator received unexpected message from front-end"
|
||||
)
|
||||
|
||||
if output_back in events:
|
||||
# We received a message from one of the engines.
|
||||
@@ -448,13 +427,12 @@ class DPCoordinatorProc:
|
||||
wave > current_wave
|
||||
or (wave == current_wave and not engines_running)
|
||||
):
|
||||
# 3. The engine received request for a non-current wave
|
||||
# so we must ensure that other engines progress to the
|
||||
# next wave (race condition handling).
|
||||
# 3. An engine started a wave (e.g. an idle engine
|
||||
# received a request), so the others must join it.
|
||||
logger.debug(
|
||||
"Starting wave %d after notification of "
|
||||
"stale wave request from engine.",
|
||||
"Starting wave %d on notification from engine %d.",
|
||||
wave,
|
||||
eng_index,
|
||||
)
|
||||
current_wave = wave
|
||||
engines_running = True
|
||||
|
||||
@@ -2269,15 +2269,14 @@ class DPEngineCoreProc(EngineCoreProc):
|
||||
|
||||
def add_request(self, request: Request, request_wave: int = 0):
|
||||
super().add_request(request, request_wave)
|
||||
if self.has_coordinator and request_wave != self.current_wave:
|
||||
if self.has_coordinator:
|
||||
if request_wave > self.current_wave:
|
||||
self.current_wave = request_wave
|
||||
elif (
|
||||
if (
|
||||
not self.engines_running
|
||||
and self.scheduler.pause_state == PauseState.UNPAUSED
|
||||
):
|
||||
# Request received for an already-completed wave, notify
|
||||
# front-end that we need to start the next one.
|
||||
# Engines alone start waves: have the coordinator wake the others.
|
||||
self.engines_running = True
|
||||
self.output_queue.put_nowait(
|
||||
(-1, EngineCoreOutputs(start_wave=self.current_wave))
|
||||
|
||||
@@ -506,8 +506,8 @@ class BackgroundResources:
|
||||
coordinator: DPCoordinator | None = None
|
||||
output_socket: zmq.Socket | zmq.asyncio.Socket | None = None
|
||||
input_socket: zmq.Socket | zmq.asyncio.Socket | None = None
|
||||
first_req_send_socket: zmq.asyncio.Socket | None = None
|
||||
first_req_rcv_socket: zmq.asyncio.Socket | None = None
|
||||
scale_send_socket: zmq.asyncio.Socket | None = None
|
||||
scale_rcv_socket: zmq.asyncio.Socket | None = None
|
||||
stats_update_socket: zmq.asyncio.Socket | None = None
|
||||
output_queue_task: asyncio.Task | None = None
|
||||
stats_update_task: asyncio.Task | None = None
|
||||
@@ -535,8 +535,8 @@ class BackgroundResources:
|
||||
sockets = (
|
||||
self.output_socket,
|
||||
self.input_socket,
|
||||
self.first_req_send_socket,
|
||||
self.first_req_rcv_socket,
|
||||
self.scale_send_socket,
|
||||
self.scale_rcv_socket,
|
||||
self.stats_update_socket,
|
||||
)
|
||||
|
||||
@@ -1493,9 +1493,9 @@ class DPAsyncMPClient(AsyncMPClient):
|
||||
|
||||
self.eep_scaling_cache: ElasticScalingCache | None = None
|
||||
|
||||
self.first_req_sock_addr = get_open_zmq_inproc_path()
|
||||
self.first_req_send_socket = self.resources.first_req_send_socket = (
|
||||
make_zmq_socket(self.ctx, self.first_req_sock_addr, zmq.PAIR, bind=True)
|
||||
self.scale_sock_addr = get_open_zmq_inproc_path()
|
||||
self.scale_send_socket = self.resources.scale_send_socket = make_zmq_socket(
|
||||
self.ctx, self.scale_sock_addr, zmq.PAIR, bind=True
|
||||
)
|
||||
try:
|
||||
# If we are running in an asyncio event loop, start the stats task.
|
||||
@@ -1518,75 +1518,48 @@ class DPAsyncMPClient(AsyncMPClient):
|
||||
with (
|
||||
make_zmq_socket(self.ctx, stats_addr, zmq.XSUB, linger=0) as socket,
|
||||
make_zmq_socket(
|
||||
self.ctx, self.first_req_sock_addr, zmq.PAIR, bind=False, linger=0
|
||||
) as first_req_rcv_socket,
|
||||
self.ctx, self.scale_sock_addr, zmq.PAIR, bind=False, linger=0
|
||||
) as scale_rcv_socket,
|
||||
):
|
||||
assert isinstance(socket, zmq.asyncio.Socket)
|
||||
assert isinstance(first_req_rcv_socket, zmq.asyncio.Socket)
|
||||
assert isinstance(scale_rcv_socket, zmq.asyncio.Socket)
|
||||
self.resources.stats_update_socket = socket
|
||||
self.resources.first_req_rcv_socket = first_req_rcv_socket
|
||||
self.resources.scale_rcv_socket = scale_rcv_socket
|
||||
# Send subscription message.
|
||||
await socket.send(b"\x01")
|
||||
|
||||
poller = zmq.asyncio.Poller()
|
||||
poller.register(socket, zmq.POLLIN)
|
||||
poller.register(first_req_rcv_socket, zmq.POLLIN)
|
||||
poller.register(scale_rcv_socket, zmq.POLLIN)
|
||||
|
||||
while True:
|
||||
events = await poller.poll()
|
||||
if (
|
||||
not self.engines_running
|
||||
and len(events) == 2
|
||||
or (events[0][0] == first_req_rcv_socket)
|
||||
):
|
||||
# Check if this is a regular request notification or
|
||||
# scale up notification
|
||||
buf = first_req_rcv_socket.recv(flags=zmq.NOBLOCK).result()
|
||||
|
||||
decoded = msgspec.msgpack.decode(buf)
|
||||
if (
|
||||
isinstance(decoded, (list, tuple))
|
||||
and len(decoded) == 2
|
||||
and decoded[0] == "SCALE_ELASTIC_EP"
|
||||
):
|
||||
# Extract new engine count from the decoded message
|
||||
new_engine_count = decoded[1]
|
||||
# Update engine_ranks_managed and count_slice
|
||||
parallel_config = self.vllm_config.parallel_config
|
||||
dp_size = parallel_config.data_parallel_size
|
||||
dp_rank = parallel_config.data_parallel_rank
|
||||
assert dp_rank == 0
|
||||
assert dp_size == new_engine_count
|
||||
assert not parallel_config.data_parallel_hybrid_lb
|
||||
# External LB clients continue to manage only local ranks.
|
||||
if not parallel_config.data_parallel_external_lb:
|
||||
self.engine_ranks_managed = list(range(dp_size))
|
||||
if len(self.lb_engines) < new_engine_count:
|
||||
self.lb_engines += [
|
||||
[0, 0, 0.0]
|
||||
for _ in range(
|
||||
new_engine_count - len(self.lb_engines)
|
||||
)
|
||||
]
|
||||
else:
|
||||
self.lb_engines = self.lb_engines[:new_engine_count]
|
||||
# Send scale up notification to coordinator
|
||||
scale_msg = msgspec.msgpack.encode(
|
||||
("SCALE_ELASTIC_EP", new_engine_count)
|
||||
)
|
||||
await socket.send(scale_msg)
|
||||
continue
|
||||
|
||||
# we're sending a request while the engines are
|
||||
# paused, so that it can wake the others up
|
||||
# (to run dummy EP loop).
|
||||
assert decoded[0] == "FIRST_REQ"
|
||||
target_eng_index = decoded[1]
|
||||
self.engines_running = True
|
||||
msg = msgspec.msgpack.encode(
|
||||
(target_eng_index, self.current_wave)
|
||||
)
|
||||
await socket.send(msg)
|
||||
events = dict(await poller.poll())
|
||||
if scale_rcv_socket in events:
|
||||
buf = scale_rcv_socket.recv(flags=zmq.NOBLOCK).result()
|
||||
msg_type, new_engine_count = msgspec.msgpack.decode(buf)
|
||||
assert msg_type == "SCALE_ELASTIC_EP"
|
||||
# Update engine_ranks_managed and count_slice
|
||||
parallel_config = self.vllm_config.parallel_config
|
||||
dp_size = parallel_config.data_parallel_size
|
||||
dp_rank = parallel_config.data_parallel_rank
|
||||
assert dp_rank == 0
|
||||
assert dp_size == new_engine_count
|
||||
assert not parallel_config.data_parallel_hybrid_lb
|
||||
# External LB clients continue to manage only local ranks.
|
||||
if not parallel_config.data_parallel_external_lb:
|
||||
self.engine_ranks_managed = list(range(dp_size))
|
||||
if len(self.lb_engines) < new_engine_count:
|
||||
self.lb_engines += [
|
||||
[0, 0, 0.0]
|
||||
for _ in range(
|
||||
new_engine_count - len(self.lb_engines)
|
||||
)
|
||||
]
|
||||
else:
|
||||
self.lb_engines = self.lb_engines[:new_engine_count]
|
||||
# Forward the scale notification to the coordinator.
|
||||
await socket.send(buf)
|
||||
continue
|
||||
|
||||
buf = None
|
||||
while True:
|
||||
@@ -1624,14 +1597,9 @@ class DPAsyncMPClient(AsyncMPClient):
|
||||
request.current_wave = self.current_wave
|
||||
request.client_index = self.client_index
|
||||
|
||||
# The engine that receives it wakes the other ranks if they are idle.
|
||||
chosen_engine = self.get_core_engine_for_request(request)
|
||||
to_await = self._send_input(EngineCoreRequestType.ADD, request, chosen_engine)
|
||||
if not self.engines_running:
|
||||
# Notify coordinator that we're sending a request
|
||||
req_msg = msgspec.msgpack.encode(("FIRST_REQ", chosen_engine))
|
||||
await self.first_req_send_socket.send(req_msg)
|
||||
|
||||
await to_await
|
||||
await self._send_input(EngineCoreRequestType.ADD, request, chosen_engine)
|
||||
|
||||
self._ensure_output_queue_task()
|
||||
|
||||
@@ -2100,7 +2068,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
scale_up_marker = msgspec.msgpack.encode(
|
||||
("SCALE_ELASTIC_EP", new_data_parallel_size)
|
||||
)
|
||||
await self.first_req_send_socket.send(scale_up_marker)
|
||||
await self.scale_send_socket.send(scale_up_marker)
|
||||
|
||||
logger.info(
|
||||
"[Elastic EP] Scale up completed, new data parallel size: %s",
|
||||
@@ -2176,7 +2144,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
scale_down_marker = msgspec.msgpack.encode(
|
||||
("SCALE_ELASTIC_EP", new_data_parallel_size)
|
||||
)
|
||||
await self.first_req_send_socket.send(scale_down_marker)
|
||||
await self.scale_send_socket.send(scale_down_marker)
|
||||
await wait_future
|
||||
await self.resume_scheduler_async()
|
||||
except Exception:
|
||||
|
||||
@@ -483,24 +483,24 @@ class TopKTopPSampler(nn.Module):
|
||||
return xpu_sample(logits, k, p, self.logprobs_mode)
|
||||
|
||||
|
||||
# Note: this is a workaround for
|
||||
# https://github.com/pytorch/pytorch/pull/151218
|
||||
@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend)
|
||||
def compiled_random_sample(logits: torch.Tensor) -> torch.Tensor:
|
||||
probs = logits.softmax(dim=-1, dtype=torch.float32)
|
||||
q = torch.empty_like(probs)
|
||||
q.exponential_()
|
||||
return probs.div(q).argmax(dim=-1).view(-1)
|
||||
|
||||
|
||||
def apply_top_k_top_p(
|
||||
logits: torch.Tensor, k: torch.Tensor | None, p: torch.Tensor | None
|
||||
logits: torch.Tensor,
|
||||
k: torch.Tensor | None,
|
||||
p: torch.Tensor | None,
|
||||
has_topp_only_rows: bool = True,
|
||||
) -> torch.Tensor:
|
||||
"""Apply top-k and/or top-p masking to logits.
|
||||
|
||||
has_topp_only_rows must be True if any row has p < 1 and
|
||||
k >= logits.shape[-1] (no active top-k).
|
||||
"""
|
||||
if p is None and k is None:
|
||||
return logits
|
||||
|
||||
if HAS_TRITON:
|
||||
return apply_top_k_top_p_triton(logits, k, p)
|
||||
return apply_top_k_top_p_triton(
|
||||
logits, k, p, has_topp_only_rows=has_topp_only_rows
|
||||
)
|
||||
|
||||
is_cpu = current_platform.is_cpu()
|
||||
return apply_top_k_top_p_pytorch(logits, k, p, allow_cpu_sync=is_cpu)
|
||||
|
||||
@@ -987,13 +987,19 @@ _SPLIT_MAX_BATCH = 64
|
||||
_SPLIT_MAX_SPLITS = 32 # power of two
|
||||
_SPLIT_FANOUT = 8
|
||||
_SPLIT_ROUNDS = 5
|
||||
# The step kernels are latency-bound at one program per SM, so keep splitting
|
||||
# past that while slices span at least this many entries; shorter slices no
|
||||
# longer amortize each program's row stats and combine replay.
|
||||
_SPLIT_MIN_SLICE = 7168
|
||||
|
||||
_TRITON_SPLIT_CACHE: dict[torch.device, dict[str, torch.Tensor]] = {}
|
||||
|
||||
|
||||
def _topp_split_count(batch_size: int, num_sm: int) -> int:
|
||||
def _topp_split_count(batch_size: int, vocab_size: int, num_sm: int) -> int:
|
||||
splits = 1
|
||||
while splits < _SPLIT_MAX_SPLITS and splits * 2 * batch_size <= num_sm:
|
||||
while splits < _SPLIT_MAX_SPLITS and (
|
||||
splits * 2 * batch_size <= num_sm or splits * 2 * _SPLIT_MIN_SLICE <= vocab_size
|
||||
):
|
||||
splits *= 2
|
||||
return splits
|
||||
|
||||
@@ -1079,7 +1085,7 @@ def _topp_split_stats(
|
||||
num_sm: int,
|
||||
) -> DispatchSpec:
|
||||
batch_size, vocab_size = logits.shape
|
||||
splits = _topp_split_count(batch_size, num_sm)
|
||||
splits = _topp_split_count(batch_size, vocab_size, num_sm)
|
||||
return (logits.shape[0] * splits,), dict(
|
||||
K=k if k is not None else logits,
|
||||
HAS_K=k is not None,
|
||||
@@ -1373,7 +1379,7 @@ def _topp_split_step(
|
||||
num_sm: int,
|
||||
) -> DispatchSpec:
|
||||
batch_size, vocab_size = logits.shape
|
||||
splits = _topp_split_count(batch_size, num_sm)
|
||||
splits = _topp_split_count(batch_size, vocab_size, num_sm)
|
||||
return (logits.shape[0] * splits,), dict(
|
||||
K=k if k is not None else logits,
|
||||
HAS_K=k is not None,
|
||||
@@ -1513,6 +1519,7 @@ def _topp_sb_mask_kernel(
|
||||
cum = tl.cumsum(dmask.to(tl.int32)).to(tl.float32) + num_preceding
|
||||
keep_dup = dmask & (cum <= numkeep)
|
||||
keep = keep & (~dmask | keep_dup)
|
||||
num_preceding += tl.sum(dmask.to(tl.float32))
|
||||
tl.store(ROW + i + offs, tl.where(keep, x, MASK_VALUE), mask=mask_n)
|
||||
|
||||
|
||||
@@ -1554,7 +1561,7 @@ def _topp_split_mask(
|
||||
num_sm: int,
|
||||
) -> DispatchSpec:
|
||||
batch_size, vocab_size = logits.shape
|
||||
splits = _topp_split_count(batch_size, num_sm)
|
||||
splits = _topp_split_count(batch_size, vocab_size, num_sm)
|
||||
return (logits.shape[0] * splits,), dict(
|
||||
K=k if k is not None else logits,
|
||||
HAS_K=k is not None,
|
||||
@@ -1618,6 +1625,7 @@ def apply_top_k_top_p_triton(
|
||||
k: torch.Tensor | None,
|
||||
p: torch.Tensor | None,
|
||||
mask_value: float = float("-inf"),
|
||||
has_topp_only_rows: bool = True,
|
||||
) -> torch.Tensor:
|
||||
"""Apply combined top-k and top-p masking using Triton.
|
||||
|
||||
@@ -1632,6 +1640,9 @@ def apply_top_k_top_p_triton(
|
||||
p: [batch_size] float32 tensor of top-p values per row (0 to 1),
|
||||
or None to disable top-p
|
||||
mask_value: Value for masked positions (default: -inf)
|
||||
has_topp_only_rows: Whether any row has p < 1 and k >= vocab_size.
|
||||
False lets small batches skip the split top-p pipeline; pass it
|
||||
only if no such row exists, as such a row would get no top-p.
|
||||
|
||||
Returns:
|
||||
The masked logits tensor. It may or may not be modified in-place.
|
||||
@@ -1714,7 +1725,7 @@ def apply_top_k_top_p_triton(
|
||||
mask_value,
|
||||
num_sm,
|
||||
)
|
||||
if use_split:
|
||||
if use_split and has_topp_only_rows:
|
||||
_apply_topp_split(logits, k_ptr, p_ptr, mask_value, num_sm)
|
||||
|
||||
return logits
|
||||
|
||||
@@ -151,6 +151,7 @@ class SimpleCPUOffloadScheduler:
|
||||
lazy_offload: bool = False,
|
||||
disk_capacity_bytes: int = 0,
|
||||
use_page_cache: bool = False,
|
||||
num_cpu_blocks: int | None = None,
|
||||
):
|
||||
self.vllm_config = vllm_config
|
||||
self.kv_cache_config = kv_cache_config
|
||||
@@ -170,7 +171,7 @@ class SimpleCPUOffloadScheduler:
|
||||
# Derive a CPU KVCacheConfig from the GPU config and build a coordinator
|
||||
assert kv_cache_config is not None
|
||||
self.cpu_kv_cache_config = self._derive_cpu_config(
|
||||
kv_cache_config, offload_capacity
|
||||
kv_cache_config, offload_capacity, num_cpu_blocks
|
||||
)
|
||||
self.num_cpu_blocks = self.cpu_kv_cache_config.num_blocks
|
||||
self.prefix_cacheable_group_ids = (
|
||||
@@ -278,7 +279,9 @@ class SimpleCPUOffloadScheduler:
|
||||
|
||||
@staticmethod
|
||||
def _derive_cpu_config(
|
||||
gpu_config: "KVCacheConfig", cpu_capacity_bytes: int
|
||||
gpu_config: "KVCacheConfig",
|
||||
cpu_capacity_bytes: int,
|
||||
num_cpu_blocks: int | None = None,
|
||||
) -> "KVCacheConfig":
|
||||
"""Derive a CPU KVCacheConfig from the GPU config.
|
||||
Same kv_cache_groups, num_blocks scaled by CPU/GPU memory ratio."""
|
||||
@@ -291,7 +294,16 @@ class SimpleCPUOffloadScheduler:
|
||||
# so its size is the total GPU KV cache size.
|
||||
gpu_total_bytes = gpu_config.kv_cache_tensors[0].size
|
||||
num_gpu_blocks = gpu_config.num_blocks
|
||||
num_cpu_blocks = max(1, num_gpu_blocks * cpu_capacity_bytes // gpu_total_bytes)
|
||||
local_capacity = num_gpu_blocks * cpu_capacity_bytes // gpu_total_bytes
|
||||
if num_cpu_blocks is None:
|
||||
num_cpu_blocks = max(1, local_capacity)
|
||||
elif (
|
||||
type(num_cpu_blocks) is not int or not 0 < num_cpu_blocks <= local_capacity
|
||||
):
|
||||
raise ValueError(
|
||||
f"num_cpu_blocks={num_cpu_blocks!r} must be between 1 and "
|
||||
f"local offload capacity {local_capacity}"
|
||||
)
|
||||
# Create CPU kv_cache_tensors mirroring GPU by scaling size proportionally.
|
||||
cpu_tensors = [
|
||||
KVCacheTensor(
|
||||
|
||||
@@ -36,10 +36,12 @@ class SimpleCPUOffloadWorker:
|
||||
disk_capacity_bytes: int = 0,
|
||||
disk_buffer_slots: int = 2,
|
||||
use_page_cache: bool = False,
|
||||
num_cpu_blocks: int | None = None,
|
||||
):
|
||||
self.vllm_config = vllm_config
|
||||
self.kv_cache_config = kv_cache_config
|
||||
self.cpu_capacity_bytes = cpu_capacity_bytes
|
||||
self._num_cpu_blocks_override = num_cpu_blocks
|
||||
self.disk_path = disk_path
|
||||
self.disk_capacity_bytes = disk_capacity_bytes
|
||||
self.disk_buffer_slots = disk_buffer_slots
|
||||
@@ -159,7 +161,10 @@ class SimpleCPUOffloadWorker:
|
||||
]
|
||||
total_bytes_per_block = sum(per_tensor_bpb)
|
||||
|
||||
self.num_cpu_blocks = max(1, self.cpu_capacity_bytes // total_bytes_per_block)
|
||||
self.num_cpu_blocks = self._resolve_num_cpu_blocks(
|
||||
self.disk_capacity_bytes if self.disk_mode else self.cpu_capacity_bytes,
|
||||
total_bytes_per_block,
|
||||
)
|
||||
|
||||
# Use lowest priority so KV cache I/O yields to compute streams.
|
||||
low_pri, _ = torch.cuda.Stream.priority_range()
|
||||
@@ -173,13 +178,27 @@ class SimpleCPUOffloadWorker:
|
||||
else:
|
||||
self._init_cpu_mode(unique_gpu_caches, total_bytes_per_block, self.device)
|
||||
|
||||
def _resolve_num_cpu_blocks(self, capacity_bytes: int, bytes_per_block: int) -> int:
|
||||
local_capacity = capacity_bytes // bytes_per_block
|
||||
requested = self._num_cpu_blocks_override
|
||||
if requested is None:
|
||||
return max(1, local_capacity)
|
||||
if type(requested) is not int or not 0 < requested <= local_capacity:
|
||||
raise ValueError(
|
||||
f"num_cpu_blocks={requested!r} must be between 1 and "
|
||||
f"local offload capacity {local_capacity}"
|
||||
)
|
||||
return requested
|
||||
|
||||
def _init_disk_mode(
|
||||
self,
|
||||
unique_gpu_caches: dict[str, torch.Tensor],
|
||||
total_bytes_per_block: int,
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
num_disk_slots = max(1, self.disk_capacity_bytes // total_bytes_per_block)
|
||||
num_disk_slots = self._resolve_num_cpu_blocks(
|
||||
self.disk_capacity_bytes, total_bytes_per_block
|
||||
)
|
||||
self.num_cpu_blocks = num_disk_slots
|
||||
|
||||
logger.info(
|
||||
|
||||
@@ -58,6 +58,7 @@ class GPUWatermarkSampler(Sampler):
|
||||
pos: torch.Tensor,
|
||||
top_k: torch.Tensor | None,
|
||||
top_p: torch.Tensor | None,
|
||||
has_topp_only_rows: bool,
|
||||
use_fused_sampler: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
enabled = self.watermarking.np[idx_mapping_np] & (
|
||||
@@ -71,10 +72,13 @@ class GPUWatermarkSampler(Sampler):
|
||||
pos,
|
||||
top_k,
|
||||
top_p,
|
||||
has_topp_only_rows,
|
||||
use_fused_sampler,
|
||||
)
|
||||
|
||||
processed_logits = apply_top_k_top_p(processed_logits, top_k, top_p)
|
||||
processed_logits = apply_top_k_top_p(
|
||||
processed_logits, top_k, top_p, has_topp_only_rows=has_topp_only_rows
|
||||
)
|
||||
contexts = self._get_contexts(expanded_idx_mapping)
|
||||
repeated_contexts = None
|
||||
if self.deduplicate_contexts != "none":
|
||||
|
||||
@@ -1036,26 +1036,22 @@ def _extrapolate_full_graph_memory(mem_samples: list[int], total_graphs: int) ->
|
||||
|
||||
def _init_minimal_kv_cache_for_profiling(runner: "GPUModelRunner") -> None:
|
||||
"""Allocate the smallest KV cache that still lets every graph be captured."""
|
||||
from vllm.v1.core.kv_cache_planning import (
|
||||
get_kv_cache_config_from_groups,
|
||||
get_kv_cache_groups,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_config_builder import get_kv_cache_config_builder
|
||||
from vllm.v1.core.kv_cache_planning import get_kv_cache_groups_from_builder
|
||||
|
||||
kv_cache_spec = runner.get_kv_cache_spec()
|
||||
kv_cache_groups = get_kv_cache_groups(runner.vllm_config, kv_cache_spec)
|
||||
builder = get_kv_cache_config_builder(runner.vllm_config)
|
||||
kv_cache_groups = get_kv_cache_groups_from_builder(
|
||||
builder, runner.vllm_config, kv_cache_spec
|
||||
)
|
||||
# At least one block per sequence is required to capture the graphs.
|
||||
min_blocks = (
|
||||
min(runner.max_num_reqs, runner.compilation_config.max_cudagraph_capture_size)
|
||||
or 1
|
||||
)
|
||||
saved_override = runner.cache_config.num_gpu_blocks_override
|
||||
runner.cache_config.num_gpu_blocks_override = min_blocks
|
||||
try:
|
||||
minimal_config = get_kv_cache_config_from_groups(
|
||||
runner.vllm_config, kv_cache_groups, available_memory=0
|
||||
)
|
||||
finally:
|
||||
runner.cache_config.num_gpu_blocks_override = saved_override
|
||||
minimal_config = builder.get_kv_cache_config_from_groups(
|
||||
runner.vllm_config, kv_cache_groups, num_blocks=min_blocks
|
||||
)
|
||||
|
||||
runner.initialize_kv_cache(minimal_config, is_profiling=True)
|
||||
runner.cache_config.num_gpu_blocks = minimal_config.num_blocks
|
||||
|
||||
@@ -298,8 +298,8 @@ class Sampler:
|
||||
seq_lens_upper_bound_np,
|
||||
skip_top_k_top_p=True,
|
||||
)
|
||||
top_k, top_p = self.sampling_states.get_top_k_top_p(
|
||||
expanded_idx_mapping, idx_mapping_np
|
||||
top_k, top_p, has_topp_only_rows = self.sampling_states.get_top_k_top_p(
|
||||
expanded_idx_mapping, idx_mapping_np, logits.shape[-1]
|
||||
)
|
||||
# Don't use a fused sampler if no requests use top_k/top_p, if there are
|
||||
# any greedy requests or per-request seeds, or if post-processed
|
||||
@@ -321,6 +321,7 @@ class Sampler:
|
||||
pos,
|
||||
top_k,
|
||||
top_p,
|
||||
has_topp_only_rows,
|
||||
use_fused_sampler,
|
||||
)
|
||||
|
||||
@@ -332,6 +333,7 @@ class Sampler:
|
||||
pos: torch.Tensor,
|
||||
top_k: torch.Tensor | None,
|
||||
top_p: torch.Tensor | None,
|
||||
has_topp_only_rows: bool,
|
||||
use_fused_sampler: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if use_fused_sampler:
|
||||
@@ -342,7 +344,9 @@ class Sampler:
|
||||
else: # Use XPU sampler
|
||||
sampled, _ = xpu_sample(processed_logits, top_k, top_p)
|
||||
else:
|
||||
processed_logits = apply_top_k_top_p(processed_logits, top_k, top_p)
|
||||
processed_logits = apply_top_k_top_p(
|
||||
processed_logits, top_k, top_p, has_topp_only_rows=has_topp_only_rows
|
||||
)
|
||||
sampled = gumbel_sample(
|
||||
processed_logits,
|
||||
expanded_idx_mapping,
|
||||
|
||||
@@ -103,13 +103,22 @@ class SamplingStates:
|
||||
apply_min_p(logits, expanded_idx_mapping, self.min_p.gpu)
|
||||
|
||||
def get_top_k_top_p(
|
||||
self, expanded_idx_mapping: torch.Tensor, idx_mapping_np: np.ndarray
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor | None]:
|
||||
do_top_k = np.any(self.top_k.np[idx_mapping_np] != self.vocab_size)
|
||||
do_top_p = np.any(self.top_p.np[idx_mapping_np] != 1.0)
|
||||
self,
|
||||
expanded_idx_mapping: torch.Tensor,
|
||||
idx_mapping_np: np.ndarray,
|
||||
vocab_size: int,
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor | None, bool]:
|
||||
"""Returns (top_k, top_p, has_topp_only_rows)."""
|
||||
top_k_np = self.top_k.np[idx_mapping_np]
|
||||
top_p_np = self.top_p.np[idx_mapping_np]
|
||||
do_top_k = np.any(top_k_np != self.vocab_size)
|
||||
do_top_p = np.any(top_p_np != 1.0)
|
||||
top_k = self.top_k.gpu[expanded_idx_mapping] if do_top_k else None
|
||||
top_p = self.top_p.gpu[expanded_idx_mapping] if do_top_p else None
|
||||
return top_k, top_p
|
||||
has_topp_only_rows = bool(
|
||||
do_top_p and np.any((top_k_np >= vocab_size) & (top_p_np < 1.0))
|
||||
)
|
||||
return top_k, top_p, has_topp_only_rows
|
||||
|
||||
def apply_top_k_top_p(
|
||||
self,
|
||||
@@ -117,10 +126,14 @@ class SamplingStates:
|
||||
expanded_idx_mapping: torch.Tensor,
|
||||
idx_mapping_np: np.ndarray,
|
||||
) -> torch.Tensor:
|
||||
top_k, top_p = self.get_top_k_top_p(expanded_idx_mapping, idx_mapping_np)
|
||||
top_k, top_p, has_topp_only_rows = self.get_top_k_top_p(
|
||||
expanded_idx_mapping, idx_mapping_np, logits.shape[-1]
|
||||
)
|
||||
if top_k is None and top_p is None:
|
||||
return logits
|
||||
return apply_top_k_top_p(logits, top_k, top_p)
|
||||
return apply_top_k_top_p(
|
||||
logits, top_k, top_p, has_topp_only_rows=has_topp_only_rows
|
||||
)
|
||||
|
||||
def any_greedy(self, idx_mapping_np: np.ndarray) -> bool:
|
||||
return bool(np.any(self.temperature.np[idx_mapping_np] == 0.0))
|
||||
|
||||
@@ -194,6 +194,16 @@ def _compute_global_logprobs_and_logsumexp(
|
||||
return target_log_prob, draft_log_prob, target_lse, draft_lse
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _is_bonus_logit(expanded_local_pos_ptr, logit_idx, num_logits):
|
||||
next_local_pos = tl.load(
|
||||
expanded_local_pos_ptr + logit_idx + 1,
|
||||
mask=logit_idx + 1 < num_logits,
|
||||
other=0,
|
||||
)
|
||||
return next_local_pos == 0
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _compute_local_logits_stats_kernel(
|
||||
# [num_logits, num_blocks]
|
||||
@@ -225,17 +235,17 @@ def _compute_local_logits_stats_kernel(
|
||||
# [max_num_reqs]
|
||||
temp_ptr,
|
||||
vocab_size,
|
||||
num_speculative_steps,
|
||||
num_logits,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
HAS_DRAFT_LOGITS: tl.constexpr,
|
||||
):
|
||||
logit_idx = tl.program_id(0).to(tl.int64)
|
||||
draft_step_idx = tl.load(expanded_local_pos_ptr + logit_idx)
|
||||
|
||||
if draft_step_idx >= num_speculative_steps:
|
||||
# Bonus token. Max/argmax and summed exponentials are not needed.
|
||||
if _is_bonus_logit(expanded_local_pos_ptr, logit_idx, num_logits):
|
||||
# Max/argmax and summed exponentials are not needed.
|
||||
return
|
||||
|
||||
draft_step_idx = tl.load(expanded_local_pos_ptr + logit_idx)
|
||||
req_state_idx = tl.load(expanded_idx_mapping_ptr + logit_idx).to(tl.int64)
|
||||
temp = tl.load(temp_ptr + req_state_idx).to(tl.float32)
|
||||
|
||||
@@ -423,17 +433,17 @@ def _compute_local_residual_mass_kernel(
|
||||
# [max_num_reqs]
|
||||
temp_ptr,
|
||||
vocab_size,
|
||||
num_speculative_steps,
|
||||
num_logits,
|
||||
vocab_num_blocks,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
PADDED_VOCAB_NUM_BLOCKS: tl.constexpr,
|
||||
):
|
||||
logit_idx = tl.program_id(0).to(tl.int64)
|
||||
# Adaptive verification can end before num_speculative_steps.
|
||||
if logit_idx + 1 >= tl.num_programs(0):
|
||||
return
|
||||
draft_step_idx = tl.load(expanded_local_pos_ptr + logit_idx)
|
||||
if draft_step_idx == 0 or draft_step_idx >= num_speculative_steps:
|
||||
if draft_step_idx == 0 or _is_bonus_logit(
|
||||
expanded_local_pos_ptr, logit_idx, num_logits
|
||||
):
|
||||
# The acceptance threshold, h, looks one position ahead and sums
|
||||
# over: max(p_i * M_b(x|x_{<i}) - M_s(x|x_{<i}), 0). Tokens at the
|
||||
# first and last (bonus) positions aren't needed for this computation.
|
||||
@@ -1138,7 +1148,7 @@ def rejection_sample(
|
||||
expanded_local_pos,
|
||||
temperature,
|
||||
vocab_size,
|
||||
num_speculative_steps,
|
||||
num_logits,
|
||||
BLOCK_SIZE=VOCAB_BLOCK_SIZE,
|
||||
HAS_DRAFT_LOGITS=has_draft_logits,
|
||||
)
|
||||
@@ -1209,7 +1219,7 @@ def rejection_sample(
|
||||
expanded_local_pos,
|
||||
temperature,
|
||||
vocab_size,
|
||||
num_speculative_steps,
|
||||
num_logits,
|
||||
vocab_num_blocks,
|
||||
BLOCK_SIZE=VOCAB_BLOCK_SIZE,
|
||||
PADDED_VOCAB_NUM_BLOCKS=padded_vocab_num_blocks,
|
||||
|
||||
@@ -6174,27 +6174,23 @@ class GPUModelRunner(
|
||||
gc.collect()
|
||||
|
||||
def _init_minimal_kv_cache_for_profiling(self) -> None:
|
||||
from vllm.v1.core.kv_cache_planning import (
|
||||
get_kv_cache_config_from_groups,
|
||||
get_kv_cache_groups,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_config_builder import get_kv_cache_config_builder
|
||||
from vllm.v1.core.kv_cache_planning import get_kv_cache_groups_from_builder
|
||||
|
||||
kv_cache_spec = self.get_kv_cache_spec()
|
||||
KVCacheSpecRegistry.check_kv_cache_spec_registry(kv_cache_spec)
|
||||
kv_cache_groups = get_kv_cache_groups(self.vllm_config, kv_cache_spec)
|
||||
builder = get_kv_cache_config_builder(self.vllm_config)
|
||||
kv_cache_groups = get_kv_cache_groups_from_builder(
|
||||
builder, self.vllm_config, kv_cache_spec
|
||||
)
|
||||
# the minimum number of blocks required is 1 block *per sequence*
|
||||
min_blocks = (
|
||||
min(self.max_num_reqs, self.compilation_config.max_cudagraph_capture_size)
|
||||
or 1
|
||||
)
|
||||
|
||||
# Temporarily change num_gpu_blocks_override to allocate a minimal KV cache
|
||||
saved_override = self.cache_config.num_gpu_blocks_override
|
||||
self.cache_config.num_gpu_blocks_override = min_blocks
|
||||
minimal_config = get_kv_cache_config_from_groups(
|
||||
self.vllm_config, kv_cache_groups, available_memory=0
|
||||
minimal_config = builder.get_kv_cache_config_from_groups(
|
||||
self.vllm_config, kv_cache_groups, num_blocks=min_blocks
|
||||
)
|
||||
self.cache_config.num_gpu_blocks_override = saved_override
|
||||
|
||||
self.initialize_kv_cache(minimal_config, is_profiling=True)
|
||||
self.cache_config.num_gpu_blocks = minimal_config.num_blocks
|
||||
|
||||
Reference in New Issue
Block a user