Merge branch 'main' into patches

This commit is contained in:
2026-10-17 08:06:56 +00:00
222 changed files with 14823 additions and 3606 deletions
+160 -33
View File
@@ -21,7 +21,7 @@ from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
)
if TYPE_CHECKING:
from aiter import ActivationType, QuantType
from aiter import ActivationType
logger = init_logger(__name__)
@@ -1492,6 +1492,74 @@ def _rocm_aiter_rmsnorm_fp8_group_quant_fake(
)
def _rocm_aiter_fused_rms_fp8_per_tensor_static_quant_impl(
x: torch.Tensor,
weight: torch.Tensor,
epsilon: float,
scale: torch.Tensor,
quant_dtype: torch.dtype,
) -> torch.Tensor:
from aiter.ops.triton.fused_fp8_quant import fused_rms_fp8_per_tensor_static_quant
x_quant, _, _, _ = fused_rms_fp8_per_tensor_static_quant(
x,
weight,
epsilon,
scale,
None,
None,
None,
dtype_quant=quant_dtype,
res1=None,
)
return x_quant
def _rocm_aiter_fused_rms_fp8_per_tensor_static_quant_fake(
x: torch.Tensor,
weight: torch.Tensor,
epsilon: float,
scale: torch.Tensor,
quant_dtype: torch.dtype,
) -> torch.Tensor:
return torch.empty_like(x, dtype=quant_dtype)
def _rocm_aiter_fused_add_rms_fp8_per_tensor_static_quant_impl(
x: torch.Tensor,
residual: torch.Tensor,
weight: torch.Tensor,
epsilon: float,
scale: torch.Tensor,
quant_dtype: torch.dtype,
) -> tuple[torch.Tensor, torch.Tensor]:
from aiter.ops.triton.fused_fp8_quant import fused_rms_fp8_per_tensor_static_quant
x_quant, _, _, residual_out = fused_rms_fp8_per_tensor_static_quant(
x,
weight,
epsilon,
scale,
None,
None,
None,
dtype_quant=quant_dtype,
res1=residual,
)
return x_quant, residual_out
def _rocm_aiter_fused_add_rms_fp8_per_tensor_static_quant_fake(
x: torch.Tensor,
residual: torch.Tensor,
weight: torch.Tensor,
epsilon: float,
scale: torch.Tensor,
quant_dtype: torch.dtype,
) -> tuple[torch.Tensor, torch.Tensor]:
return torch.empty_like(x, dtype=quant_dtype), torch.empty_like(residual)
def _rocm_aiter_fused_rms_gated_fp8_group_quant_impl(
x: torch.Tensor,
weight: torch.Tensor,
@@ -2202,38 +2270,6 @@ class rocm_aiter_ops:
}
return mapping.get(name)
@staticmethod
def get_aiter_quant_type(quant_type_str: str) -> "QuantType | None":
"""Given a quantization type as a string, returns the corresponding aiter QuantType enum.
Supported quantization types: "no", "per_tensor", "per_token", "per_1x32", "per_1x128", "per_128x128".
Returns None if the mapping fails.
Args:
quant_type_str (str): Quantization type as string.
Returns:
Aiter QuantType enum value, or None if not found.
"""
try:
from aiter import QuantType
except ImportError:
return None
if not isinstance(quant_type_str, str):
return None
name = quant_type_str.strip().lower()
mapping = {
"no": QuantType.No,
"per_tensor": QuantType.per_Tensor,
"per_token": QuantType.per_Token,
"per_1x32": QuantType.per_1x32,
"per_1x128": QuantType.per_1x128,
"per_128x128": QuantType.per_128x128,
}
return mapping.get(name)
@classmethod
@if_aiter_supported
def is_enabled(cls) -> bool:
@@ -2570,6 +2606,27 @@ class rocm_aiter_ops:
return cls.is_linear_enabled() and on_gfx950()
@staticmethod
@functools.cache
def _kda_gluon_prefill_importable() -> bool:
try:
from aiter.ops.triton.attention.chunk_kda import chunk_kda # noqa: F401
return True
except ImportError:
return False
@classmethod
@if_aiter_supported
def is_kda_gluon_prefill_available(cls) -> bool:
from vllm.platforms.rocm import get_cdna_version
return (
cls._AITER_ENABLED
and get_cdna_version() >= 4
and cls._kda_gluon_prefill_importable()
)
@classmethod
def get_aiter_allreduce(cls):
"""Return the TP device communicator's AITER custom-allreduce if it has
@@ -3082,6 +3139,18 @@ class rocm_aiter_ops:
fake_impl=_rocm_aiter_rmsnorm_fp8_group_quant_fake,
)
direct_register_custom_op(
op_name="rocm_aiter_fused_rms_fp8_per_tensor_static_quant",
op_func=_rocm_aiter_fused_rms_fp8_per_tensor_static_quant_impl,
fake_impl=_rocm_aiter_fused_rms_fp8_per_tensor_static_quant_fake,
)
direct_register_custom_op(
op_name="rocm_aiter_fused_add_rms_fp8_per_tensor_static_quant",
op_func=_rocm_aiter_fused_add_rms_fp8_per_tensor_static_quant_impl,
fake_impl=_rocm_aiter_fused_add_rms_fp8_per_tensor_static_quant_fake,
)
direct_register_custom_op(
op_name="rocm_aiter_fused_rms_gated_fp8_group_quant",
op_func=_rocm_aiter_fused_rms_gated_fp8_group_quant_impl,
@@ -3211,6 +3280,16 @@ class rocm_aiter_ops:
def get_rmsnorm_group_fused_quant_op() -> OpOverload:
return torch.ops.vllm.rocm_aiter_rmsnorm_fp8_group_quant.default
@staticmethod
def get_fused_rms_fp8_per_tensor_static_quant_op() -> OpOverload:
return torch.ops.vllm.rocm_aiter_fused_rms_fp8_per_tensor_static_quant.default
@staticmethod
def get_fused_add_rms_fp8_per_tensor_static_quant_op() -> OpOverload:
return (
torch.ops.vllm.rocm_aiter_fused_add_rms_fp8_per_tensor_static_quant.default
)
@staticmethod
def get_fused_rms_gated_fp8_group_quant_op() -> OpOverload:
"""Return the fused gated-RMSNorm + FP8 group quant custom op."""
@@ -3224,6 +3303,10 @@ class rocm_aiter_ops:
def get_per_token_quant_op() -> OpOverload:
return torch.ops.vllm.rocm_aiter_per_token_quant.default
@staticmethod
def get_per_tensor_quant_op() -> OpOverload:
return torch.ops.vllm.rocm_aiter_per_tensor_quant.default
@staticmethod
def get_group_quant_op() -> OpOverload:
return torch.ops.vllm.rocm_aiter_group_fp8_quant.default
@@ -5049,6 +5132,50 @@ class rocm_aiter_ops:
layer_input.view(*outer_shape, hidden_size),
)
@staticmethod
def kda_chunk_prefill(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
g: torch.Tensor,
beta: torch.Tensor,
A_log: torch.Tensor,
dt_bias: torch.Tensor | None,
lower_bound: float | None,
cu_seqlens: torch.Tensor | None,
chunk_indices: torch.Tensor | None,
chunk_offsets: torch.Tensor | None,
scale: float,
out: torch.Tensor | None,
initial_state: torch.Tensor | None,
output_final_state: bool,
state_cache: torch.Tensor | None,
state_indices: torch.Tensor | None,
has_initial_state: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
from aiter.ops.triton.attention.chunk_kda import chunk_kda
return chunk_kda(
q=q,
k=k,
v=v,
g=g,
beta=beta,
A_log=A_log,
dt_bias=dt_bias,
lower_bound=lower_bound,
cu_seqlens=cu_seqlens,
chunk_indices=chunk_indices,
chunk_offsets=chunk_offsets,
scale=scale,
out=out,
initial_state=initial_state,
output_final_state=output_final_state,
state_cache=state_cache,
state_indices=state_indices,
has_initial_state=has_initial_state,
)
rocm_aiter_ops.register_ops_once()
_sync_aiter_situv2_moe_env()
+5
View File
@@ -2424,6 +2424,11 @@ def grouped_topk(
raise NotImplementedError(
"The fused grouped_topk kernel is only available on CUDA and XPU platforms"
)
if scores.shape[0] == 0:
return (
scores.new_empty((0, topk), dtype=torch.float32),
scores.new_empty((0, topk), dtype=torch.int32),
)
return torch.ops._moe_C.grouped_topk(
scores,
num_expert_group,
+8
View File
@@ -120,6 +120,14 @@ def _make_engine_args(args: argparse.Namespace, cases: list[BenchCase]):
def run_case(runner, case: BenchCase, args: argparse.Namespace) -> dict[str, Any]:
required_drafts = max((r.num_draft_tokens for r in case.requests), default=0)
if required_drafts > runner.num_speculative_steps:
raise ValueError(
f"Case {case.name!r} needs {required_drafts} draft tokens per request, "
f"but the runner supports {runner.num_speculative_steps}. "
"Set --speculative-config with a sufficient num_speculative_tokens."
)
from vllm.benchmarks.forward.batch import build_batch
from vllm.benchmarks.forward.capture import ForwardBench
from vllm.benchmarks.forward.kv_init import touched_kv_views, zero_kv_blocks
@@ -394,10 +394,13 @@ class MatcherQuantFP8(MatcherCustomOp):
self.is_tma_aligned = is_tma_aligned
if match_rocm_aiter:
assert not quant_key.scale.group_shape.is_per_tensor(), (
"ROCm aiter fusion pass does not support per tensor quantization"
)
if quant_key.scale.group_shape.is_per_token():
if quant_key.scale.group_shape.is_per_tensor():
assert quant_key.scale.static, (
"ROCm aiter fusion pass only supports static per-tensor "
"quantization"
)
self.QUANT_OP = rocm_aiter_ops.get_per_tensor_quant_op()
elif quant_key.scale.group_shape.is_per_token():
self.QUANT_OP = rocm_aiter_ops.get_per_token_quant_op()
else:
assert quant_key.scale.group_shape.col == 128, (
@@ -432,14 +435,24 @@ class MatcherQuantFP8(MatcherCustomOp):
scale: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
quant_key_group_shape = self.quant_key.scale.group_shape
if quant_key_group_shape == GroupShape.PER_TENSOR:
# rocm_aiter_per_tensor_quant mutates `out` and `scale` and returns
# None, so it has to be matched through auto_functionalized.
assert scale is not None
out = torch.empty_like(input, dtype=self.quant_key.dtype)
result = auto_functionalized(
self.QUANT_OP, out=out, x=input, scale=scale, is_dynamic=False
)
# `scale` is mutable, so its functionalized output is a distinct
# value; return it to keep the quant node inside the match.
return result[1], result[2]
if quant_key_group_shape == GroupShape.PER_TOKEN:
return self.QUANT_OP( # type: ignore[no-any-return]
x=input,
quant_dtype=self.quant_key.dtype,
scale=scale,
)
else:
return self.QUANT_OP(input, quant_key_group_shape.col) # type: ignore[no-any-return]
return self.QUANT_OP(input, quant_key_group_shape.col) # type: ignore[no-any-return]
def forward_custom(
self,
@@ -19,6 +19,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
ScaleDesc,
kFp8Dynamic128Sym,
kStaticTensorScale,
)
from vllm.platforms import current_platform
@@ -561,6 +562,140 @@ class AiterRMSNormGatedFp8GroupQuantPattern(AiterRMSNormQuantPattern):
)
class AiterRMSNormStaticQuantPattern(AiterRMSNormQuantPattern):
"""This pattern fuses rms_norm & aiter static per-tensor fp8 quant
into an aiter fused_rms_fp8_per_tensor_static_quant op.
"""
FUSED_OP = rocm_aiter_ops.get_fused_rms_fp8_per_tensor_static_quant_op()
def __init__(
self,
epsilon: float,
quant_dtype: torch.dtype,
symmetric: bool = True,
) -> None:
key = FusedRMSQuantKey(
fused_add=False,
quant=QuantKey(
dtype=quant_dtype, scale=kStaticTensorScale, symmetric=symmetric
),
)
super().__init__(epsilon, key)
def register(self, pm_pass: PatternMatcherPass) -> None:
def pattern(
input: torch.Tensor,
weight: torch.Tensor,
scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
result_rms = torch.ops.vllm_ir.rms_norm(input, weight, self.epsilon)
result, scale = self.quant_matcher(result_rms, scale)
return result, scale
def replacement(
input: torch.Tensor,
weight: torch.Tensor,
scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
result = self.FUSED_OP(
x=input,
weight=weight,
epsilon=self.epsilon,
scale=scale,
quant_dtype=self.quant_dtype,
)
# The static scale is an input, so it passes through unchanged.
return result, scale
inputs = [
self.empty(5, 16), # input
self.empty(16), # weight
self.empty_f32(1, 1), # scale
]
pm.register_replacement(
pattern,
replacement,
inputs,
pm.fwd_only,
pm_pass,
)
class AiterFusedAddRMSNormStaticQuantPattern(AiterRMSNormQuantPattern):
"""This pattern fuses fused_add_rms_norm & aiter static per-tensor fp8
quant into an aiter fused_add_rms_fp8_per_tensor_static_quant op.
"""
FUSED_OP = rocm_aiter_ops.get_fused_add_rms_fp8_per_tensor_static_quant_op()
def __init__(
self,
epsilon: float,
quant_dtype: torch.dtype,
symmetric: bool = True,
) -> None:
key = FusedRMSQuantKey(
fused_add=True,
quant=QuantKey(
dtype=quant_dtype, scale=kStaticTensorScale, symmetric=symmetric
),
)
super().__init__(epsilon, key)
def register(self, pm_pass: PatternMatcherPass) -> None:
def pattern(
input: torch.Tensor,
weight: torch.Tensor,
residual: torch.Tensor,
scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
result_rms, residual_out = torch.ops.vllm_ir.fused_add_rms_norm(
input, residual, weight, self.epsilon
)
result, scale = self.quant_matcher(result_rms, scale)
return result, residual_out, scale
def replacement(
input: torch.Tensor,
weight: torch.Tensor,
residual: torch.Tensor,
scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
result = self.FUSED_OP(
x=input,
residual=residual,
weight=weight,
epsilon=self.epsilon,
scale=scale,
quant_dtype=self.quant_dtype,
)
# The static scale is an input, so it passes through unchanged.
return result[0], result[1], scale
inputs = [
self.empty(5, 16), # input
self.empty(16), # weight
self.empty(5, 16), # residual
self.empty_f32(1, 1), # scale
]
pm.register_replacement(
pattern,
replacement,
inputs,
pm.fwd_only,
pm_pass,
)
class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
"""This pass fuses aiter rms_norm & vllm/aiter quant custom ops
into a fused rms_norm_quant op.
@@ -630,6 +765,14 @@ class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
epsilon, FP8_DTYPE, GroupShape(1, 128), match_aiter_quant_op
).register(self.patterns)
# Fuse aiter rms_norm + aiter static per-tensor fp8 quant
AiterRMSNormStaticQuantPattern(epsilon, FP8_DTYPE).register(self.patterns)
# Fuse aiter fused_add_rms_norm + aiter static per-tensor fp8 quant
AiterFusedAddRMSNormStaticQuantPattern(epsilon, FP8_DTYPE).register(
self.patterns
)
# When quant_fp8 custom ops are disabled, both AITER and native
# quant matchers trace through QuantFP8's native implementation.
# Registering both variants would create duplicate Inductor
@@ -690,6 +833,8 @@ class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
AiterFusedAddRMSNormDynamicQuantPattern,
AiterRMSFp8GroupQuantPattern,
AiterFusedAddRMSFp8GroupQuantPattern,
AiterRMSNormStaticQuantPattern,
AiterFusedAddRMSNormStaticQuantPattern,
DoubleAiterRMSFp8GroupQuantPattern,
DoubleAiterRMSFp8GroupQuantViewPattern,
AiterRMSNormGatedFp8GroupQuantPattern,
+5
View File
@@ -85,6 +85,11 @@ class AttentionConfig:
use_prefill_query_quantization: bool = False
"""If set, quantize query for attention in prefill."""
trtllm_ragged_backend: Literal["trtllm-gen", "cute-dsl"] = "trtllm-gen"
"""Kernel backend for cached-context TRTLLM_RAGGED MLA prefill.
CuTe DSL requires FP8 queries for head dimension 192 and does not support
head dimension 256. New-token prefill always uses TRTLLM."""
indexer_kv_dtype: IndexerKVDType = "auto"
"""Data type for the sparse-attention indexer K cache. "auto" picks the
model's default (bf16 for MiniMax M3, fp8 for the DeepSeek sparse
+4
View File
@@ -2034,6 +2034,10 @@ class ModelConfig:
def has_noops(self) -> bool:
return self._model_info.has_noops
@property
def supports_kv_cache_grouping(self) -> bool:
return self._model_info.supports_kv_cache_grouping
@property
def has_inner_state(self):
return self._model_info.has_inner_state
@@ -838,7 +838,7 @@ class ExternalElasticEPScaleCoordinator:
if prepared.dp_rank == 0:
self.client._coord_store = self.reconfig_store_ref
self.client._ensure_stats_update_task()
await self.client.first_req_send_socket.send(
await self.client.scale_send_socket.send(
msgspec.msgpack.encode(
("SCALE_ELASTIC_EP", bootstrap.new_data_parallel_size)
)
@@ -63,12 +63,13 @@ class WriteTask:
request_id: ReqId
transfer_id: TransferId
dst_engine_id: str
local_block_ids: list[int]
remote_block_ids_hint: list[int] | None
local_block_ids: BlockIds
remote_block_ids_hint: BlockIds | None
layer_name: str
event: torch.cuda.Event
remote_notify_port: int
remote_ip: str
remote_tp_size: int | None = None
multi_pod_hosts: list[str] = field(default_factory=list)
remote_dp_size_local: int = 0
remote_dp_size: int = 1
@@ -94,7 +95,7 @@ class LayerTransferPlan:
class RemoteAllocInfo:
"""Information about remote block allocation."""
block_ids: list[int]
block_ids: BlockIds | None
writes_done: int = 0
writes_expected: int | None = None
decode_dp_rank: int = 0
@@ -338,7 +338,10 @@ class MoRIIOConnector(KVConnectorBase_V1, SupportsHMA):
block_ids: BlockIds,
) -> tuple[bool, dict[str, Any] | None]:
assert self.connector_scheduler is not None
if self.connector_scheduler._has_mamba:
if (
self.connector_scheduler._has_mamba
and self.connector_scheduler.mode == MoRIIOMode.READ
):
attn_block_ids, mamba_block_groups = (
self.connector_scheduler.split_block_groups(block_ids)
)
@@ -420,6 +423,16 @@ class MoRIIOConnector(KVConnectorBase_V1, SupportsHMA):
)
self.connector_worker.wait_for_save(self._connector_metadata)
def has_pending_push_work(self) -> bool:
return (
self.mode == MoRIIOMode.WRITE
and self.connector_scheduler is not None
and bool(self.connector_scheduler._deferred_send_deadlines)
)
def get_loaded_kv_cache_group_ids(self, request: "Request") -> tuple[int, ...]:
return self._kv_cache_config.transfer_group_ids
def shutdown(self):
if self.connector_worker is not None:
self.connector_worker.shutdown()
@@ -463,11 +476,10 @@ def _validate_hybrid_speculation(vllm_config: VllmConfig) -> None:
speculative_config = vllm_config.speculative_config
if speculative_config is None:
return
# DSpark is allowlisted because it is the only hybrid speculative method
# validated end to end with MoRIIO READ.
# Other hybrid speculative methods need their own transfer validation.
if not speculative_config.use_dspark():
raise MoRIIOError(
"MoRIIO hybrid READ supports DSpark speculative decoding only, got "
"MoRIIO hybrid transfer supports DSpark speculative decoding only, got "
f"method={speculative_config.method!r}"
)
@@ -480,14 +492,14 @@ def _validate_mamba_specs(specs: Collection[MambaSpec]) -> MambaSpec | None:
for spec in specs:
if len(spec.shapes) != 2 or len(spec.dtypes) != 2:
raise MoRIIOError(
"MoRIIO hybrid READ supports exactly two Mamba states "
"MoRIIO hybrid transfer supports exactly two Mamba states "
"(conv and SSM), got "
f"{len(spec.shapes)} shapes and {len(spec.dtypes)} dtypes"
)
reference = specs[0]
if any(spec != reference for spec in specs[1:]):
raise MoRIIOError(
"MoRIIO hybrid READ requires all transferable Mamba groups "
"MoRIIO hybrid transfer requires all transferable Mamba groups "
"to use the same cache spec"
)
return reference
@@ -515,7 +527,9 @@ class MoRIIOConnectorScheduler:
self._has_mamba = bool(self._mamba_group_ids)
mamba_spec = None
if self._has_mamba:
if len(self._attn_group_ids) != 1:
if not self._attn_group_ids:
raise MoRIIOError("MoRIIO hybrid transfer requires an attention group")
if self.mode == MoRIIOMode.READ and len(self._attn_group_ids) != 1:
raise MoRIIOError(
"MoRIIO hybrid READ requires exactly one transferable "
"attention group, got "
@@ -529,11 +543,6 @@ class MoRIIOConnectorScheduler:
]
mamba_spec = _validate_mamba_specs(mamba_specs)
_validate_hybrid_speculation(vllm_config)
if self.mode != MoRIIOMode.READ:
raise MoRIIOError(
"MoRIIO hybrid (mamba/KDA) transfer is implemented for READ "
"mode only; set kv_connector_extra_config.read_mode=true"
)
self._num_ssm_scratch_blocks = (
mamba_spec.num_speculative_blocks if mamba_spec is not None else 0
)
@@ -581,11 +590,13 @@ class MoRIIOConnectorScheduler:
f"cache group spec(s): {sorted(set(unsupported))}. Pass "
"--disable-hybrid-kv-cache-manager to run without HMA."
)
if self.mode == MoRIIOMode.WRITE:
# TODO(simondanielsson): support HMA in WRITE mode
if self.mode == MoRIIOMode.WRITE and any(
isinstance(g.kv_cache_spec, SlidingWindowSpec)
for g in kv_cache_config.transfer_groups
):
raise NotImplementedError(
"MoRIIO WRITE mode does not support hybrid KV cache groups "
"(sliding-window attention). Use READ mode, or pass "
"MoRIIO WRITE mode does not support sliding-window "
"cache groups. Use READ mode, or pass "
"--disable-hybrid-kv-cache-manager."
)
@@ -784,10 +795,11 @@ class MoRIIOConnectorScheduler:
if not params.get("do_remote_prefill"):
return 0, False
if self.mode == MoRIIOMode.WRITE:
# MoriiO in write mode, no remote prefill. Hybrid models never get
# here: register_kv_caches refuses WRITE mode for them, so there is
# no recurrent-state accounting to do.
return num_external_tokens, True
if self._has_mamba:
# P pushes h(N-1). Do not publish it under the N-token hash
# when the scheduler caches a completed asynchronous load.
num_external_tokens = max(num_external_tokens - 1, 0)
return num_external_tokens, num_external_tokens > 0
# READ mode always recomputes the last token locally on the decoder.
#
@@ -1178,6 +1190,16 @@ class MoRIIOConnectorScheduler:
f"remote_notify_port={remote_notify_port!r})"
)
block_notify_list: list[int] | list[list[int]]
if self._has_mamba:
block_notify_list = list(
self._clamp_to_prompt_blocks(
request, blocks.get_block_ids()
)
)
else:
block_notify_list = blocks.get_block_ids()[0]
# Wide-EP multi-pod: a pod binds notify sockets only for
# its LOCAL ranks, so the port offset must use the per-pod
# local rank (% dp_local), not the global rank. Single-pod
@@ -1204,7 +1226,7 @@ class MoRIIOConnectorScheduler:
self.send_notify_block(
req_id=request.request_id,
transfer_id=request.kv_transfer_params["transfer_id"],
block_notify_list=blocks.get_block_ids()[0],
block_notify_list=block_notify_list,
host=_notify_host,
port=target_port,
)
@@ -1227,6 +1249,26 @@ class MoRIIOConnectorScheduler:
sliding-window groups are left unchanged.
"""
if self._has_mamba:
# WRITE preserves transfer-group order, including a separate drafter
# attention group. READ retains its attention-first wire format.
transfer_blocks = self.kv_cache_config.select_transfer_block_ids(block_ids)
dcp_size = self.vllm_config.parallel_config.decode_context_parallel_size
clamped_hybrid: list[list[int]] = []
for group, blocks in zip(
self.kv_cache_config.transfer_groups, transfer_blocks, strict=True
):
spec = group.kv_cache_spec
if isinstance(spec, MambaSpec):
clamped_hybrid.append(
clip_ssm_state_blocks(list(blocks), spec.num_speculative_blocks)
)
else:
block_size = spec.block_size * (dcp_size if spec.dcp_sharded else 1)
clamped_hybrid.append(
list(blocks[: cdiv(req.num_prompt_tokens, block_size)])
)
return clamped_hybrid
clamped = list(block_ids)
changed = False
for g, group_blocks in enumerate(block_ids):
@@ -1252,6 +1294,19 @@ class MoRIIOConnectorScheduler:
num_scheduled = scheduler_output.num_scheduled_tokens.get(req.request_id, 0)
return req.num_computed_tokens + num_scheduled >= req.num_prompt_tokens
def _get_write_blocks(
self, req: "Request", block_ids: BlockIds, scheduler_output: SchedulerOutput
) -> BlockIds:
if self._has_mamba:
# Align-mode state slots can move without appending a new block.
# Resolve the authoritative table at the final prefill step.
block_state = scheduler_output.kv_connector_block_state
assert block_state is not None
current_blocks = block_state.get_block_ids(req.request_id)
assert current_blocks is not None
block_ids = current_blocks
return self._clamp_to_prompt_blocks(req, block_ids)
def build_connector_meta(
self,
scheduler_output: SchedulerOutput,
@@ -1274,10 +1329,10 @@ class MoRIIOConnectorScheduler:
# reserved speculative-lookahead slack), so new_block_ids is None;
# still fall through to the final-chunk check below.
new_block_ids = scheduler_output.scheduled_cached_reqs.new_block_ids[i]
if new_block_ids is not None:
if new_block_ids is not None and not self._has_mamba:
existing_blocks = [
existing_blocks[g] + new_block_ids[g]
for g in range(len(new_block_ids))
old + new
for old, new in zip(existing_blocks, new_block_ids, strict=True)
]
self._reqs_need_pending_save[req_id] = (req, existing_blocks)
if not self._is_final_prefill_chunk(req, scheduler_output):
@@ -1287,7 +1342,9 @@ class MoRIIOConnectorScheduler:
kv_params = self._req_kv_params.pop(
req_id, req.kv_transfer_params or {}
)
save_block_ids = self._clamp_to_prompt_blocks(req, existing_blocks)
save_block_ids = self._get_write_blocks(
req, existing_blocks, scheduler_output
)
meta.add_new_req(
request_id=req_id,
local_block_ids=save_block_ids,
@@ -1328,7 +1385,7 @@ class MoRIIOConnectorScheduler:
if not self._is_final_prefill_chunk(req, scheduler_output):
self._reqs_need_pending_save[req_id] = (req, block_ids)
continue
save_block_ids = self._clamp_to_prompt_blocks(req, block_ids)
save_block_ids = self._get_write_blocks(req, block_ids, scheduler_output)
meta.add_new_req(
request_id=req_id,
local_block_ids=save_block_ids,
@@ -1425,7 +1482,7 @@ class MoRIIOConnectorScheduler:
"""Once a request is finished, determine whether request blocks
should be freed now or will be sent asynchronously and freed later.
``block_ids`` normally contains all cache groups. Hybrid callers from
``block_ids`` normally contains all cache groups. Hybrid READ callers from
``request_finished_all_groups`` pass a flat attention list and the
recurrent-state groups separately in ``mamba_block_groups``.
"""
@@ -1494,6 +1551,10 @@ class MoRIIOConnectorScheduler:
list(cast(list[int], block_ids)),
*[list(group) for group in mamba_block_groups],
]
elif self._has_mamba and self.mode == MoRIIOMode.WRITE:
computed_block_ids = self._clamp_to_prompt_blocks(
request, cast(BlockIds, block_ids)
)
elif self._has_mamba:
raise MoRIIOError("Hybrid request completion requires all KV cache groups")
else:
@@ -1660,7 +1721,7 @@ class MoRIIOConnectorWorker:
_, mamba_transfer_group_ids = _split_kv_cache_group_kinds(kv_cache_config)
self._num_mamba_transfer_groups = len(mamba_transfer_group_ids)
self._mamba_payload_index_by_layer = {
layer_name: payload_index
layer_name: group_id if self.mode == MoRIIOMode.WRITE else payload_index
for payload_index, group_id in enumerate(mamba_transfer_group_ids, start=1)
for layer_name in kv_cache_config.transfer_groups[group_id].layer_names
}
@@ -1884,8 +1945,8 @@ class MoRIIOConnectorWorker:
request_id: ReqId,
transfer_id: TransferId,
dst_engine_id: str,
local_block_ids: list[int],
remote_block_ids: list[int] | None,
local_block_ids: BlockIds,
remote_block_ids: BlockIds | None,
layer_name: str,
kv_layer: torch.Tensor,
remote_notify_port: int,
@@ -1893,6 +1954,7 @@ class MoRIIOConnectorWorker:
multi_pod_hosts: list[str],
remote_dp_size_local: int,
remote_dp_size: int,
remote_tp_size: int | None = None,
) -> None:
"""Schedule a block write operation.
@@ -1909,6 +1971,7 @@ class MoRIIOConnectorWorker:
multi_pod_hosts: List of pod IPs for multi-pod Wide-EP
remote_dp_size_local: Per-pod DP size for multi-pod
remote_dp_size: Global DP size of the decode instance
remote_tp_size: Tensor parallel size advertised by the peer
"""
# synchronization to prevent dirty reads between
@@ -1933,6 +1996,7 @@ class MoRIIOConnectorWorker:
multi_pod_hosts=multi_pod_hosts,
remote_dp_size_local=remote_dp_size_local,
remote_dp_size=remote_dp_size,
remote_tp_size=remote_tp_size,
)
self._writer.schedule_write(task)
@@ -2359,15 +2423,6 @@ class MoRIIOConnectorWorker:
self._has_mamba = any(
self._is_mamba_layer(layer_name) for layer_name in kv_caches
)
if self._has_mamba and self.mode == MoRIIOMode.WRITE:
# WRITE needs a distinct conv/ssm transfer plan. Reject before
# allocating or registering any regions rather than silently
# serving a zero recurrent state.
raise MoRIIOError(
"MoRIIO hybrid (mamba/KDA) transfer is implemented for READ "
"mode only; set kv_connector_extra_config.read_mode=true "
"for hybrid models"
)
if self._has_mamba:
mamba_specs = [
cast(MambaSpec, self.layer_to_spec[layer_name])
@@ -3292,6 +3347,9 @@ class MoRIIOConnectorWorker:
)
def _write_blocks_for_req(self, req_id: ReqId, meta: ReqMeta, layer_name, kv_layer):
if self._has_mamba:
# Reject before even an attention-layer write can be queued.
self._mamba_tp_ratio(int(meta.tp_size))
# Compute per-request values to pass through the task (no shared state).
# This avoids race conditions when concurrent requests target different
# decode pods - each WriteTask carries its own routing info.
@@ -3303,16 +3361,12 @@ class MoRIIOConnectorWorker:
if meta.remote_dp_size_local
else int(meta.remote_dp_size)
)
# WRITE does not support HMA, so unwrap blocks into flat lists.
# remote_block_ids can itself be empty so need to be careful when unwrapping.
local_block_ids = meta.local_block_ids[0]
remote_block_ids = meta.remote_block_ids[0] if meta.remote_block_ids else []
self.schedule_write_blocks(
request_id=req_id,
transfer_id=meta.transfer_id,
dst_engine_id=meta.remote_engine_id,
local_block_ids=local_block_ids,
remote_block_ids=remote_block_ids,
local_block_ids=meta.local_block_ids,
remote_block_ids=meta.remote_block_ids,
layer_name=layer_name,
kv_layer=kv_layer,
remote_notify_port=meta.remote_notify_port,
@@ -3320,6 +3374,7 @@ class MoRIIOConnectorWorker:
multi_pod_hosts=hosts,
remote_dp_size_local=dp_local,
remote_dp_size=int(meta.remote_dp_size),
remote_tp_size=int(meta.tp_size),
)
def merge_contiguous_blocks(
@@ -3,6 +3,7 @@
import threading
import time
from collections import OrderedDict, defaultdict
from math import ceil
from queue import Empty, Queue
from typing import TYPE_CHECKING, Any
from weakref import ref as weakref_ref
@@ -61,6 +62,11 @@ try:
except ImportError:
logger.error("MoRIIO is not available")
try:
from mori.io import StatusCode
except ImportError:
StatusCode = None
"""Write task execution logic for MoRIIO connector."""
@@ -372,11 +378,15 @@ class MoRIIOWriter:
task.dst_engine_id
)
# Prepare transfer plan
plan = self._prepare_transfer_plan(task, request_info, remote_moriio_meta)
# Execute transfer
transfer_statuses = self._do_layer_write(plan, sessions)
if self.worker._is_mamba_layer(task.layer_name):
plans = self._prepare_mamba_transfer_plans(task, request_info)
else:
plans = [
self._prepare_transfer_plan(task, request_info, remote_moriio_meta)
]
transfer_statuses = []
for plan in plans:
transfer_statuses.extend(self._do_layer_write(plan, sessions))
with self._write_state_lock:
request_info.transfer_statuses.extend(transfer_statuses)
@@ -404,10 +414,17 @@ class MoRIIOWriter:
key = (task.layer_name, *_get_write_geometry_key(layer_cache))
offsets = request_info.transfer_offsets.get(key)
if offsets is None:
group = self.worker.layer_to_group[task.layer_name]
assert request_info.block_ids is not None
local_blocks = task.local_block_ids[group]
remote_blocks = request_info.block_ids[group]
if self.worker._has_mamba:
# D can reserve a tail block for the token recomputed locally.
remote_blocks = remote_blocks[: len(local_blocks)]
offsets = self.worker._compute_block_transfer_offsets(
task.layer_name,
task.local_block_ids,
request_info.block_ids,
local_blocks,
remote_blocks,
remote_moriio_meta,
remote_engine_id=task.dst_engine_id,
)
@@ -430,6 +447,44 @@ class MoRIIOWriter:
use_batch=True,
)
def _prepare_mamba_transfer_plans(
self, task: WriteTask, request_info: RemoteAllocInfo
) -> list[LayerTransferPlan]:
"""Plan both registered regions as one scheduled layer write."""
assert request_info.block_ids is not None
local_slots = self.worker._mamba_blocks_for_layer(
task.layer_name, task.local_block_ids
)
remote_slots = self.worker._mamba_blocks_for_layer(
task.layer_name, request_info.block_ids
)
if len(local_slots) != 1 or len(remote_slots) != 1:
raise MoRIIOError(
"Hybrid WRITE requires one running-state slot per Mamba group"
)
local, remote, sizes, n_conv = self.worker._compute_mamba_transfer_offsets(
task.layer_name,
local_slots,
remote_slots,
remote_tp_size=task.remote_tp_size,
)
conv_session, ssm_session = self.worker._region_session_indices(task.layer_name)
return [
LayerTransferPlan(
request_id=task.request_id,
transfer_id=task.transfer_id,
layer_name=task.layer_name,
sess_idx=session,
transfer_local_offsets=local[region],
transfer_remote_offsets=remote[region],
transfer_sizes=sizes[region],
)
for session, region in (
(conv_session, slice(0, n_conv)),
(ssm_session, slice(n_conv, None)),
)
]
def _do_layer_write(self, plan: LayerTransferPlan, sessions: list) -> list[Any]:
"""Perform the actual layer write.
@@ -438,6 +493,8 @@ class MoRIIOWriter:
sessions: List of transfer sessions
"""
if not plan.transfer_sizes:
return []
if plan.use_batch:
return [
self.worker.moriio_wrapper.write_remote_data(
@@ -721,7 +778,28 @@ class MoRIIOWrapper:
if not transfers_to_wait:
return
self._poll_transfers_until_done(transfers_to_wait)
wait_all = getattr(self.moriio_engine, "wait_all", None)
if wait_all is None or StatusCode is None:
self._poll_transfers_until_done(transfers_to_wait)
return
# Zero means a nonblocking progress call in Mori, not an immediate
# deadline. Round up so even sub-millisecond waits remain blocking.
timeout_ms = ceil(self._transfer_timeout * 1000)
result = wait_all(transfers_to_wait, timeout_ms=timeout_ms)
if result != StatusCode.SUCCESS:
details = [
f"{status.Message()} (code={status.Code()})"
for status in transfers_to_wait
if status.Failed()
]
if not details and result in (StatusCode.INIT, StatusCode.IN_PROGRESS):
details.append(
f"RDMA transfer timed out after {self._transfer_timeout:g}s"
)
raise TransferError(
f"MoRIIO batch wait failed ({result}): " + "; ".join(details)
)
def _poll_transfers_until_done(self, transfers_to_wait: list[Any]) -> None:
"""Fallback for mori builds without the batched wait.
@@ -858,7 +936,12 @@ class MoRIIOWrapper:
)
return
self.done_remote_allocate_req_dict[transfer_id] = RemoteAllocInfo(
block_ids=block_notify_list, decode_dp_rank=decode_dp_rank
block_ids=(
[block_notify_list]
if isinstance(block_notify_list[0], int)
else block_notify_list
),
decode_dp_rank=decode_dp_rank,
)
def _handle_write_done_message(self, data: dict):
@@ -125,11 +125,9 @@ class MultiKVConnectorPromMetrics(KVConnectorPromMetrics):
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
for connector_id, stats_data in transfer_stats_data.items():
assert connector_id in self._prom_metrics, (
f"{connector_id} is not contained in the list of registered connectors "
f"with Prometheus metrics support: {self._prom_metrics.keys()}"
)
self._prom_metrics[connector_id].observe(stats_data, engine_idx)
prom_metrics = self._prom_metrics.get(connector_id)
if prom_metrics is not None:
prom_metrics.observe(stats_data, engine_idx)
class MultiConnector(KVConnectorBase_V1, SupportsHMA):
@@ -101,6 +101,16 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
)
cpu_capacity_per_rank = explicit
# When set, num_cpu_blocks overrides automatic pool sizing from byte budgets
# (including cpu_bytes_to_use and cpu_bytes_to_use_per_rank), allowing ranks
# with different KV layouts (e.g. PP stages) to use the same pool size.
# Byte budgets still bound capacity.
num_cpu_blocks = extra_config.get("num_cpu_blocks")
if num_cpu_blocks is not None and (
type(num_cpu_blocks) is not int or num_cpu_blocks <= 0
):
raise ValueError("num_cpu_blocks must be a positive integer")
lazy_offload = bool(extra_config.get("lazy_offload", False))
kv_offload_backend = str(extra_config.get("kv_offload_backend", "cpu"))
@@ -160,6 +170,12 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
kv_offload_backend,
disk_path or "none",
)
if num_cpu_blocks is not None:
logger.info(
"Using explicit num_cpu_blocks=%d for offload pool sizing "
"instead of automatic sizing from byte budgets.",
num_cpu_blocks,
)
if role == KVConnectorRole.SCHEDULER:
from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes
@@ -177,6 +193,7 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
lazy_offload=lazy_offload,
disk_capacity_bytes=disk_capacity_bytes if disk_mode else 0,
use_page_cache=use_page_cache if disk_mode else False,
num_cpu_blocks=num_cpu_blocks,
)
elif role == KVConnectorRole.WORKER:
self.worker_handler = SimpleCPUOffloadWorker(
@@ -188,6 +205,7 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
disk_capacity_bytes=disk_capacity_bytes,
disk_buffer_slots=disk_buffer_slots,
use_page_cache=use_page_cache,
num_cpu_blocks=num_cpu_blocks,
)
# --- Worker-side methods ---
+646 -120
View File
@@ -9,16 +9,19 @@ runtime import, layout descriptors, and the conversion to `nccl.m2n` types —
so the engine module stays about the transfer itself.
"""
from collections.abc import Sequence
import hashlib
import json
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, cast
import torch
from vllm.distributed.weight_transfer.base import ParamMeta
from vllm.utils.nccl import unpinned_nccl_env
if TYPE_CHECKING:
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.distributed.utils import StatelessProcessGroup
# Placement code for a mesh axis that replicates. Any other (non-negative) code
# is the tensor dim that axis shards. Ints rather than `nccl.m2n` objects so
@@ -35,15 +38,15 @@ MESH_NDIMS = 2
DESTINATION_REPLICA_AXIS = 0
DESTINATION_SHARD_AXIS = 1
# m2n bounds how many source shards may feed one destination shard, and how many
# destination shards one source shard may feed, with compile-time arrays in
# `reshard_limits.h`. The bindings do not expose them, so they are mirrored
# here; a build with larger arrays makes these conservative, never wrong.
# Conservative capacities accepted by the supported nccl_m2n algorithms. They
# are not exposed by its Python API yet; remove these mirrors once nccl_m2n
# provides communication-free plan validation.
MAX_SOURCE_SHARDS = 16
MAX_DEST_SHARDS = 64
# Wire dtypes `ncclReshard` accepts. Notably excludes fp4 and any packed
# sub-byte type, so quantized checkpoints are out of scope for now.
# Increment when the JSON init payload changes incompatibly.
M2N_WIRE_SCHEMA_VERSION = 2
SUPPORTED_DTYPES: frozenset[torch.dtype] = frozenset(
dtype
for dtype in (
@@ -70,12 +73,62 @@ _IMPORT_HINT = (
"The nccl_m2n weight transfer backend requires the `nccl-extensions` "
"package (and its `nccl4py` dependency), which is not installed by vLLM. "
"See https://github.com/NVIDIA/nccl-extensions for build "
"and install instructions. It needs NCCL 2.30.5 or newer, and "
"VLLM_NCCL_SO_PATH must point at the same libnccl.so that libnccl_m2n.so "
"was linked against."
"and install instructions. It needs NCCL 2.30.5 or newer."
)
def prepare_m2n_local_runtime(m2n: Any, max_cta: int | None) -> tuple[Any, int, Any]:
"""Prepare rank-local resources before any rank enters NCCL init."""
import nccl.core as nccl
device = torch.accelerator.current_device_index()
if not isinstance(device, int) or isinstance(device, bool) or device < 0:
raise RuntimeError(f"invalid current accelerator device index: {device!r}")
handle = m2n.Handle.create(m2n.Config(max_cta=max_cta))
return nccl, device, handle
def create_m2n_communicator(
nccl: Any,
group: "StatelessProcessGroup",
unique_id_bytes: bytes | None,
) -> Any:
"""Create the M2N data-plane communicator through NCCL4Py."""
if unique_id_bytes is None:
envelope: dict[str, bytes | str | None] | None = None
if group.rank == 0:
try:
envelope = {
"unique_id": bytes(nccl.get_unique_id().as_bytes),
"error": None,
}
except Exception as exc:
envelope = {
"unique_id": None,
"error": f"{type(exc).__name__}: {exc}",
}
envelope = group.broadcast_obj(envelope, src=0)
if not isinstance(envelope, Mapping) or set(envelope) != {"unique_id", "error"}:
raise RuntimeError("invalid NCCL4Py unique-id envelope")
error = envelope["error"]
received_unique_id = envelope["unique_id"]
if error is not None:
if not isinstance(error, str) or not error:
raise RuntimeError("invalid NCCL4Py unique-id error")
raise RuntimeError(f"NCCL4Py unique-id generation failed: {error}")
if not isinstance(received_unique_id, bytes):
raise RuntimeError("NCCL4Py unique-id broadcast did not return bytes")
unique_id_bytes = received_unique_id
unique_id = nccl.UniqueId.from_bytes(unique_id_bytes)
with unpinned_nccl_env():
return nccl.Communicator.init(
nranks=group.world_size,
rank=group.rank,
unique_id=unique_id,
)
def import_m2n() -> Any:
"""Import `nccl.m2n` lazily, with an actionable error when it is missing.
@@ -97,17 +150,25 @@ class M2NMesh:
`[start_rank, start_rank + dims[0] * dims[1])`. There is no 1-D mesh; a
single-axis topology is spelled with a second axis of size 1.
One mesh describes every tensor on its side, so it is exchanged once at the
init handshake rather than per parameter.
A model may use a different factorization for each tensor. For example,
dense weights can use a DP x TP mesh while expert weights use EDP x EP.
"""
dims: tuple[int, int]
start_rank: int
def __post_init__(self) -> None:
if len(self.dims) != MESH_NDIMS or any(d <= 0 for d in self.dims):
object.__setattr__(self, "dims", tuple(self.dims))
if len(self.dims) != MESH_NDIMS or any(
not isinstance(dim, int) or isinstance(dim, bool) or dim <= 0
for dim in self.dims
):
raise ValueError(f"mesh dims must be {MESH_NDIMS} positive ints")
if self.start_rank < 0:
if (
not isinstance(self.start_rank, int)
or isinstance(self.start_rank, bool)
or self.start_rank < 0
):
raise ValueError(
f"mesh start_rank must be non-negative, got {self.start_rank}"
)
@@ -124,16 +185,26 @@ Placements = tuple[int, int]
REPLICATED: Placements | None = None
def check_placements(placements: Placements, context: str = "placements") -> None:
"""Reject placement pairs m2n cannot express.
@dataclass(frozen=True)
class M2NLayout:
"""A tensor's mesh and placement on one side of a transfer."""
The header requires exactly one SHARD axis and one REPLICATE axis.
`{REPLICATE, REPLICATE}` hits a degenerate prepare branch, which is why
full replication is carried as `REPLICATED` and resolved separately;
sharding both axes is not expressible at all.
"""
mesh: M2NMesh
placements: Placements | None
def __post_init__(self) -> None:
if not isinstance(self.mesh, M2NMesh):
raise TypeError("M2N layout mesh must be an M2NMesh")
if self.placements is not None:
object.__setattr__(self, "placements", tuple(self.placements))
def check_placements(placements: Placements, context: str = "placements") -> None:
"""Reject placement pairs nccl_m2n cannot express."""
if len(placements) != MESH_NDIMS:
raise ValueError(f"{context} must have {MESH_NDIMS} entries")
if any(not isinstance(code, int) or isinstance(code, bool) for code in placements):
raise TypeError(f"{context} must contain integer placement codes")
invalid = [code for code in placements if code < REPLICATE]
if invalid:
raise ValueError(
@@ -144,8 +215,7 @@ def check_placements(placements: Placements, context: str = "placements") -> Non
if num_sharded == 0:
raise ValueError(
f"{context}: a fully replicated tensor is carried as REPLICATED, "
"not as two "
"REPLICATE axes"
"not as two REPLICATE axes"
)
if num_sharded == MESH_NDIMS:
raise ValueError(
@@ -175,87 +245,46 @@ def resolve_layout(
return mesh, placements
def shard_count(mesh: M2NMesh, placements: Placements) -> int:
"""How many pieces this layout splits the tensor into."""
return next(
(mesh.dims[axis] for axis, code in enumerate(placements) if code != REPLICATE),
1,
def validate_layout(
mesh: M2NMesh,
placements: Placements,
shape: Sequence[int],
context: str,
) -> tuple[int, ...]:
"""Validate one resolved layout and return its required local shape."""
local_shape = list(shape)
for axis, tensor_dim in enumerate(placements):
if tensor_dim == REPLICATE:
continue
if tensor_dim >= len(shape):
raise ValueError(
f"{context} shards tensor dim {tensor_dim}, but parameter has "
f"rank {len(shape)}"
)
factor = mesh.dims[axis]
if shape[tensor_dim] % factor:
raise ValueError(
f"{context} shards dim {tensor_dim} (size {shape[tensor_dim]}) "
f"over {factor} ranks, which does not divide evenly"
)
local_shape[tensor_dim] //= factor
return tuple(local_shape)
def expected_local_shape(meta: "M2NParamMeta") -> tuple[int, ...]:
"""Validate a parameter's source layout and return its local shape."""
mesh, placements = resolve_layout(
meta.source_layout.mesh,
meta.source_layout.placements,
f"parameter '{meta.name}' source placements",
)
return validate_layout(
mesh, placements, meta.shape, f"parameter '{meta.name}' source layout"
)
def check_plan_limits(
src: tuple[M2NMesh, Placements],
dst: tuple[M2NMesh, Placements],
name: str,
) -> None:
"""Reject plans that exceed m2n's static per-shard fan-in / fan-out arrays.
Only the unambiguous cases are checked: when one side is a single shard it
is fed by (or feeds) every shard on the other side, so the count is exactly
the other side's shard count. In the general sharded-to-sharded case the
overlap depends on the library's chunking, and reproducing that arithmetic
here would duplicate internals that can change under us.
m2n re-checks authoritatively and, because the plan is derived from the
shared descriptors, fails identically on every rank -- so this is about
reporting at init with a message that names the parameter, not about
avoiding a hang.
"""
src_shards = shard_count(*src)
dst_shards = shard_count(*dst)
if dst_shards == 1 and src_shards > MAX_SOURCE_SHARDS:
raise ValueError(
f"parameter '{name}' would feed {src_shards} source shards into one "
f"replicated destination, over m2n's MAX_SOURCES={MAX_SOURCE_SHARDS}. "
"Shard the destination, replicate the source, or rebuild m2n with "
"larger arrays."
)
if src_shards == 1 and dst_shards > MAX_DEST_SHARDS:
raise ValueError(
f"parameter '{name}' would fan one source shard out to {dst_shards} "
f"destination shards, over m2n's MAX_TARGETS={MAX_DEST_SHARDS}. "
"Rebuild m2n with larger arrays to raise the bound."
)
def validate_layout(
mesh: M2NMesh, placements: Placements, shape: Sequence[int], side: str
) -> None:
"""Check a resolved layout can describe `shape` before any collective runs.
Called on both sides at init so a bad plan surfaces as an error from the
init RPC rather than as a hang inside the first reshard.
"""
for axis, code in enumerate(placements):
if code == REPLICATE:
continue
if code >= len(shape):
raise ValueError(
f"{side} layout shards tensor dim {code}, but the tensor "
f"has rank {len(shape)}"
)
factor = mesh.dims[axis]
if shape[code] % factor:
raise ValueError(
f"{side} layout shards dim {code} (size {shape[code]}) over "
f"{factor} ranks, which does not divide evenly"
)
@dataclass(frozen=True)
class M2NParamMeta(ParamMeta):
"""`ParamMeta` extended with how the trainer places this tensor.
The base class carries only name / dtype / full shape, which is not enough
to plan a reshard. `placements` is relative to its side's `M2NMesh`, or
`REPLICATED` when every rank holds the whole tensor.
"""
placements: Placements | None
def check_transferable(name: str, dtype: torch.dtype, shape: Sequence[int]) -> None:
"""Reject tensors m2n cannot move at all, with the parameter named."""
"""Reject parameters nccl_m2n cannot transfer."""
if dtype not in SUPPORTED_DTYPES:
raise ValueError(
f"parameter '{name}' has dtype {dtype}, which nccl_m2n does not "
@@ -268,6 +297,514 @@ def check_transferable(name: str, dtype: torch.dtype, shape: Sequence[int]) -> N
)
def _shard_spec(mesh: M2NMesh, placements: Placements) -> tuple[int, int]:
return next(
(code, mesh.dims[axis])
for axis, code in enumerate(placements)
if code != REPLICATE
)
def _max_partition_overlaps(
extent: int,
partitions: int,
other_partitions: int,
) -> int:
"""Return the maximum other partitions intersected by any one partition."""
if extent == 0:
return 0
width = extent // partitions
other_width = extent // other_partitions
max_overlaps = 0
for index in range(partitions):
start = index * width
end = start + width
first_other = start // other_width
last_other = (end - 1) // other_width
max_overlaps = max(max_overlaps, last_other - first_other + 1)
return max_overlaps
def check_plan_limits(
src: tuple[M2NMesh, Placements],
dst: tuple[M2NMesh, Placements],
name: str,
shape: Sequence[int],
) -> None:
"""Reject plans whose per-shard fan-in or fan-out exceeds M2N capacity."""
src_dim, src_shards = _shard_spec(*src)
dst_dim, dst_shards = _shard_spec(*dst)
if any(extent == 0 for extent in shape):
max_sources = max_targets = 0
elif src_dim != dst_dim:
max_sources = src_shards
max_targets = dst_shards
else:
extent = shape[src_dim]
max_sources = _max_partition_overlaps(extent, dst_shards, src_shards)
max_targets = _max_partition_overlaps(extent, src_shards, dst_shards)
if max_sources > MAX_SOURCE_SHARDS:
raise ValueError(
f"parameter '{name}' would feed up to {max_sources} source shards "
f"into one destination shard, over nccl_m2n MAX_SOURCES="
f"{MAX_SOURCE_SHARDS}"
)
if max_targets > MAX_DEST_SHARDS:
raise ValueError(
f"parameter '{name}' would fan one source shard out to up to "
f"{max_targets} destination shards, over nccl_m2n "
f"MAX_TARGETS={MAX_DEST_SHARDS}"
)
def validate_transfer_plan(
metas: Sequence["M2NParamMeta"],
dst_mesh: M2NMesh,
dst_placements: Sequence[Placements | None],
) -> None:
"""Validate the complete ordered source-to-destination transfer plan."""
if len(dst_placements) != len(metas):
raise ValueError(
f"destination plan has {len(dst_placements)} parameters, expected "
f"{len(metas)}"
)
for meta, placement in zip(metas, dst_placements):
check_transferable(meta.name, meta.dtype, meta.shape)
src = resolve_layout(
meta.source_layout.mesh,
meta.source_layout.placements,
f"parameter '{meta.name}' source placements",
)
validate_layout(*src, meta.shape, f"parameter '{meta.name}' source layout")
dst = resolve_layout(
dst_mesh,
placement,
f"parameter '{meta.name}' destination placements",
)
validate_layout(*dst, meta.shape, f"parameter '{meta.name}' destination layout")
check_plan_limits(src, dst, meta.name, meta.shape)
@dataclass(frozen=True)
class M2NParamMeta(ParamMeta):
"""`ParamMeta` extended with how the trainer places this tensor.
The base class carries only name / dtype / full shape, which is not enough
to plan a reshard. `source_layout` carries both the per-parameter mesh and
its placement on that mesh.
"""
source_layout: M2NLayout
def __post_init__(self) -> None:
if not isinstance(self.source_layout, M2NLayout):
raise TypeError("source_layout must be an M2NLayout")
@dataclass(frozen=True)
class M2NWireParam:
"""JSON-safe source metadata for one parameter in the init handshake."""
name: str
dtype_name: str
shape: tuple[int, ...]
src_mesh_dims: tuple[int, int]
src_placements: Placements | None
def __post_init__(self) -> None:
shape = tuple(self.shape)
dims = tuple(self.src_mesh_dims)
placements = None if self.src_placements is None else tuple(self.src_placements)
object.__setattr__(self, "shape", shape)
object.__setattr__(self, "src_mesh_dims", dims)
object.__setattr__(self, "src_placements", placements)
if not isinstance(self.name, str) or not self.name:
raise ValueError("M2N wire parameter name must not be empty")
if not isinstance(self.dtype_name, str) or not self.dtype_name:
raise ValueError(f"parameter '{self.name}' has an empty wire dtype name")
if not shape or any(
not isinstance(dim, int) or isinstance(dim, bool) or dim <= 0
for dim in shape
):
raise ValueError(
f"parameter '{self.name}' shape must contain positive integers"
)
M2NMesh(cast(tuple[int, int], dims), 0)
if placements is not None:
check_placements(
cast(Placements, placements),
f"parameter '{self.name}' source placements",
)
@classmethod
def from_dict(cls, value: Mapping[str, Any]) -> "M2NWireParam":
"""Parse one strict wire record from its JSON-decoded mapping."""
fields = {
"name",
"dtype_name",
"shape",
"src_mesh_dims",
"src_placements",
}
missing = fields - value.keys()
extra = value.keys() - fields
if missing or extra:
details = []
if missing:
details.append(f"missing {sorted(missing)}")
if extra:
details.append(f"unexpected {sorted(extra)}")
raise ValueError(f"invalid M2N wire parameter: {', '.join(details)}")
return cls(**value)
def to_dict(self) -> dict[str, Any]:
"""Serialize this record using only JSON-compatible values."""
return {
"name": self.name,
"dtype_name": self.dtype_name,
"shape": list(self.shape),
"src_mesh_dims": list(self.src_mesh_dims),
"src_placements": (
None if self.src_placements is None else list(self.src_placements)
),
}
def source_plan_digest(params: Sequence[M2NWireParam]) -> str:
"""Return a rank-independent digest of the ordered source wire plan."""
payload = json.dumps(
{
"schema_version": M2N_WIRE_SCHEMA_VERSION,
"params": [param.to_dict() for param in params],
},
sort_keys=True,
separators=(",", ":"),
).encode()
return hashlib.sha256(payload).hexdigest()
def complete_plan_digest(
source_digest: str,
dst_mesh: M2NMesh,
dst_placements: Sequence[Placements | None],
) -> str:
"""Return a digest of the agreed source and destination plan."""
if not isinstance(source_digest, str) or not source_digest:
raise ValueError("source digest must be a non-empty string")
payload = json.dumps(
{
"schema_version": M2N_WIRE_SCHEMA_VERSION,
"source_digest": source_digest,
"destination_mesh": {
"dims": list(dst_mesh.dims),
"start_rank": dst_mesh.start_rank,
},
"destination_placements": [
None if placement is None else list(placement)
for placement in dst_placements
],
},
sort_keys=True,
separators=(",", ":"),
).encode()
return hashlib.sha256(payload).hexdigest()
def check_source_plan_agreement(
group: "StatelessProcessGroup",
params: Sequence[M2NWireParam],
expected_digest: str | None = None,
local_error: str | None = None,
) -> str:
"""Verify every participant entered with the same source plan."""
prepared_error = local_error
digest: str | None = None
declared: str | None = expected_digest
if prepared_error is None:
try:
digest = source_plan_digest(params)
declared = digest if expected_digest is None else expected_digest
if not isinstance(declared, str) or not declared:
raise ValueError("declared source digest must be a non-empty string")
except Exception as exc:
prepared_error = f"{type(exc).__name__}: {exc}"
elif not isinstance(prepared_error, str) or not prepared_error:
prepared_error = "TypeError: local source error must be a non-empty string"
gathered = group.all_gather_obj(
{
"phase": "source",
"digest": digest,
"declared_digest": declared,
"error": prepared_error,
}
)
errors: list[str] = []
expected_fields = {"phase", "digest", "declared_digest", "error"}
if len(gathered) != group.world_size:
errors.append(
"control-plane all-gather returned "
f"{len(gathered)} records for world size {group.world_size}"
)
agreed: list[tuple[int, str, str]] = []
for rank, item in enumerate(gathered):
if not isinstance(item, Mapping) or set(item) != expected_fields:
errors.append(f"rank {rank}: invalid source preflight envelope")
continue
if item["phase"] != "source":
errors.append(f"rank {rank}: invalid source preflight phase")
continue
error = item["error"]
if error is not None:
if not isinstance(error, str) or not error:
errors.append(f"rank {rank}: invalid source preflight error")
else:
errors.append(f"rank {rank}: {error}")
continue
computed = item["digest"]
claimed = item["declared_digest"]
if (
not isinstance(computed, str)
or not computed
or not isinstance(claimed, str)
or not claimed
):
errors.append(f"rank {rank}: invalid source digest")
continue
agreed.append((rank, computed, claimed))
if errors:
raise RuntimeError(
"nccl_m2n source preflight rejected before NCCL init: " + "; ".join(errors)
)
canonical = agreed[0][1]
for rank, computed, claimed in agreed:
if computed != claimed:
errors.append(
f"rank {rank}: computed digest {computed} does not match "
f"declared digest {claimed}"
)
if computed != canonical:
errors.append(f"rank {rank}: source digest disagrees with rank 0")
if errors:
raise RuntimeError(
"nccl_m2n source preflight rejected before NCCL init: " + "; ".join(errors)
)
return canonical
def check_complete_plan_agreement(
group: "StatelessProcessGroup",
source_digest: str,
dst_mesh: M2NMesh,
dst_placements: Sequence[Placements | None],
local_error: str | None = None,
) -> str:
"""Require every rank to accept the same complete transfer plan."""
prepared_error = local_error
digest: str | None = None
if prepared_error is None:
try:
digest = complete_plan_digest(source_digest, dst_mesh, dst_placements)
except Exception as exc:
prepared_error = f"{type(exc).__name__}: {exc}"
elif not isinstance(prepared_error, str) or not prepared_error:
prepared_error = "TypeError: local plan error must be a non-empty string"
gathered = group.all_gather_obj(
{"phase": "complete_plan", "digest": digest, "error": prepared_error}
)
errors: list[str] = []
valid: list[tuple[int, str]] = []
expected_fields = {"phase", "digest", "error"}
if len(gathered) != group.world_size:
errors.append(
"control-plane all-gather returned "
f"{len(gathered)} records for world size {group.world_size}"
)
for rank, item in enumerate(gathered):
if (
not isinstance(item, Mapping)
or set(item) != expected_fields
or item.get("phase") != "complete_plan"
):
errors.append(f"rank {rank}: invalid complete-plan envelope")
continue
error = item["error"]
if error is not None:
if not isinstance(error, str) or not error:
errors.append(f"rank {rank}: invalid complete-plan error")
else:
errors.append(f"rank {rank}: {error}")
continue
rank_digest = item["digest"]
if not isinstance(rank_digest, str) or not rank_digest:
errors.append(f"rank {rank}: invalid complete-plan digest")
continue
valid.append((rank, rank_digest))
if not errors:
canonical = valid[0][1]
for rank, rank_digest in valid[1:]:
if rank_digest != canonical:
errors.append(
f"rank {rank}: complete-plan digest disagrees with rank 0"
)
if errors:
raise RuntimeError(
"nccl_m2n complete-plan preflight rejected before first reshard: "
+ "; ".join(errors)
)
return valid[0][1]
def check_runtime_ready_agreement(
group: "StatelessProcessGroup", local_error: str | None
) -> None:
"""Require every rank to prepare local resources before NCCL init."""
prepared_error = local_error
if prepared_error is not None and (
not isinstance(prepared_error, str) or not prepared_error
):
prepared_error = "TypeError: local runtime error must be a non-empty string"
gathered = group.all_gather_obj({"phase": "runtime_ready", "error": prepared_error})
errors: list[str] = []
if len(gathered) != group.world_size:
errors.append(
"control-plane all-gather returned "
f"{len(gathered)} records for world size {group.world_size}"
)
for rank, item in enumerate(gathered):
if (
not isinstance(item, Mapping)
or set(item) != {"phase", "error"}
or item.get("phase") != "runtime_ready"
):
errors.append(f"rank {rank}: invalid runtime readiness envelope")
continue
error = item.get("error")
if error is not None:
if not isinstance(error, str) or not error:
errors.append(f"rank {rank}: invalid runtime readiness error")
else:
errors.append(f"rank {rank}: {error}")
if errors:
raise RuntimeError(
"nccl_m2n local runtime preflight failed before NCCL init: "
+ "; ".join(errors)
)
def check_data_plane_agreement(
group: "StatelessProcessGroup",
unique_id_bytes: bytes | None,
local_error: str | None = None,
) -> None:
"""Agree on the NCCL rendezvous identity before NCCL init."""
prepared_error = local_error
uid_digest: str | None = None
if prepared_error is None:
try:
if unique_id_bytes is not None:
if not isinstance(unique_id_bytes, bytes):
raise TypeError("NCCL unique id must decode to bytes")
uid_digest = hashlib.sha256(unique_id_bytes).hexdigest()
except Exception as exc:
prepared_error = f"{type(exc).__name__}: {exc}"
elif not isinstance(prepared_error, str) or not prepared_error:
prepared_error = "TypeError: local data-plane error must be non-empty"
gathered = group.all_gather_obj(
{
"phase": "data_plane",
"mode": "uid" if unique_id_bytes is not None else "tcp",
"uid_digest": uid_digest,
"error": prepared_error,
}
)
expected_fields = {
"phase",
"mode",
"uid_digest",
"error",
}
errors: list[str] = []
valid: list[tuple[int, str, str | None]] = []
if len(gathered) != group.world_size:
errors.append(
"control-plane all-gather returned "
f"{len(gathered)} records for world size {group.world_size}"
)
for rank, item in enumerate(gathered):
if (
not isinstance(item, Mapping)
or set(item) != expected_fields
or item.get("phase") != "data_plane"
):
errors.append(f"rank {rank}: invalid data-plane preflight envelope")
continue
error = item["error"]
if error is not None:
if not isinstance(error, str) or not error:
errors.append(f"rank {rank}: invalid data-plane preflight error")
else:
errors.append(f"rank {rank}: {error}")
continue
mode = item["mode"]
uid = item["uid_digest"]
if mode not in {"tcp", "uid"} or (mode == "uid") != isinstance(uid, str):
errors.append(f"rank {rank}: invalid data-plane identity")
continue
valid.append((rank, mode, uid))
if not errors:
_, mode, uid = valid[0]
for rank, other_mode, other_uid in valid[1:]:
if (other_mode, other_uid) != (mode, uid):
errors.append(f"rank {rank}: NCCL data-plane identity disagrees")
if errors:
raise RuntimeError(
"nccl_m2n data-plane preflight rejected before NCCL init: "
+ "; ".join(errors)
)
def validate_local_buffer(m2n: Any, meta: M2NParamMeta, buffer: Any) -> None:
"""Check source dtype/shape while delegating buffer validity to M2N."""
# A raw pointer carries no inspectable metadata. The source owns its
# correctness; _send supplies the declared dtype and local shape to M2N.
if isinstance(buffer, int):
return
mesh, placements = resolve_layout(
meta.source_layout.mesh, meta.source_layout.placements
)
actual = m2n.DistTensor(
buffer,
mesh=to_mesh(m2n, mesh),
placements=to_placements(m2n, placements),
)
declared = m2n.DistTensor(
None,
local_shape=actual.local_shape,
dtype=meta.dtype,
mesh=to_mesh(m2n, mesh),
placements=to_placements(m2n, placements),
)
if actual.dtype != declared.dtype:
raise ValueError(
f"parameter '{meta.name}' source returned dtype {actual.dtype}, "
f"but its metadata declared {declared.dtype}"
)
expected = expected_local_shape(meta)
if actual.local_shape != expected:
raise ValueError(
f"parameter '{meta.name}' source returned local shape "
f"{actual.local_shape}, but layout {meta.source_layout} implies "
f"{expected} from global shape {meta.shape}"
)
def to_mesh(m2n: Any, mesh: M2NMesh) -> Any:
return m2n.Mesh(mesh.dims, start_rank=mesh.start_rank)
@@ -279,7 +816,9 @@ def to_placements(m2n: Any, placements: Placements) -> list[Any]:
def publish_destination_placements(
comm: "PyNcclCommunicator",
comm: Any,
device: torch.device,
stream: torch.cuda.Stream,
first_worker_rank: int,
placements: "Sequence[Placements | None] | None",
num_parameters: int,
@@ -294,15 +833,15 @@ def publish_destination_placements(
rather than deadlocking later.
Only placements travel: the destination *mesh* is still derived from the
rank split, identically on both sides. Use the NCCL communicator directly
because unique-id rendezvous deliberately has no bootstrap process group.
rank split, identically on both sides. The plan is published on the data
plane before any M2N reshard collective.
"""
replicated_sentinel = REPLICATE - 1
encoded = torch.full(
(num_parameters, MESH_NDIMS),
replicated_sentinel,
dtype=torch.int8,
device=comm.device,
device=device,
)
if comm.rank == first_worker_rank:
if placements is None or len(placements) != num_parameters:
@@ -314,25 +853,12 @@ def publish_destination_placements(
for placement in placements
]
if rows:
encoded.copy_(torch.tensor(rows, dtype=torch.int8, device=comm.device))
encoded.copy_(torch.tensor(rows, dtype=torch.int8, device=device))
comm.broadcast(encoded, src=first_worker_rank)
comm.broadcast(encoded, encoded, root=first_worker_rank, stream=stream)
return [
REPLICATED
if all(code == replicated_sentinel for code in row)
else cast(Placements, tuple(row))
for row in encoded.tolist()
]
def comm_ptr(comm: "PyNcclCommunicator") -> int:
"""Raw `ncclComm_t` behind vLLM's `PyNcclCommunicator`.
m2n links its own NCCL, so the handle only means anything if vLLM loaded
the same `libnccl.so` — set `VLLM_NCCL_SO_PATH` accordingly.
"""
handle = comm.comm
ptr = getattr(handle, "value", handle)
if not ptr:
raise RuntimeError("PyNcclCommunicator has no live NCCL communicator")
return int(ptr)
+288 -144
View File
@@ -20,7 +20,7 @@ inside `receive_weights` while the trainer is sending. Driving that from the
trainer is the trainer engine's job.
"""
from collections.abc import Iterator
from collections.abc import Iterator, Mapping
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, cast
@@ -34,20 +34,28 @@ from vllm.distributed.weight_transfer.base import (
)
from vllm.distributed.weight_transfer.m2n_common import (
DESTINATION_SHARD_AXIS,
M2N_WIRE_SCHEMA_VERSION,
MESH_NDIMS,
REPLICATED,
M2NLayout,
M2NMesh,
M2NParamMeta,
M2NWireParam,
Placements,
check_plan_limits,
check_complete_plan_agreement,
check_data_plane_agreement,
check_runtime_ready_agreement,
check_source_plan_agreement,
check_transferable,
comm_ptr,
create_m2n_communicator,
expected_local_shape,
import_m2n,
prepare_m2n_local_runtime,
publish_destination_placements,
resolve_layout,
source_plan_digest,
to_mesh,
to_placements,
validate_layout,
validate_transfer_plan,
)
from vllm.distributed.weight_transfer.m2n_layout import (
M2NDestination,
@@ -56,12 +64,14 @@ from vllm.distributed.weight_transfer.m2n_layout import (
from vllm.distributed.weight_transfer.nccl_common import (
NCCLWeightTransferInitInfo,
decode_nccl_unique_id,
worker_init_process_group,
worker_init_metadata_group,
)
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
logger = init_logger(__name__)
__all__ = [
"M2NWeightTransferInitInfo",
@@ -84,59 +94,99 @@ class M2NWeightTransferInitInfo(WeightTransferInitInfo):
plain JSON so the HTTP control plane carries it unchanged.
"""
master_address: str | None = None
master_port: int | None = None
schema_version: int
master_address: str
master_port: int
rank_offset: int
"""First worker rank, i.e. the number of trainer ranks."""
world_size: int
"""Trainer ranks + all inference workers."""
src_mesh_dims: list[int]
"""The trainer's mesh, shared by every parameter (`start_rank` is 0: the
trainer occupies the front of the communicator)."""
dst_mesh_dims: list[int]
"""The inference mesh, starting at `rank_offset`. Declared rather than
derived so both sides describe the destination identically."""
names: list[str]
dtype_names: list[str]
shapes: list[list[int]]
src_placements: list[list[int] | None]
"""Per parameter, relative to `src_mesh_dims`; `None` means replicated."""
source_digest: str
"""SHA-256 of the ordered, versioned source parameter plan."""
params: list[dict[str, Any]]
"""Ordered per-parameter source metadata and layouts."""
nccl_unique_id_b64: str | None = field(default=None, repr=False)
max_cta: int | None = None
def __post_init__(self) -> None:
_ = self.nccl_unique_id_bytes
num_params = len(self.names)
for label, values in (
("dtype_names", self.dtype_names),
("shapes", self.shapes),
("src_placements", self.src_placements),
if not isinstance(self.master_address, str) or not self.master_address:
raise ValueError("`master_address` must name the metadata rendezvous")
if (
not isinstance(self.master_port, int)
or isinstance(self.master_port, bool)
or not 0 < self.master_port < 65536
):
raise ValueError("`master_port` must be a valid metadata port")
if (
not isinstance(self.rank_offset, int)
or isinstance(self.rank_offset, bool)
or not isinstance(self.world_size, int)
or isinstance(self.world_size, bool)
or self.rank_offset < 1
or self.rank_offset >= self.world_size
):
if len(values) != num_params:
raise ValueError(
f"`{label}` should be of the same size as `names`: "
f"got {len(values)} and {num_params}"
)
if self.rank_offset < 1 or self.rank_offset >= self.world_size:
raise ValueError(
f"`rank_offset` ({self.rank_offset}) must leave at least one "
f"trainer rank and one worker in world_size {self.world_size}"
)
num_workers = self.world_size - self.rank_offset
dst = tuple(self.dst_mesh_dims)
if len(dst) != MESH_NDIMS or dst[0] * dst[1] != num_workers:
try:
dst_mesh = M2NMesh(cast(tuple[int, int], dst), self.rank_offset)
except (TypeError, ValueError) as exc:
raise ValueError(f"invalid `dst_mesh_dims` {dst}: {exc}") from exc
if dst_mesh.size != num_workers:
raise ValueError(
f"`dst_mesh_dims` {dst} must be {MESH_NDIMS} dims covering the "
f"{num_workers} inference workers"
)
def parse_wire_params(self) -> tuple[M2NWireParam, ...]:
"""Parse and authenticate the ordered source plan."""
if self.schema_version != M2N_WIRE_SCHEMA_VERSION:
raise ValueError(
"unsupported nccl_m2n init schema version "
f"{self.schema_version}; expected {M2N_WIRE_SCHEMA_VERSION}"
)
if not isinstance(self.params, list):
raise TypeError("`params` must be a list of wire parameter mappings")
if not isinstance(self.source_digest, str) or not self.source_digest:
raise ValueError("`source_digest` must be a non-empty string")
wire_params: list[M2NWireParam] = []
names: set[str] = set()
for index, value in enumerate(self.params):
if not isinstance(value, Mapping):
raise TypeError(
f"`params[{index}]` must be a mapping, got {type(value).__name__}"
)
param = M2NWireParam.from_dict(value)
if param.name in names:
raise ValueError(
f"nccl_m2n init params contain duplicate parameter '{param.name}'"
)
names.add(param.name)
wire_params.append(param)
parsed = tuple(wire_params)
computed_digest = source_plan_digest(parsed)
if self.source_digest != computed_digest:
raise ValueError(
"nccl_m2n init source digest does not match params: "
f"computed {computed_digest}, declared {self.source_digest}"
)
return parsed
@property
def nccl_unique_id_bytes(self) -> bytes | None:
if self.nccl_unique_id_b64 is None:
return None
return decode_nccl_unique_id(
master_address=self.master_address,
master_port=self.master_port,
master_address=None,
master_port=None,
nccl_unique_id_b64=self.nccl_unique_id_b64,
ctx="M2NWeightTransferInitInfo",
ctx="M2NWeightTransferInitInfo data plane",
)
@@ -182,33 +232,172 @@ class M2NWeightTransferEngine(
model: torch.nn.Module,
) -> None:
super().__init__(config, vllm_config, device, model)
self.model_update_group: PyNcclCommunicator | None = None
self.model_update_group: Any = None
self._m2n: Any = None
self._handle: Any = None
self._metas: list[M2NParamMeta] = []
self._src_mesh: M2NMesh | None = None
self._dst_mesh: M2NMesh | None = None
self._parameter_destinations: list[M2NDestination] = []
self._index: dict[str, int] = {}
self._uses_load_weights = False
def init_transfer_engine(self, init_info: M2NWeightTransferInitInfo) -> None:
"""Join the trainer's communicator and rebuild the transfer plan.
Every precondition (dtype, tensor rank, divisibility) is checked here so
a bad plan fails the init RPC instead of hanging the first collective.
"""
self._m2n = import_m2n()
self._metas = []
self._src_mesh = M2NMesh(
cast(tuple[int, int], tuple(init_info.src_mesh_dims)), 0
)
if self._src_mesh.size != init_info.rank_offset:
raise ValueError(
f"source mesh covers {self._src_mesh.size} ranks, but there are "
f"{init_info.rank_offset} trainer ranks"
"""Agree on source and local readiness before constructing NCCL."""
if self._handle is not None or self.model_update_group is not None:
raise RuntimeError(
"nccl_m2n weight transfer is already initialized; call "
"shutdown_weight_transfer_engine() before initializing again"
)
rendezvous_info = NCCLWeightTransferInitInfo(
master_address=init_info.master_address,
master_port=init_info.master_port,
rank_offset=init_info.rank_offset,
world_size=init_info.world_size,
)
metadata_group = worker_init_metadata_group(
rendezvous_info, self.parallel_config
)
wire_params: tuple[M2NWireParam, ...] = ()
source_error: str | None = None
try:
self._m2n = import_m2n()
wire_params = init_info.parse_wire_params()
self._prepare_source_plan(wire_params, init_info.rank_offset)
except Exception as exc:
source_error = f"{type(exc).__name__}: {exc}"
self._metas = []
check_source_plan_agreement(
metadata_group,
wire_params,
expected_digest=init_info.source_digest,
local_error=source_error,
)
destination_error: str | None = None
try:
self._prepare_destination_plan(init_info)
except Exception as exc:
destination_error = f"{type(exc).__name__}: {exc}"
self._parameter_destinations = []
unique_id_bytes: bytes | None = None
data_plane_error: str | None = None
try:
unique_id_bytes = init_info.nccl_unique_id_bytes
except Exception as exc:
data_plane_error = f"{type(exc).__name__}: {exc}"
check_data_plane_agreement(
metadata_group,
unique_id_bytes,
data_plane_error,
)
handle = None
group = None
try:
runtime_error = destination_error
nccl = None
device = None
if runtime_error is None:
try:
nccl, device, handle = prepare_m2n_local_runtime(
self._m2n, init_info.max_cta
)
except Exception as exc:
runtime_error = f"{type(exc).__name__}: {exc}"
check_runtime_ready_agreement(metadata_group, runtime_error)
assert nccl is not None and device is not None and handle is not None
group = create_m2n_communicator(nccl, metadata_group, unique_id_bytes)
# Keep the merged destination-placement publication unchanged: source
# agreement is independent of worker-side destination selection.
mine = [
destination.placements for destination in self._parameter_destinations
]
agreed = publish_destination_placements(
group,
self.device,
torch.cuda.current_stream(),
init_info.rank_offset,
mine,
len(self._metas),
)
complete_plan_error: str | None = None
try:
if agreed != mine:
mismatched = next(
meta.name
for meta, ours, theirs in zip(self._metas, mine, agreed)
if ours != theirs
)
raise RuntimeError(
"inference workers resolved different nccl_m2n "
f"destinations (first mismatch: '{mismatched}'); all "
"workers must run the same model and parallel config"
)
assert self._dst_mesh is not None
validate_transfer_plan(self._metas, self._dst_mesh, agreed)
except Exception as exc:
complete_plan_error = f"{type(exc).__name__}: {exc}"
assert self._dst_mesh is not None
check_complete_plan_agreement(
metadata_group,
init_info.source_digest,
self._dst_mesh,
agreed,
complete_plan_error,
)
except BaseException:
if group is not None:
try:
# Transport cleanup only; reload validity belongs to the
# model-update lifecycle, not this backend.
group.abort()
except BaseException:
logger.exception(
"failed to abort partially initialized nccl_m2n communicator"
)
if handle is not None:
try:
handle.destroy()
except BaseException:
logger.exception(
"failed to destroy partially initialized nccl_m2n handle"
)
raise
self._handle = handle
self.model_update_group = group
def _prepare_source_plan(
self,
wire_params: tuple[M2NWireParam, ...],
num_trainer_ranks: int,
) -> None:
"""Validate worker-visible source metadata without constructing NCCL."""
self._metas = []
for param in wire_params:
dtype = getattr(torch, param.dtype_name, None)
if not isinstance(dtype, torch.dtype):
raise ValueError(
f"parameter '{param.name}' has invalid dtype name "
f"'{param.dtype_name}'; expected the name of a torch.dtype"
)
check_transferable(param.name, dtype, param.shape)
layout = M2NLayout(M2NMesh(param.src_mesh_dims, 0), param.src_placements)
if layout.mesh.size != num_trainer_ranks:
raise ValueError(
f"parameter '{param.name}' source mesh covers "
f"{layout.mesh.size} ranks, but there are "
f"{num_trainer_ranks} trainer ranks"
)
meta = M2NParamMeta(param.name, dtype, param.shape, layout)
expected_local_shape(meta)
self._metas.append(meta)
def _prepare_destination_plan(self, init_info: M2NWeightTransferInitInfo) -> None:
"""Resolve the merged-#51520 in-place/full-fallback destination plan."""
# The inference topology, as declared by the trainer. This is the mesh
# a sharded destination is placed over; a replicated one is described
# by the alternate descriptor `resolve_layout` derives from it, which
@@ -217,33 +406,6 @@ class M2NWeightTransferEngine(
cast(tuple[int, int], tuple(init_info.dst_mesh_dims)),
init_info.rank_offset,
)
for name, name_dtype, shape, placements in zip(
init_info.names,
init_info.dtype_names,
init_info.shapes,
init_info.src_placements,
):
dtype = getattr(torch, name_dtype, None)
if not isinstance(dtype, torch.dtype):
raise ValueError(
f"parameter '{name}' has invalid dtype name "
f"'{name_dtype}'; expected the name of a torch.dtype"
)
check_transferable(name, dtype, shape)
codes = (
REPLICATED
if placements is None
else cast(Placements, tuple(placements))
)
src_mesh, src_placements = resolve_layout(
self._src_mesh, codes, f"parameter '{name}' source placements"
)
validate_layout(src_mesh, src_placements, shape, "source")
# Destination checks are deferred until loader-aware resolution;
# assuming replication here can reject a valid sharded plan.
self._metas.append(M2NParamMeta(name, dtype, tuple(shape), codes))
parallel_config = self.parallel_config
shard_axis_size = self._dst_mesh.dims[DESTINATION_SHARD_AXIS]
quantization = getattr(self.model_config, "quantization", None)
@@ -266,27 +428,11 @@ class M2NWeightTransferEngine(
shard_axis_size=shard_axis_size,
allow_direct=allow_direct,
)
# Check the resolved layout rather than a provisional replicated one.
# For example, 32 source shards feeding 4 destination shards may need
# only 8 sources per destination, while a replicated plan needs all 32.
for meta, destination in zip(self._metas, self._parameter_destinations):
src_mesh, src_placements = resolve_layout(
self._src_mesh,
meta.placements,
f"parameter '{meta.name}' source placements",
)
dst_mesh, dst_placements = resolve_layout(
self._dst_mesh,
destination.placements,
f"parameter '{meta.name}' destination placements",
)
validate_layout(dst_mesh, dst_placements, meta.shape, "destination")
check_plan_limits(
(src_mesh, src_placements),
(dst_mesh, dst_placements),
meta.name,
)
validate_transfer_plan(
self._metas,
self._dst_mesh,
[destination.placements for destination in self._parameter_destinations],
)
# Update requests carry names only, so cache their plan indices. Track
# whether any fallback entry requires the `load_weights` lifecycle.
self._index = {meta.name: i for i, meta in enumerate(self._metas)}
@@ -294,38 +440,6 @@ class M2NWeightTransferEngine(
not destination.direct for destination in self._parameter_destinations
)
self.model_update_group = worker_init_process_group(
NCCLWeightTransferInitInfo(
master_address=init_info.master_address,
master_port=init_info.master_port,
nccl_unique_id_b64=init_info.nccl_unique_id_b64,
rank_offset=init_info.rank_offset,
world_size=init_info.world_size,
),
parallel_config,
)
# Destinations are per-parameter and depend on the inference model, so
# the trainer cannot derive them. The first worker publishes the plan
# and every other rank checks its own against it — a silent
# disagreement between workers would mean mismatched collectives.
mine = [destination.placements for destination in self._parameter_destinations]
agreed = publish_destination_placements(
self.model_update_group, init_info.rank_offset, mine, len(self._metas)
)
if agreed != mine:
mismatched = next(
meta.name for meta, a, b in zip(self._metas, mine, agreed) if a != b
)
raise RuntimeError(
"inference workers resolved different nccl_m2n destinations "
f"(first mismatch: '{mismatched}'); all workers must run the "
"same model and parallel config"
)
self._handle = self._m2n.Handle.create(
self._m2n.Config(max_cta=init_info.max_cta)
)
def start_weight_update(self) -> None:
"""Set up layerwise reloading, but only if some parameter needs it.
@@ -335,7 +449,9 @@ class M2NWeightTransferEngine(
"""
if not self._uses_load_weights:
return
from vllm.model_executor.model_loader.reload import initialize_layerwise_reload
from vllm.model_executor.model_loader.reload import (
initialize_layerwise_reload,
)
initialize_layerwise_reload(self.model)
@@ -354,6 +470,10 @@ class M2NWeightTransferEngine(
"nccl_m2n weight transfer not initialized. "
"Call init_transfer_engine() first."
)
if not isinstance(update_info.names, list) or any(
not isinstance(name, str) for name in update_info.names
):
raise TypeError("nccl_m2n update names must be a list of strings")
requested = []
for name in update_info.names:
@@ -369,7 +489,6 @@ class M2NWeightTransferEngine(
disable_mtp_completeness_check,
)
comm = comm_ptr(self.model_update_group)
stream = torch.cuda.current_stream()
# Reshard lazily so one `load_weights` invocation sees the complete
@@ -385,7 +504,13 @@ class M2NWeightTransferEngine(
meta.shape, dtype=meta.dtype, device=self.device
)
self._reshard(comm, stream, meta, destination.placements, buffer)
self._reshard(
self.model_update_group,
stream,
meta,
destination.placements,
buffer,
)
if destination.tensor is None:
# `load_weights` reads on the host stream, so the transfer
@@ -410,7 +535,7 @@ class M2NWeightTransferEngine(
def _reshard(
self,
comm: int,
comm: Any,
stream: torch.cuda.Stream,
src_meta: M2NParamMeta,
dst_placements: Placements | None,
@@ -421,11 +546,10 @@ class M2NWeightTransferEngine(
`dst_placements` describes the buffer's logical placement over the
worker mesh; `dst_buffer` is this rank's physical storage.
"""
assert self._src_mesh is not None
assert self._dst_mesh is not None
m2n = self._m2n
src_mesh, resolved_src_placements = resolve_layout(
self._src_mesh, src_meta.placements
src_meta.source_layout.mesh, src_meta.source_layout.placements
)
dst_mesh, resolved_dst_placements = resolve_layout(
self._dst_mesh, dst_placements
@@ -444,12 +568,32 @@ class M2NWeightTransferEngine(
def shutdown(self) -> None:
"""Finish pending GPU work and release M2N communication state."""
if self._handle is not None:
# M2N does not synchronize caller streams on finalize.
torch.accelerator.synchronize()
self._handle.destroy()
self._handle = None
if self.model_update_group is not None:
self.model_update_group.destroy()
self.model_update_group = None
handle, self._handle = self._handle, None
group, self.model_update_group = self.model_update_group, None
self._m2n = None
self._metas = []
self._dst_mesh = None
self._parameter_destinations = []
self._index = {}
self._uses_load_weights = False
first_error: BaseException | None = None
def attempt(action: Any, message: str) -> None:
nonlocal first_error
try:
action()
except BaseException as exc:
if first_error is None:
first_error = exc
logger.exception(message)
if handle is not None or group is not None:
# M2N does not synchronize caller streams on finalize.
attempt(torch.accelerator.synchronize, "failed to synchronize nccl_m2n")
if handle is not None:
attempt(handle.destroy, "failed to destroy nccl_m2n handle")
if group is not None:
attempt(group.abort, "failed to abort nccl_m2n communicator")
if first_error is not None:
raise first_error
+88 -36
View File
@@ -5,21 +5,22 @@
m2n plans a transfer from both sides' layouts, so the trainer has to say how it
holds each parameter — which the base `WeightSource` / `ParamMeta` pair does not
express. This module supplies the m2n flavor of both, and the DTensor-backed
implementation that covers the common trainer.
A source declares its mesh once (`mesh()`) and a placement per parameter, since
one topology describes every tensor on the side.
implementation that covers the common trainer. Each parameter owns its source
mesh and placements because dense and expert weights can use different
factorizations over the same trainer ranks.
"""
from collections.abc import Iterator
from collections.abc import Callable, Iterator, Sequence
from dataclasses import dataclass
from typing import Any
import torch
from vllm.distributed.weight_transfer.base import ParamMeta, WeightSource
from vllm.distributed.weight_transfer.base import WeightSource
from vllm.distributed.weight_transfer.m2n_common import (
REPLICATE,
REPLICATED,
M2NLayout,
M2NMesh,
M2NParamMeta,
Placements,
@@ -27,6 +28,8 @@ from vllm.distributed.weight_transfer.m2n_common import (
__all__ = [
"M2NWeightSource",
"M2NManifestEntry",
"ManifestM2NWeightSource",
"DTensorModuleSource",
"mesh_from_tensor",
"placements_from_tensor",
@@ -40,17 +43,76 @@ class M2NWeightSource(WeightSource):
materialized full tensor: gathering is exactly the cost m2n removes.
"""
def mesh(self) -> M2NMesh:
"""The trainer's rank topology, shared by every parameter."""
def metadata(self) -> list[M2NParamMeta]: # type: ignore[override]
"""Name, dtype, full shape, and source layout for each parameter."""
raise NotImplementedError
def metadata(self) -> list[ParamMeta]:
"""Name, dtype, full shape, and trainer placement for each parameter."""
def __iter__(self) -> Iterator[tuple[str, Any]]: # type: ignore[override]
"""Yield `(name, local buffer)` pairs in metadata order.
Providers must enqueue materialization on the caller's current CUDA
stream, or return a buffer whose writes are already visible to it.
Buffers may be torch tensors, CUDA Array Interface objects, or raw
device pointers accepted by nccl4py. Providers own correct
communicator-device placement. Raw-pointer providers additionally own
allocation lifetime, dtype, and shape correctness.
"""
raise NotImplementedError
def __iter__(self) -> Iterator[tuple[str, torch.Tensor]]:
"""Yield `(name, local shard)` pairs in the same order as `metadata()`."""
raise NotImplementedError
@dataclass(frozen=True)
class M2NManifestEntry:
"""One immutable source descriptor with a lazy rank-local tensor view."""
key: str
dtype: torch.dtype
global_shape: tuple[int, ...]
source_layout: M2NLayout
local_tensor: Callable[[], Any]
def __post_init__(self) -> None:
object.__setattr__(self, "global_shape", tuple(self.global_shape))
if not self.key:
raise ValueError("M2N manifest key must not be empty")
if not isinstance(self.dtype, torch.dtype):
raise TypeError(
f"manifest entry '{self.key}' has invalid dtype {self.dtype!r}"
)
if not isinstance(self.source_layout, M2NLayout):
raise TypeError(
f"manifest entry '{self.key}' source_layout must be an M2NLayout"
)
if not callable(self.local_tensor):
raise TypeError(
f"manifest entry '{self.key}' local_tensor must be callable"
)
@property
def metadata(self) -> M2NParamMeta:
"""Return the serializable portion consumed during initialization."""
return M2NParamMeta(
self.key,
self.dtype,
self.global_shape,
self.source_layout,
)
class ManifestM2NWeightSource(M2NWeightSource):
"""An ordered M2N source backed by lazy, rank-local manifest entries."""
def __init__(self, entries: Sequence[M2NManifestEntry]) -> None:
self._entries = tuple(entries)
self._metadata = tuple(entry.metadata for entry in self._entries)
def metadata(self) -> list[M2NParamMeta]: # type: ignore[override]
"""Return a fresh list while preserving the immutable manifest order."""
return list(self._metadata)
def __iter__(self) -> Iterator[tuple[str, Any]]: # type: ignore[override]
"""Materialize one rank-local buffer at a time."""
for entry in self._entries:
yield entry.key, entry.local_tensor()
def _placement_code(placement: Any) -> int:
@@ -69,7 +131,8 @@ def _placement_code(placement: Any) -> int:
def mesh_from_tensor(tensor: torch.Tensor, num_trainer_ranks: int) -> M2NMesh:
"""The mesh a parameter lives on, as an `M2NMesh`.
A plain tensor is identical on every trainer rank, so it spans all of them.
Tensors without DeviceMesh metadata are treated as replicated. Implicitly
sharded tensors require a custom source with an explicit layout.
"""
device_mesh = getattr(tensor, "device_mesh", None)
if device_mesh is None:
@@ -122,29 +185,18 @@ class DTensorModuleSource(M2NWeightSource):
self._module = module
self._num_trainer_ranks = num_trainer_ranks
def mesh(self) -> M2NMesh:
"""The mesh every sharded parameter agrees on.
Replicated parameters span the trainer without a device mesh of their
own, so they never decide this; a model whose *sharded* parameters
disagree cannot be described by one side mesh and is rejected.
"""
meshes = {
mesh_from_tensor(p, self._num_trainer_ranks)
for _, p in self._module.named_parameters()
if placements_from_tensor(p) is not REPLICATED
}
if len(meshes) > 1:
raise ValueError(
"nccl_m2n needs one mesh per side, but this module's sharded "
f"parameters span several: {sorted(m.dims for m in meshes)}"
)
return meshes.pop() if meshes else M2NMesh((self._num_trainer_ranks, 1), 0)
def metadata(self) -> list[ParamMeta]:
"""Read global shape/dtype and placements without gathering shards."""
def metadata(self) -> list[M2NParamMeta]: # type: ignore[override]
"""Read each global shape and source layout without gathering shards."""
return [
M2NParamMeta(name, p.dtype, tuple(p.shape), placements_from_tensor(p))
M2NParamMeta(
name,
p.dtype,
tuple(p.shape),
M2NLayout(
mesh_from_tensor(p, self._num_trainer_ranks),
placements_from_tensor(p),
),
)
for name, p in self._module.named_parameters()
]
+416 -203
View File
@@ -13,6 +13,8 @@ engine -- where the `TrainerWeightTransferEngine` contract puts it -- rather tha
in every caller.
"""
import socket
from collections.abc import Mapping
from concurrent.futures import Future, ThreadPoolExecutor
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, ClassVar, Self, cast
@@ -26,39 +28,43 @@ from vllm.distributed.weight_transfer.base import (
WeightSource,
)
from vllm.distributed.weight_transfer.m2n_common import (
M2N_WIRE_SCHEMA_VERSION,
M2NMesh,
M2NParamMeta,
M2NWireParam,
Placements,
check_complete_plan_agreement,
check_data_plane_agreement,
check_runtime_ready_agreement,
check_source_plan_agreement,
check_transferable,
comm_ptr,
create_m2n_communicator,
expected_local_shape,
import_m2n,
prepare_m2n_local_runtime,
publish_destination_placements,
resolve_layout,
source_plan_digest,
to_mesh,
to_placements,
validate_layout,
validate_local_buffer,
validate_transfer_plan,
)
from vllm.distributed.weight_transfer.m2n_source import M2NWeightSource
from vllm.distributed.weight_transfer.nccl_common import (
decode_nccl_unique_id,
trainer_init,
uid_init_process_group,
stateless_init_metadata_group,
)
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.distributed.utils import StatelessProcessGroup
logger = init_logger(__name__)
__all__ = ["M2NTrainerInitInfo", "M2NTrainerWeightTransferEngine"]
def _dtype_name(dtype: torch.dtype) -> str:
"""Short name for a dtype, e.g. `bfloat16`, for the worker-side init dict."""
return str(dtype).split(".")[-1]
@dataclass(kw_only=True)
class M2NTrainerInitInfo(TrainerInitInfo):
"""Trainer-side init info for nccl_m2n.
@@ -70,8 +76,8 @@ class M2NTrainerInitInfo(TrainerInitInfo):
backend: ClassVar[str] = "nccl_m2n"
master_address: str | None = None
master_port: int | None = None
master_address: str
master_port: int
world_size: int
"""Trainer ranks + all inference workers."""
num_trainer_ranks: int = 1
@@ -79,20 +85,34 @@ class M2NTrainerInitInfo(TrainerInitInfo):
"""How the inference ranks are laid out: axis 0 replicates and axis 1
shards. The trainer declares it so both sides describe the destination
identically; defaults to a flat `(num_workers, 1)`, which is all a
replicated destination needs."""
max_cta: int | None = None
replicated destination needs.
Destination meshes remain common to every parameter; per-parameter
destination meshes are intentionally future work.
"""
nccl_unique_id_b64: str | None = field(default=None, repr=False)
"""Optional UID for the NCCL data plane; TCP remains the metadata plane."""
max_cta: int | None = None
listen_socket: socket.socket | None = field(default=None, repr=False, compare=False)
"""Optional rank-zero TCPStore reservation; never sent to workers."""
@property
def destination_mesh_dims(self) -> tuple[int, int]:
"""`dst_mesh_dims`, or a flat mesh over every inference worker."""
if self.dst_mesh_dims is not None:
return self.dst_mesh_dims
return cast(tuple[int, int], tuple(self.dst_mesh_dims))
return (self.world_size - self.num_trainer_ranks, 1)
def __post_init__(self) -> None:
"""Reject rank counts or destination meshes that cannot form a group."""
_ = self.nccl_unique_id_bytes
if not isinstance(self.master_address, str) or not self.master_address:
raise ValueError("`master_address` must name the metadata rendezvous")
if (
not isinstance(self.master_port, int)
or isinstance(self.master_port, bool)
or not 0 < self.master_port < 65536
):
raise ValueError("`master_port` must be a valid metadata port")
if not 0 < self.num_trainer_ranks < self.world_size:
raise ValueError(
f"`num_trainer_ranks` ({self.num_trainer_ranks}) must leave at "
@@ -101,7 +121,11 @@ class M2NTrainerInitInfo(TrainerInitInfo):
num_workers = self.world_size - self.num_trainer_ranks
if self.dst_mesh_dims is not None:
dims = tuple(self.dst_mesh_dims)
if len(dims) != 2 or dims[0] * dims[1] != num_workers:
try:
mesh = M2NMesh(cast(tuple[int, int], dims), self.num_trainer_ranks)
except (TypeError, ValueError) as exc:
raise ValueError(f"invalid `dst_mesh_dims` {dims}: {exc}") from exc
if mesh.size != num_workers:
raise ValueError(
f"`dst_mesh_dims` {dims} must be two dims covering the "
f"{num_workers} inference workers"
@@ -111,14 +135,23 @@ class M2NTrainerInitInfo(TrainerInitInfo):
f"trainer `rank` ({self.rank}) must be below "
f"`num_trainer_ranks` ({self.num_trainer_ranks})"
)
if self.listen_socket is not None:
if self.rank != 0:
raise ValueError("only trainer rank zero may reserve the TCPStore")
if not isinstance(self.listen_socket, socket.socket):
raise TypeError("listen_socket must be a socket.socket")
if self.listen_socket.fileno() < 0:
raise ValueError("listen_socket is already closed")
@property
def nccl_unique_id_bytes(self) -> bytes | None:
if self.nccl_unique_id_b64 is None:
return None
return decode_nccl_unique_id(
master_address=self.master_address,
master_port=self.master_port,
master_address=None,
master_port=None,
nccl_unique_id_b64=self.nccl_unique_id_b64,
ctx="M2NTrainerInitInfo",
ctx="M2NTrainerInitInfo data plane",
)
@@ -136,22 +169,20 @@ class M2NTrainerWeightTransferEngine(TrainerWeightTransferEngine[M2NTrainerInitI
*,
client: VLLMWeightSyncClient,
source: M2NWeightSource,
is_controller: bool,
num_trainer_ranks: int,
is_sender: bool,
) -> None:
"""Hold the client and source; `trainer_init` fills in the group."""
super().__init__(client=client, source=source, is_sender=is_controller)
self.source: M2NWeightSource = source
self.is_controller = is_controller
self.num_trainer_ranks = num_trainer_ranks
self.group: PyNcclCommunicator | None = None
super().__init__(client=client, source=source, is_sender=is_sender)
self.group: Any = None
self._m2n: Any = None
self._handle: Any = None
self._metas: list[M2NParamMeta] = []
self._src_mesh: M2NMesh | None = None
self._wire_params: list[M2NWireParam] = []
self._source_digest = ""
self._dst_mesh: M2NMesh | None = None
self._dst_placements: list[Placements | None] = []
self._executor: ThreadPoolExecutor | None = None
self._metadata_group: StatelessProcessGroup | None = None
self._num_trainer_ranks = 0
@classmethod
def trainer_init(
@@ -168,204 +199,351 @@ class M2NTrainerWeightTransferEngine(TrainerWeightTransferEngine[M2NTrainerInitI
build local state and join the communicator. Every rank participates
in every reshard and sends its local shard.
The ordering here is not arbitrary -- see the comments inline. In
short: validate before anything blocks; start the init RPC without
waiting (workers join the communicator inside that RPC, so it cannot
return until this side has joined too); then join the NCCL
communicator; then create the M2N handle; and only then wait for the RPC
to finish.
Build a local validation envelope, start the worker RPC without
waiting, join the metadata rendezvous, and require unanimous source,
data-plane, and runtime readiness before constructing NCCL.
"""
# The base WeightSource carries name/dtype/shape but not layout, which
# is not enough to plan a reshard. Fail here rather than at the first
# missing attribute deep inside the send loop.
if not isinstance(source, M2NWeightSource):
raise TypeError(
"nccl_m2n needs per-parameter layouts, so its source must be an "
f"M2NWeightSource (e.g. DTensorModuleSource); got "
f"{type(source).__name__}"
)
# Trainer-side weight transfer engine for this rank. `__init__` only
# holds the client and source; the communicator, meshes, and M2N handle
# are filled in below.
engine = cls(
client=client,
source=source,
is_controller=init_info.rank == 0,
num_trainer_ranks=init_info.num_trainer_ranks,
source=cast(M2NWeightSource, source),
is_sender=init_info.is_sender,
)
engine._m2n = import_m2n()
engine._src_mesh = source.mesh()
if engine._src_mesh.size != init_info.num_trainer_ranks:
raise ValueError(
f"source mesh covers {engine._src_mesh.size} ranks, but there "
f"are {init_info.num_trainer_ranks} trainer ranks"
)
# Validate the whole plan before any rendezvous. A layout m2n cannot
# express is a mismatched collective at send time, which hangs rather
# than raises -- so every precondition is checked while an exception can
# still propagate to the caller.
engine._metas = cast(list[M2NParamMeta], source.metadata())
for meta in engine._metas:
check_transferable(meta.name, meta.dtype, meta.shape)
mesh, placements = resolve_layout(engine._src_mesh, meta.placements)
validate_layout(mesh, placements, meta.shape, "source")
rendezvous: Future | None = None
if engine.is_controller:
engine._executor = ThreadPoolExecutor(
max_workers=1, thread_name_prefix="m2n-weight-sync"
)
# Only trainer rank 0 starts the control-plane RPC. Workers join
# the communicator inside it, so submit it first, join below, and
# wait for it at the end.
rendezvous = engine._executor.submit(
client.init_weight_transfer_engine,
engine._worker_init_info(init_info),
)
# Every trainer rank joins -- reshard is collective over the whole
# communicator. Trainer ranks take
# [0, num_trainer_ranks) and the workers follow, which is the
# contiguous-interval layout m2n requires of both meshes.
unique_id_bytes = init_info.nccl_unique_id_bytes
if unique_id_bytes is not None:
engine.group = uid_init_process_group(
unique_id_bytes,
rank=init_info.rank,
world_size=init_info.world_size,
device=torch.accelerator.current_device_index(),
)
else:
assert init_info.master_address is not None
assert init_info.master_port is not None
engine.group = trainer_init(
{
"master_address": init_info.master_address,
"master_port": init_info.master_port,
"world_size": init_info.world_size,
},
rank=init_info.rank,
)
# The trainer declares the inference topology; the worker uses exactly
# this, so neither side has to infer the other's factorization.
engine._dst_mesh = M2NMesh(
init_info.destination_mesh_dims, init_info.num_trainer_ranks
)
engine._dst_placements = engine._receive_destination_plan(
init_info.num_trainer_ranks
)
logger.info(
"nccl_m2n trainer ready: %d parameters, trainer ranks [0, %d), "
"inference ranks [%d, %d)",
len(engine._metas),
init_info.num_trainer_ranks,
init_info.num_trainer_ranks,
init_info.world_size,
)
# Created after the group so a failed rendezvous does not leave a live
# handle behind; shutdown() releases it once caller streams are synced.
engine._handle = engine._m2n.Handle.create(
engine._m2n.Config(max_cta=init_info.max_cta)
)
# Now safe to collect: both sides have joined, so the RPC has returned
# or raised. Doing this last means a worker-side init failure surfaces
# here, as an exception from trainer_init, rather than at the first send.
if rendezvous is not None:
rendezvous.result()
engine._initialize(init_info, source)
return engine
def _receive_destination_plan(
self, first_worker_rank: int
) -> list[Placements | None]:
"""Receive and validate the inference-side per-parameter placements."""
if self.group is None or self._dst_mesh is None:
raise RuntimeError(
"destination plan requires the communicator and destination mesh"
def _initialize(
self,
init_info: M2NTrainerInitInfo,
source: WeightSource | None,
) -> None:
"""Run source validation, metadata agreement, and NCCL rendezvous."""
self._num_trainer_ranks = init_info.num_trainer_ranks
local_source_error: str | None = None
try:
if not isinstance(source, M2NWeightSource):
raise TypeError(
"nccl_m2n needs per-parameter layouts, so its source must "
f"be an M2NWeightSource; got {type(source).__name__}"
)
self._m2n = import_m2n()
self._prepare_source_plan(source, init_info.num_trainer_ranks)
except Exception as exc:
local_source_error = f"{type(exc).__name__}: {exc}"
self._metas = []
self._wire_params = []
self._source_digest = source_plan_digest([])
executor: ThreadPoolExecutor | None = None
init_future: Future[None] | None = None
if self.is_sender:
executor = ThreadPoolExecutor(
max_workers=1, thread_name_prefix="m2n-weight-sync"
)
init_future = executor.submit(
self.client.init_weight_transfer_engine,
self._worker_init_info(init_info),
)
handle = None
group = None
try:
reservation = init_info.listen_socket
try:
metadata_group = stateless_init_metadata_group(
init_info.master_address,
init_info.master_port,
init_info.rank,
init_info.world_size,
listen_socket=reservation,
)
finally:
init_info.listen_socket = None
if reservation is not None:
reservation.close()
plan_digest = check_source_plan_agreement(
metadata_group,
self._wire_params,
expected_digest=self._source_digest,
local_error=local_source_error,
)
# Retain the common destination mesh and the merged NCCL publication
# of per-parameter placements.
self._dst_mesh = M2NMesh(
init_info.destination_mesh_dims, init_info.num_trainer_ranks
)
unique_id_bytes: bytes | None = None
data_plane_error: str | None = None
try:
unique_id_bytes = init_info.nccl_unique_id_bytes
except Exception as exc:
data_plane_error = f"{type(exc).__name__}: {exc}"
check_data_plane_agreement(
metadata_group,
unique_id_bytes,
data_plane_error,
)
runtime_error: str | None = None
nccl = None
device = None
try:
nccl, device, handle = prepare_m2n_local_runtime(
self._m2n, init_info.max_cta
)
except Exception as exc:
runtime_error = f"{type(exc).__name__}: {exc}"
check_runtime_ready_agreement(metadata_group, runtime_error)
assert nccl is not None and device is not None and handle is not None
group = create_m2n_communicator(nccl, metadata_group, unique_id_bytes)
destination_placements = self._receive_destination_plan(
group,
init_info.num_trainer_ranks,
torch.device("cuda", device),
)
complete_plan_error: str | None = None
try:
assert self._dst_mesh is not None
validate_transfer_plan(
self._metas, self._dst_mesh, destination_placements
)
except Exception as exc:
complete_plan_error = f"{type(exc).__name__}: {exc}"
assert self._dst_mesh is not None
check_complete_plan_agreement(
metadata_group,
plan_digest,
self._dst_mesh,
destination_placements,
complete_plan_error,
)
if init_future is not None:
init_future.result()
logger.info(
"nccl_m2n trainer ready: %d parameters, trainer ranks [0, %d), "
"inference ranks [%d, %d)",
len(self._metas),
init_info.num_trainer_ranks,
init_info.num_trainer_ranks,
init_info.world_size,
)
logger.info("nccl_m2n source plan digest: %s", plan_digest)
except BaseException:
if group is not None:
try:
# Transport cleanup only; reload validity belongs to the
# model-update lifecycle, not this backend.
group.abort()
except BaseException:
logger.exception(
"failed to abort partially initialized nccl_m2n communicator"
)
if handle is not None:
try:
handle.destroy()
except BaseException:
logger.exception(
"failed to destroy partially initialized nccl_m2n handle"
)
raise
finally:
if executor is not None:
executor.shutdown(wait=False, cancel_futures=True)
self._metadata_group = metadata_group
self._handle = handle
self.group = group
self._dst_placements = destination_placements
def _receive_destination_plan(
self, group: Any, first_worker_rank: int, device: torch.device
) -> list[Placements | None]:
"""Receive the inference-side per-parameter placements."""
# The first inference worker publishes one placement per parameter.
# Every trainer rank receives the complete plan in this single,
# initialization-time broadcast.
placements = publish_destination_placements(
self.group, first_worker_rank, None, len(self._metas)
group,
device,
torch.cuda.current_stream(),
first_worker_rank,
None,
len(self._metas),
)
# The send loop matches placements to metadata by position, so reject
# an incomplete or oversized plan before entering any M2N collective.
if len(placements) != len(self._metas):
raise RuntimeError(
f"inference side planned {len(placements)} parameters, but "
f"this trainer announced {len(self._metas)}"
)
# Check every worker-selected destination against the corresponding
# global parameter shape before the first transfer.
for meta, placement in zip(self._metas, placements):
mesh, resolved = resolve_layout(self._dst_mesh, placement)
validate_layout(mesh, resolved, meta.shape, "destination")
return placements
def _worker_init_info(self, init_info: M2NTrainerInitInfo) -> dict[str, Any]:
"""Handshake payload: rendezvous, both meshes, and the transfer plan."""
assert self._src_mesh is not None
payload: dict[str, Any] = {
"schema_version": M2N_WIRE_SCHEMA_VERSION,
"master_address": init_info.master_address,
"master_port": init_info.master_port,
"nccl_unique_id_b64": init_info.nccl_unique_id_b64,
"rank_offset": init_info.num_trainer_ranks,
"world_size": init_info.world_size,
"src_mesh_dims": list(self._src_mesh.dims),
"dst_mesh_dims": list(init_info.destination_mesh_dims),
"names": [m.name for m in self._metas],
"dtype_names": [_dtype_name(m.dtype) for m in self._metas],
"shapes": [list(m.shape) for m in self._metas],
"src_placements": [
None if m.placements is None else list(m.placements)
for m in self._metas
],
"source_digest": self._source_digest,
"params": [param.to_dict() for param in self._wire_params],
"max_cta": init_info.max_cta,
}
return {key: value for key, value in payload.items() if value is not None}
def _prepare_source_plan(
self, source: M2NWeightSource, num_trainer_ranks: int
) -> None:
"""Build and validate this rank's immutable source plan."""
self._metas = source.metadata()
names: set[str] = set()
for meta in self._metas:
if not isinstance(meta, M2NParamMeta):
raise TypeError(
"nccl_m2n source metadata must contain M2NParamMeta "
f"entries, got {type(meta).__name__}"
)
if meta.name in names:
raise ValueError(
"nccl_m2n source metadata contains duplicate parameter "
f"'{meta.name}'"
)
names.add(meta.name)
check_transferable(meta.name, meta.dtype, meta.shape)
layout = meta.source_layout
if layout.mesh.start_rank != 0:
raise ValueError(
f"parameter '{meta.name}' source mesh must start at rank 0, "
f"got {layout.mesh.start_rank}"
)
if layout.mesh.size != num_trainer_ranks:
raise ValueError(
f"parameter '{meta.name}' source mesh covers "
f"{layout.mesh.size} ranks, but there are "
f"{num_trainer_ranks} trainer ranks"
)
expected_local_shape(meta)
self._wire_params = [
M2NWireParam(
name=meta.name,
dtype_name=str(meta.dtype).split(".")[-1],
shape=meta.shape,
src_mesh_dims=meta.source_layout.mesh.dims,
src_placements=meta.source_layout.placements,
)
for meta in self._metas
]
self._source_digest = source_plan_digest(self._wire_params)
def send_weights(self) -> None:
"""Drive one update round: start, reshard concurrently, then finish."""
if self._handle is None or self.group is None:
raise RuntimeError("nccl_m2n trainer engine is not initialized")
update: Future | None = None
if self.is_controller:
self.client.start_weight_update()
# The workers reshard from inside `update_weights`, concurrently
# with the sends below.
assert self._executor is not None
update = self._executor.submit(
self.client.update_weights,
{"names": [m.name for m in self._metas]},
)
if not self.is_sender:
try:
self._send()
except BaseException:
try:
self._receive_round_outcome()
except BaseException:
logger.exception(
"failed to consume nccl_m2n round outcome after local failure"
)
raise
self._receive_round_outcome()
return
try:
self._send()
finally:
if update is not None:
self.client.start_weight_update()
executor = ThreadPoolExecutor(
max_workers=1, thread_name_prefix="m2n-weight-sync"
)
try:
update = executor.submit(
self.client.update_weights,
{"names": [m.name for m in self._metas]},
)
# Surface an immediate worker-side rejection before trainers
# enter the collective sequence.
if update.done():
update.result()
self._send()
update.result()
if self.is_controller:
finally:
executor.shutdown(wait=False, cancel_futures=True)
self.client.finish_weight_update()
except BaseException as exc:
try:
self._publish_round_outcome(exc)
except BaseException:
logger.exception("failed to publish nccl_m2n round failure")
raise
self._publish_round_outcome(None)
def _publish_round_outcome(self, failure: BaseException | None) -> None:
"""Tell non-sender trainer ranks whether worker finalization passed."""
if self._num_trainer_ranks <= 1:
return
if self._metadata_group is None:
raise RuntimeError("nccl_m2n metadata group is unavailable")
envelope = {
"ok": failure is None,
"error": (
None if failure is None else f"{type(failure).__name__}: {failure}"
),
}
for rank in range(1, self._num_trainer_ranks):
self._metadata_group.send_obj(envelope, rank)
# Keep rank zero's TCPStore alive until every trainer consumed its
# outcome. This also keeps the per-rank message counters aligned.
expected_ack = {"phase": "round_outcome_consumed"}
for rank in range(1, self._num_trainer_ranks):
ack = self._metadata_group.recv_obj(rank)
if ack != expected_ack:
raise RuntimeError(
f"trainer rank {rank} sent an invalid round-outcome acknowledgement"
)
def _receive_round_outcome(self) -> None:
"""Wait until trainer rank zero observed every worker's finish RPC."""
if self._metadata_group is None:
raise RuntimeError("nccl_m2n metadata group is unavailable")
outcome = self._metadata_group.recv_obj(0)
self._metadata_group.send_obj({"phase": "round_outcome_consumed"}, 0)
if not isinstance(outcome, Mapping) or set(outcome) != {"ok", "error"}:
raise RuntimeError("trainer rank zero sent an invalid round outcome")
ok = outcome["ok"]
error = outcome["error"]
if not isinstance(ok, bool) or ok != (error is None):
raise RuntimeError("trainer rank zero sent an invalid round outcome")
if not ok:
if not isinstance(error, str) or not error:
raise RuntimeError("trainer rank zero sent an invalid round error")
raise RuntimeError(
"nccl_m2n worker update failed on trainer rank zero: " + error
)
def _send(self) -> None:
"""Reshard each local shard into its worker-planned destination."""
assert self.group is not None
assert self._src_mesh is not None
assert self._dst_mesh is not None
assert self.source is not None
m2n = self._m2n
comm = comm_ptr(self.group)
stream = torch.cuda.current_stream()
for index, (name, tensor) in enumerate(self.source):
device_index = torch.accelerator.current_device_index()
# Optimizer work may have used another CUDA stream. Make prior writes
# visible before a provider materializes current-stream local shards.
torch.accelerator.synchronize(device_index)
inflight_buffers: list[Any] = []
sent = 0
for index, (name, buffer) in enumerate(self.source):
if index >= len(self._metas):
raise RuntimeError(
"weight source yielded more parameters than metadata() "
f"declared (first extra parameter: '{name}')"
)
meta = self._metas[index]
if name != meta.name:
raise RuntimeError(
@@ -373,28 +551,47 @@ class M2NTrainerWeightTransferEngine(TrainerWeightTransferEngine[M2NTrainerInitI
f"its metadata declared '{meta.name}'; the source must "
"iterate in a stable order"
)
shard = tensor.detach().contiguous()
src_mesh, src_placements = resolve_layout(self._src_mesh, meta.placements)
validate_local_buffer(m2n, meta, buffer)
src_mesh, src_placements = resolve_layout(
meta.source_layout.mesh, meta.source_layout.placements
)
dst_mesh, dst_placements = resolve_layout(
self._dst_mesh, self._dst_placements[index]
)
raw_pointer_args = {}
if isinstance(buffer, int):
raw_pointer_args = {
"src_local_shape": expected_local_shape(meta),
"src_dtype": meta.dtype,
}
m2n.reshard(
shard,
buffer,
None,
comm,
self.group,
stream,
src_mesh=to_mesh(m2n, src_mesh),
src_placements=to_placements(m2n, src_placements),
dst_mesh=to_mesh(m2n, dst_mesh),
dst_placements=to_placements(m2n, dst_placements),
handle=self._handle,
**raw_pointer_args,
)
# Let allocator-aware buffers recycle storage in stream order.
# Other object owners must stay alive until the final synchronize;
# raw-pointer lifetime remains the source's responsibility.
record_stream = getattr(buffer, "record_stream", None)
if callable(record_stream):
record_stream(stream)
elif not isinstance(buffer, int):
inflight_buffers.append(buffer)
sent += 1
if sent != len(self._metas):
missing = self._metas[sent].name
raise RuntimeError(
f"weight source yielded {sent} parameters, but metadata() "
f"declared {len(self._metas)} (first missing: '{missing}')"
)
# A custom source may yield temporary storage allocated on another
# stream. M2N retains only its device pointer for asynchronous work,
# so tell the allocator not to reuse that storage until this stream
# has finished with it. For storage allocated on this stream,
# record_stream is effectively a no-op.
shard.record_stream(stream)
# As in the other weight-transfer backends, wait before returning
# rather than relying on same-stream ordering. The caller may then
@@ -402,14 +599,30 @@ class M2NTrainerWeightTransferEngine(TrainerWeightTransferEngine[M2NTrainerInitI
stream.synchronize()
def shutdown(self) -> None:
"""Destroy the m2n handle, join the helper thread, and drop the group."""
if self._handle is not None:
torch.accelerator.synchronize()
self._handle.destroy()
self._handle = None
if self._executor is not None:
self._executor.shutdown()
self._executor = None
if self.group is not None:
self.group.destroy()
self.group = None
"""Finish pending GPU work and release M2N communication state."""
handle, self._handle = self._handle, None
group, self.group = self.group, None
self._metadata_group = None
self._dst_placements = []
self._dst_mesh = None
self._m2n = None
first_error: BaseException | None = None
def attempt(action: Any, message: str) -> None:
nonlocal first_error
try:
action()
except BaseException as exc:
if first_error is None:
first_error = exc
logger.exception(message)
if handle is not None or group is not None:
attempt(torch.accelerator.synchronize, "failed to synchronize nccl_m2n")
if handle is not None:
attempt(handle.destroy, "failed to destroy nccl_m2n handle")
if group is not None:
attempt(group.abort, "failed to abort nccl_m2n communicator")
if first_error is not None:
raise first_error
@@ -8,6 +8,7 @@ The dense (`NCCLWeightTransferEngine`) and sparse
sparse engine does not have to subclass the dense one.
"""
import socket
from dataclasses import asdict, dataclass, field
from typing import TYPE_CHECKING, Protocol
@@ -17,6 +18,7 @@ import torch
if TYPE_CHECKING:
from vllm.config.parallel import ParallelConfig
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.distributed.utils import StatelessProcessGroup
from vllm.distributed.device_communicators.pynccl_wrapper import (
NCCL_UNIQUE_ID_BYTES,
@@ -136,6 +138,26 @@ class NCCLRendezvous(Protocol):
world_size: int
def stateless_init_metadata_group(
master_address: str,
master_port: int,
rank: int,
world_size: int,
*,
listen_socket: socket.socket | None = None,
) -> "StatelessProcessGroup":
"""Create only the TCPStore group used for metadata agreement."""
from vllm.distributed.utils import StatelessProcessGroup
return StatelessProcessGroup.create(
host=master_address,
port=master_port,
rank=rank,
world_size=world_size,
listen_socket=listen_socket,
)
def stateless_init_process_group(
master_address: str,
master_port: int,
@@ -181,6 +203,29 @@ def uid_init_process_group(
)
def _worker_group_rank(rank_offset: int, parallel_config: "ParallelConfig") -> int:
"""Return this worker's rank in a trainer-to-worker process group."""
dp_rank = parallel_config.data_parallel_index
world_size_per_dp = parallel_config.world_size # TP * PP
rank_within_dp = parallel_config.rank
return rank_offset + dp_rank * world_size_per_dp + rank_within_dp
def worker_init_metadata_group(
init_info: NCCLWeightTransferInitInfo,
parallel_config: "ParallelConfig",
) -> "StatelessProcessGroup":
"""Join the trainer-to-worker TCP metadata group without constructing NCCL."""
assert init_info.master_address is not None
assert init_info.master_port is not None
return stateless_init_metadata_group(
init_info.master_address,
init_info.master_port,
_worker_group_rank(init_info.rank_offset, parallel_config),
init_info.world_size,
)
def worker_init_process_group(
init_info: NCCLWeightTransferInitInfo,
parallel_config: "ParallelConfig",
@@ -190,15 +235,7 @@ def worker_init_process_group(
Computes a unique rank for this worker across all data-parallel groups and
joins the trainer via whichever rendezvous mode `init_info` carries.
"""
# Calculate the global rank in the trainer-worker process group.
# Must account for data parallel to get unique ranks across all workers.
dp_rank = parallel_config.data_parallel_index
world_size_per_dp = parallel_config.world_size # TP * PP
rank_within_dp = parallel_config.rank
# Unique rank across all DP groups
worker_rank = dp_rank * world_size_per_dp + rank_within_dp
rank = worker_rank + init_info.rank_offset
rank = _worker_group_rank(init_info.rank_offset, parallel_config)
device = torch.accelerator.current_device_index()
unique_id_bytes = init_info.nccl_unique_id_bytes
+206 -229
View File
@@ -820,7 +820,14 @@ class AnthropicServingMessages(OpenAIServingChat):
elif isinstance(generator, ChatCompletionResponse):
return self.messages_full_converter(generator)
return self.message_stream_converter(generator)
return generator
def chat_completion_stream_generator(
self, *args: Any, **kwargs: Any
) -> AsyncGenerator[str, None]:
return self.message_stream_converter(
self.chat_completion_stream_generator_raw(*args, **kwargs)
)
def messages_full_converter(
self,
@@ -881,7 +888,7 @@ class AnthropicServingMessages(OpenAIServingChat):
async def message_stream_converter(
self,
generator: AsyncGenerator[str, None],
generator: AsyncGenerator[ChatCompletionStreamResponse | ErrorResponse, None],
) -> AsyncGenerator[str, None]:
try:
@@ -991,238 +998,208 @@ class AnthropicServingMessages(OpenAIServingChat):
events.extend(stop_active_block())
return events
async for item in generator:
if item.startswith("data:"):
data_str = item[5:].strip().rstrip("\n")
if data_str == "[DONE]":
for event in stop_and_flush():
yield event
stop_message = AnthropicStreamEvent(
type="message_stop",
)
data = stop_message.model_dump_json(
exclude_unset=True, exclude_none=True
)
yield wrap_data_with_event(data, "message_stop")
else:
origin_chunk = ChatCompletionStreamResponse.model_validate_json(
data_str
)
if first_item:
chunk = AnthropicStreamEvent(
type="message_start",
message=AnthropicMessagesResponse(
id=origin_chunk.id,
# Set explicitly: this event is serialized
# with exclude_unset=True, which drops
# default-valued fields, while strict
# Anthropic SDK clients require
# message.type/role (issue #45367).
type="message",
role="assistant",
content=[],
model=origin_chunk.model,
stop_reason=None,
stop_sequence=None,
usage=_build_anthropic_usage(origin_chunk.usage),
),
)
first_item = False
data = chunk.model_dump_json(exclude_unset=True)
yield wrap_data_with_event(data, "message_start")
continue
# last chunk including usage info
if len(origin_chunk.choices) == 0:
for event in stop_and_flush():
yield event
anthropic_stop_reason, anthropic_stop_sequence = (
_to_anthropic_stop_reason(
finish_reason or "stop",
stop_sequence,
tool_used=tool_used,
tool_complete=(
tool_used and _is_json("".join(last_tool_args))
),
)
)
stop_delta = AnthropicDelta(
stop_reason=anthropic_stop_reason,
# Set explicitly so exclude_unset=True keeps it.
stop_sequence=anthropic_stop_sequence,
)
chunk = AnthropicStreamEvent(
type="message_delta",
delta=stop_delta,
usage=_build_anthropic_usage(origin_chunk.usage),
)
data = chunk.model_dump_json(exclude_unset=True)
yield wrap_data_with_event(data, "message_delta")
continue
if origin_chunk.choices[0].finish_reason is not None:
finish_reason = origin_chunk.choices[0].finish_reason
stop_sequence = origin_chunk.choices[0].stop_reason
# continue
# thinking / text content
reasoning_delta = origin_chunk.choices[0].delta.reasoning
if reasoning_delta is not None:
if reasoning_delta == "":
pass
else:
if state.block_type != "thinking":
for event in stop_and_flush():
yield event
start_event = start_block(
AnthropicContentBlock(
type="thinking", thinking=""
)
)
yield start_event
chunk = AnthropicStreamEvent(
index=(
state.block_index
if state.block_index is not None
else state.content_block_index
),
type="content_block_delta",
delta=AnthropicDelta(
type="thinking_delta",
thinking=reasoning_delta,
),
)
data = chunk.model_dump_json(exclude_unset=True)
yield wrap_data_with_event(data, "content_block_delta")
if origin_chunk.choices[0].delta.content is not None:
if origin_chunk.choices[0].delta.content == "":
pass
elif state.block_type == "tool_use":
state.pending_content.append(
origin_chunk.choices[0].delta.content
)
else:
if state.block_type != "text":
for event in stop_and_flush():
yield event
start_event = start_block(
AnthropicContentBlock(type="text", text="")
)
yield start_event
chunk = AnthropicStreamEvent(
index=(
state.block_index
if state.block_index is not None
else state.content_block_index
),
type="content_block_delta",
delta=AnthropicDelta(
type="text_delta",
text=origin_chunk.choices[0].delta.content,
),
)
data = chunk.model_dump_json(exclude_unset=True)
yield wrap_data_with_event(data, "content_block_delta")
# tool calls - process all tool calls in the delta
if len(origin_chunk.choices[0].delta.tool_calls) > 0:
for tool_call in origin_chunk.choices[0].delta.tool_calls:
if tool_call.id is not None:
# Update mapping for incremental updates
tool_index_to_id[tool_call.index] = tool_call.id
# Only create new block if different tool call
# AND has a name
tool_name = (
tool_call.function.name
if tool_call.function
else None
)
if (
state.tool_use_id != tool_call.id
and tool_name is not None
):
for event in stop_and_flush():
yield event
tool_used = True
last_tool_args = []
start_event = start_block(
AnthropicContentBlock(
type="tool_use",
id=tool_call.id,
name=tool_name,
input={},
)
)
yield start_event
# Handle initial arguments if present
if (
tool_call.function
and tool_call.function.arguments
and state.tool_use_id == tool_call.id
):
last_tool_args.append(
tool_call.function.arguments
)
chunk = AnthropicStreamEvent(
index=(
state.block_index
if state.block_index is not None
else state.content_block_index
),
type="content_block_delta",
delta=AnthropicDelta(
type="input_json_delta",
partial_json=tool_call.function.arguments,
),
)
data = chunk.model_dump_json(exclude_unset=True)
yield wrap_data_with_event(
data, "content_block_delta"
)
else:
# Incremental update - use index to find tool_use_id
tool_use_id = tool_index_to_id.get(tool_call.index)
if (
tool_use_id is not None
and tool_call.function
and tool_call.function.arguments
and state.tool_use_id == tool_use_id
):
last_tool_args.append(
tool_call.function.arguments
)
chunk = AnthropicStreamEvent(
index=(
state.block_index
if state.block_index is not None
else state.content_block_index
),
type="content_block_delta",
delta=AnthropicDelta(
type="input_json_delta",
partial_json=tool_call.function.arguments,
),
)
data = chunk.model_dump_json(exclude_unset=True)
yield wrap_data_with_event(
data, "content_block_delta"
)
continue
else:
error_response = AnthropicStreamEvent(
type="error",
error=AnthropicError(
type="internal_error",
message="Invalid data format received",
async for origin_chunk in generator:
if isinstance(origin_chunk, ErrorResponse):
raise RuntimeError(origin_chunk.error.message)
if first_item:
chunk = AnthropicStreamEvent(
type="message_start",
message=AnthropicMessagesResponse(
id=origin_chunk.id,
# Set explicitly: this event is serialized
# with exclude_unset=True, which drops
# default-valued fields, while strict
# Anthropic SDK clients require
# message.type/role (issue #45367).
type="message",
role="assistant",
content=[],
model=origin_chunk.model,
stop_reason=None,
stop_sequence=None,
usage=_build_anthropic_usage(origin_chunk.usage),
),
)
data = error_response.model_dump_json(exclude_unset=True)
yield wrap_data_with_event(data, "error")
first_item = False
data = chunk.model_dump_json(exclude_unset=True)
yield wrap_data_with_event(data, "message_start")
continue
# last chunk including usage info
if len(origin_chunk.choices) == 0:
for event in stop_and_flush():
yield event
anthropic_stop_reason, anthropic_stop_sequence = (
_to_anthropic_stop_reason(
finish_reason or "stop",
stop_sequence,
tool_used=tool_used,
tool_complete=(
tool_used and _is_json("".join(last_tool_args))
),
)
)
stop_delta = AnthropicDelta(
stop_reason=anthropic_stop_reason,
# Set explicitly so exclude_unset=True keeps it.
stop_sequence=anthropic_stop_sequence,
)
chunk = AnthropicStreamEvent(
type="message_delta",
delta=stop_delta,
usage=_build_anthropic_usage(origin_chunk.usage),
)
data = chunk.model_dump_json(exclude_unset=True)
yield wrap_data_with_event(data, "message_delta")
continue
if origin_chunk.choices[0].finish_reason is not None:
finish_reason = origin_chunk.choices[0].finish_reason
stop_sequence = origin_chunk.choices[0].stop_reason
# continue
# thinking / text content
reasoning_delta = origin_chunk.choices[0].delta.reasoning
if reasoning_delta is not None:
if reasoning_delta == "":
pass
else:
if state.block_type != "thinking":
for event in stop_and_flush():
yield event
start_event = start_block(
AnthropicContentBlock(type="thinking", thinking="")
)
yield start_event
chunk = AnthropicStreamEvent(
index=(
state.block_index
if state.block_index is not None
else state.content_block_index
),
type="content_block_delta",
delta=AnthropicDelta(
type="thinking_delta",
thinking=reasoning_delta,
),
)
data = chunk.model_dump_json(exclude_unset=True)
yield wrap_data_with_event(data, "content_block_delta")
if origin_chunk.choices[0].delta.content is not None:
if origin_chunk.choices[0].delta.content == "":
pass
elif state.block_type == "tool_use":
state.pending_content.append(
origin_chunk.choices[0].delta.content
)
else:
if state.block_type != "text":
for event in stop_and_flush():
yield event
start_event = start_block(
AnthropicContentBlock(type="text", text="")
)
yield start_event
chunk = AnthropicStreamEvent(
index=(
state.block_index
if state.block_index is not None
else state.content_block_index
),
type="content_block_delta",
delta=AnthropicDelta(
type="text_delta",
text=origin_chunk.choices[0].delta.content,
),
)
data = chunk.model_dump_json(exclude_unset=True)
yield wrap_data_with_event(data, "content_block_delta")
# tool calls - process all tool calls in the delta
if len(origin_chunk.choices[0].delta.tool_calls) > 0:
for tool_call in origin_chunk.choices[0].delta.tool_calls:
if tool_call.id is not None:
# Update mapping for incremental updates
tool_index_to_id[tool_call.index] = tool_call.id
# Only create new block if different tool call
# AND has a name
tool_name = (
tool_call.function.name if tool_call.function else None
)
if (
state.tool_use_id != tool_call.id
and tool_name is not None
):
for event in stop_and_flush():
yield event
tool_used = True
last_tool_args = []
start_event = start_block(
AnthropicContentBlock(
type="tool_use",
id=tool_call.id,
name=tool_name,
input={},
)
)
yield start_event
# Handle initial arguments if present
if (
tool_call.function
and tool_call.function.arguments
and state.tool_use_id == tool_call.id
):
last_tool_args.append(tool_call.function.arguments)
chunk = AnthropicStreamEvent(
index=(
state.block_index
if state.block_index is not None
else state.content_block_index
),
type="content_block_delta",
delta=AnthropicDelta(
type="input_json_delta",
partial_json=tool_call.function.arguments,
),
)
data = chunk.model_dump_json(exclude_unset=True)
yield wrap_data_with_event(data, "content_block_delta")
else:
# Incremental update - use index to find tool_use_id
tool_use_id = tool_index_to_id.get(tool_call.index)
if (
tool_use_id is not None
and tool_call.function
and tool_call.function.arguments
and state.tool_use_id == tool_use_id
):
last_tool_args.append(tool_call.function.arguments)
chunk = AnthropicStreamEvent(
index=(
state.block_index
if state.block_index is not None
else state.content_block_index
),
type="content_block_delta",
delta=AnthropicDelta(
type="input_json_delta",
partial_json=tool_call.function.arguments,
),
)
data = chunk.model_dump_json(exclude_unset=True)
yield wrap_data_with_event(data, "content_block_delta")
continue
for event in stop_and_flush():
yield event
stop_message = AnthropicStreamEvent(type="message_stop")
data = stop_message.model_dump_json(exclude_unset=True, exclude_none=True)
yield wrap_data_with_event(data, "message_stop")
except Exception as e:
logger.exception("Error in message stream converter.")
for event in stop_active_block():
yield event
error_response = AnthropicStreamEvent(
type="error",
error=AnthropicError(
+17 -23
View File
@@ -321,7 +321,14 @@ class CohereServingChatV2(OpenAIServingChat):
case ChatCompletionResponse():
return self._chat_completion_to_v2(generator, request)
case _:
return self._chat_completion_stream_to_v2(generator, request)
return generator
def chat_completion_stream_generator(
self, *args: Any, **kwargs: Any
) -> AsyncGenerator[str, None]:
return self._chat_completion_stream_to_v2(
self.chat_completion_stream_generator_raw(*args, **kwargs)
)
def to_chat_completion_request(
self, request: CohereChatV2Request
@@ -1428,10 +1435,9 @@ class CohereServingChatV2(OpenAIServingChat):
async def _chat_completion_stream_to_v2(
self,
generator: AsyncGenerator[str, None],
request: CohereChatV2Request,
generator: AsyncGenerator[ChatCompletionStreamResponse | ErrorResponse, None],
) -> AsyncGenerator[str, None]:
"""Translate an OpenAI-style chat completion SSE stream into Cohere's
"""Translate raw OpenAI chat completion chunks into Cohere's
v2 stream-event format.
Cohere's v2 stream lifecycle is:
@@ -1445,19 +1451,9 @@ class CohereServingChatV2(OpenAIServingChat):
state = _StreamState()
try:
async for item in generator:
if not item.startswith("data:"):
continue
data_str = item[len("data:") :].strip().rstrip("\n")
if not data_str:
continue
if data_str == "[DONE]":
# OpenAI's stream terminator. Fall through to the
# post-loop cleanup so we always emit ``message-end``
# even if the usage-only chunk was skipped.
break
chunk = ChatCompletionStreamResponse.model_validate_json(data_str)
async for chunk in generator:
if isinstance(chunk, ErrorResponse):
raise RuntimeError(chunk.error.message)
state.last_chunk_id = chunk.id
if not state.started:
@@ -1514,7 +1510,9 @@ class CohereServingChatV2(OpenAIServingChat):
except Exception as exc:
logger.exception("Error converting chat completion stream to v2")
if state.started and not state.ended:
if not state.ended:
for ev in self._close_open_blocks(state):
yield ev
yield _sse(
json.dumps(
{
@@ -1530,11 +1528,7 @@ class CohereServingChatV2(OpenAIServingChat):
yield _DONE_FRAME
return
# Normal completion or ``[DONE]``: ensure ``message-end`` is always
# emitted. Upstream may close the stream without sending the final
# usage-only chunk (e.g. on shutdown, or when ``[DONE]`` is the only
# terminator); without this fallback Cohere clients would hang
# waiting for the closing event.
# Ensure message-end is emitted even without a final usage chunk.
if state.started and not state.ended:
for ev in self._close_open_blocks(state):
yield ev
@@ -40,6 +40,7 @@ from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger
from vllm.logprobs import Logprob
from vllm.outputs import WeightVersionSpan
from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
from vllm.renderers.chat_utils import (
ChatCompletionMessageParam,
@@ -129,6 +130,8 @@ class ChatCompletionResponseChoice(OpenAIBaseModel):
# ``None`` if (a) the request was aborted before any forward pass,
# or (b) ``enable_return_routed_experts`` is off server-side.
routed_experts: str | None = None
# Contiguous weight version spans over output-token indexes.
weight_versions: list[WeightVersionSpan] | None = None
class ChatCompletionResponse(OpenAIBaseModel):
@@ -168,6 +171,8 @@ class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
stop_reason: int | str | None = None
# not part of the OpenAI spec but for tracing the tokens
token_ids: list[int] | None = None
# Set on the final chunk only; covers the whole output.
weight_versions: list[WeightVersionSpan] | None = None
class ChatCompletionStreamResponse(OpenAIBaseModel):
@@ -430,6 +435,15 @@ class ChatCompletionRequest(OpenAIBaseModel):
"need to map generated text back to input tokens."
),
)
return_weight_versions: bool = Field(
default=False,
description=(
"If true, each choice includes weight_versions: the weight-version "
"label that sampled each half-open range of its output tokens. "
"When streaming, only the last chunk of each choice includes it. "
"Beam search does not return it."
),
)
routed_experts_prompt_start: int = Field(
default=0,
ge=0,
@@ -478,6 +478,44 @@ class OpenAIServingChat(GenerateBaseServing):
chat_template_kwargs: dict[str, Any] | None = None,
mm_token_counts: dict[str, int] | None = None,
) -> AsyncGenerator[str, None]:
try:
async for chunk in self.chat_completion_stream_generator_raw(
request,
result_generator,
request_id,
model_name,
conversation,
tokenizer,
request_metadata,
chat_template_kwargs=chat_template_kwargs,
mm_token_counts=mm_token_counts,
):
if isinstance(chunk, ErrorResponse):
data = chunk.model_dump_json()
else:
data = chunk.model_dump_json(
exclude_unset=True, exclude_none=not chunk.choices
)
yield f"data: {data}\n\n"
except Exception as e:
logger.exception("Error serializing chat completion stream.")
data = self.create_streaming_error_response(e)
yield f"data: {data}\n\n"
yield "data: [DONE]\n\n"
async def chat_completion_stream_generator_raw(
self,
request: ChatCompletionRequest,
result_generator: AsyncIterator[RequestOutput],
request_id: str,
model_name: str,
conversation: list[ConversationMessage],
tokenizer: TokenizerLike,
request_metadata: RequestResponseMetadata,
chat_template_kwargs: dict[str, Any] | None = None,
mm_token_counts: dict[str, int] | None = None,
) -> AsyncGenerator[ChatCompletionStreamResponse | ErrorResponse, None]:
"""Yield chat chunks or errors without serializing them into SSE frames."""
created_time = int(time.time())
chunk_object_type: Final = "chat.completion.chunk"
first_iteration = True
@@ -510,9 +548,7 @@ class OpenAIServingChat(GenerateBaseServing):
parsers = [None] * num_choices
except Exception as e:
logger.exception("Error in parser creation.")
data = self.create_streaming_error_response(e)
yield f"data: {data}\n\n"
yield "data: [DONE]\n\n"
yield self.create_error_response(e)
return
stream_options = request.stream_options
@@ -583,8 +619,7 @@ class OpenAIServingChat(GenerateBaseServing):
),
)
data = chunk.model_dump_json(exclude_unset=True)
yield f"data: {data}\n\n"
yield chunk
# Send response to echo the input portion of the
# last message
@@ -624,8 +659,7 @@ class OpenAIServingChat(GenerateBaseServing):
),
)
data = chunk.model_dump_json(exclude_unset=True)
yield f"data: {data}\n\n"
yield chunk
first_iteration = False
for output in res.outputs:
@@ -651,17 +685,21 @@ class OpenAIServingChat(GenerateBaseServing):
delta_text = output.text
if (
nothing_generated = (
not delta_text
and not output.token_ids
and not previous_num_tokens[i]
):
)
if nothing_generated and output.finish_reason is None:
# Chunked prefill case, don't return empty chunks
continue
delta_message: DeltaMessage | None
if parser is not None:
if nothing_generated:
# Finished without output: send the final chunk unparsed.
delta_message = DeltaMessage()
elif parser is not None:
delta_message = parser.parse_delta(
delta_text=delta_text,
delta_token_ids=as_list(output.token_ids),
@@ -773,6 +811,11 @@ class OpenAIServingChat(GenerateBaseServing):
token_ids=(
as_list(output.token_ids) if include_token_ids else None
),
weight_versions=(
output.weight_versions
if request.return_weight_versions
else None
),
)
finish_reason_sent[i] = True
@@ -812,8 +855,7 @@ class OpenAIServingChat(GenerateBaseServing):
),
)
data = chunk.model_dump_json(exclude_unset=True)
yield f"data: {data}\n\n"
yield chunk
if self._include_reasoning_tokens_details and not include_continuous_usage:
for i, parser in enumerate(parsers):
@@ -873,10 +915,7 @@ class OpenAIServingChat(GenerateBaseServing):
system_fingerprint=self.system_fingerprint,
metrics=stream_per_request_metrics,
)
final_usage_data = final_usage_chunk.model_dump_json(
exclude_unset=True, exclude_none=True
)
yield f"data: {final_usage_data}\n\n"
yield final_usage_chunk
# report to FastAPI middleware aggregate usage across all choices
num_completion_tokens = sum(previous_num_tokens)
@@ -910,13 +949,12 @@ class OpenAIServingChat(GenerateBaseServing):
)
except GenerationError as e:
yield f"data: {self._convert_generation_error_to_streaming_response(e)}\n\n"
yield self.create_error_response(
str(e), err_type="InternalServerError", status_code=e.status_code
)
except Exception as e:
logger.exception("Error in chat completion stream generator.")
data = self.create_streaming_error_response(e)
yield f"data: {data}\n\n"
# Send the final done message after all response.n are finished
yield "data: [DONE]\n\n"
yield self.create_error_response(e)
async def chat_completion_full_generator(
self,
@@ -1097,6 +1135,9 @@ class OpenAIServingChat(GenerateBaseServing):
else None
),
routed_experts=routed_experts_b64,
weight_versions=(
output.weight_versions if request.return_weight_versions else None
),
)
choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
@@ -26,6 +26,7 @@ from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger
from vllm.logprobs import Logprob
from vllm.outputs import WeightVersionSpan
from vllm.renderers import TokenizeParams
from vllm.sampling_params import (
BeamSearchParams,
@@ -178,6 +179,15 @@ class CompletionRequest(OpenAIBaseModel):
"need to map generated text back to input tokens."
),
)
return_weight_versions: bool = Field(
default=False,
description=(
"If true, each choice includes weight_versions: the weight-version "
"label that sampled each half-open range of its output tokens. "
"When streaming, only the last chunk of each choice includes it. "
"Beam search does not return it."
),
)
routed_experts_prompt_start: int = Field(
default=0,
ge=0,
@@ -677,6 +687,8 @@ class CompletionResponseChoice(OpenAIBaseModel):
# ``None`` if (a) the request was aborted before any forward pass,
# or (b) ``enable_return_routed_experts`` is off server-side.
routed_experts: str | None = None
# Contiguous weight version spans over output-token indexes.
weight_versions: list[WeightVersionSpan] | None = None
class CompletionResponse(OpenAIBaseModel):
@@ -716,6 +728,8 @@ class CompletionResponseStreamChoice(OpenAIBaseModel):
# prompt tokens is put into choice to align with CompletionResponseChoice
prompt_token_ids: list[int] | None = None
token_ids: list[int] | None = None
# Set on the final chunk only; covers the whole output.
weight_versions: list[WeightVersionSpan] | None = None
class CompletionStreamResponse(OpenAIBaseModel):
@@ -376,6 +376,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
not delta_text
and not delta_token_ids
and not previous_num_tokens[i]
and finish_reason is None
):
# Chunked prefill case, don't return empty chunks
continue
@@ -421,6 +422,11 @@ class OpenAIServingCompletion(GenerateBaseServing):
if request.return_token_ids
else None
),
weight_versions=(
output.weight_versions
if request.return_weight_versions
else None
),
)
],
)
@@ -594,6 +600,11 @@ class OpenAIServingCompletion(GenerateBaseServing):
as_list(output.token_ids) if request.return_token_ids else None
),
routed_experts=routed_experts_b64,
weight_versions=(
output.weight_versions
if request.return_weight_versions
else None
),
)
choices.append(choice_data)
@@ -53,6 +53,7 @@ from openai_harmony import Message as OpenAIHarmonyMessage
from pydantic import (
BeforeValidator,
Field,
PrivateAttr,
ValidationError,
field_serializer,
model_validator,
@@ -469,6 +470,14 @@ class ResponsesRequest(OpenAIBaseModel):
max_output_tokens_param="max_output_tokens",
)
_grammar_from_parser: bool = PrivateAttr(default=False)
"""CAUTION: Should only be set by the parser's adjust_request."""
_parser_structured_outputs: StructuredOutputsParams | None = PrivateAttr(
default=None
)
"""Parser-derived constraints that preserve the public request's text metadata."""
_DEFAULT_SAMPLING_PARAMS = {
"temperature": 1.0,
"top_p": 1.0,
@@ -478,6 +487,8 @@ class ResponsesRequest(OpenAIBaseModel):
def extract_structured_outputs(self) -> StructuredOutputsParams | None:
"""Normalize request constraints into ``StructuredOutputsParams``."""
if self._parser_structured_outputs is not None:
return self._parser_structured_outputs
if self.text is None or self.text.format is None:
return self.structured_outputs
@@ -526,6 +526,7 @@ class OpenAIServingResponses(GenerateBaseServing):
trace_headers=trace_headers,
session_id=session_id,
reasoning_parser_kwargs=reasoning_parser_kwargs,
reasoning_ended=True if request._grammar_from_parser else None,
)
generators.append(generator)
@@ -647,6 +648,7 @@ class OpenAIServingResponses(GenerateBaseServing):
trace_headers: Mapping[str, str] | None = None,
session_id: str | None = None,
reasoning_parser_kwargs: dict[str, Any] | None = None,
reasoning_ended: bool | None = None,
):
max_model_len = self.model_config.max_model_len
cache_salt = cast(str | None, engine_input.get("cache_salt"))
@@ -673,6 +675,7 @@ class OpenAIServingResponses(GenerateBaseServing):
priority=priority,
session_id=session_id,
reasoning_parser_kwargs=reasoning_parser_kwargs,
reasoning_ended=reasoning_ended,
)
async for res in generator:
+2 -2
View File
@@ -325,8 +325,8 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
it is used to create the progress bar.
If `False`, no progress bar is created.
lora_request: LoRA request to use for generation, if any.
chat_template: The chat template to use for the scoring. If None, we
use the model's default chat template.
chat_template: The chat template to use for scoring. If None, use
the chat template passed to the LLM constructor.
tokenization_kwargs: Overrides for `tokenizer.encode`.
Returns:
@@ -548,7 +548,11 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
yield ScoringRenderParams(
data_1=data_1[i],
data_2=data_2[i],
chat_template=ctx.chat_template,
chat_template=(
self.chat_template
if ctx.chat_template is None
else ctx.chat_template
),
max_tokens_per_query=max_tokens_per_query,
max_tokens_per_doc=max_tokens_per_doc,
tok_params=tok_params,
+7
View File
@@ -247,6 +247,7 @@ if TYPE_CHECKING:
VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: int = 163840
VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS: int = 1
VLLM_ENFORCE_STRICT_TOOL_CALLING: bool = True
VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT: bool = True
VLLM_MQ_MAX_CHUNK_BYTES_MB: int = 16
VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: int = 300
VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS: int = 5
@@ -1854,6 +1855,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_ENFORCE_STRICT_TOOL_CALLING": lambda: (
os.getenv("VLLM_ENFORCE_STRICT_TOOL_CALLING", "True").lower() in ("true", "1")
),
# Constrain every `beam` output to its response template's format, also
# without tool or structured output constraints.
"VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT": lambda: (
os.getenv("VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT", "True").lower() in ("true", "1")
),
# Control the max chunk bytes (in MB) for the rpc message queue.
# Object larger than this threshold will be broadcast to worker
# processes via zmq.
@@ -2376,6 +2382,7 @@ def compile_factors() -> dict[str, object]:
"VLLM_RANDOMIZE_DP_DUMMY_INPUTS",
"VLLM_MODEL_REDIRECT_PATH",
"VLLM_HOST_IP",
"VLLM_RAY_BUNDLE_INDICES",
"VLLM_ELASTIC_EP_SCALE_UP_LAUNCH",
"VLLM_FORCE_AOT_LOAD",
"S3_ACCESS_KEY_ID",
@@ -15,6 +15,7 @@ from vllm.lora.ops.triton_ops.utils import (
get_lora_op_configs,
supports_pdl,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import direct_register_custom_op
@@ -231,6 +232,10 @@ def _lora_expand(
torch.bfloat16,
]:
CAST_TYPE = True
# Pipelined wgmma with a register A operand is miscompiled by
# ptxas < 13.3 on SM90 (triton-lang/triton#9433).
if current_platform.is_cuda() and current_platform.is_device_capability(90):
NUM_STAGES = 1
# TODO (varun): This grid formulation maximizes parallelization at the
# cost of wasteful thread block launch when only a few input tokens require
@@ -356,10 +356,12 @@ class LLBf16Dotprod:
if tidx == 0:
for m in cutlass.range_constexpr(M):
partials = sm[m, None].load()
gC[m, n_idx] = partials.reduce(
cute.ReductionOp.ADD,
init_val=cutlass.Float32(0.0),
reduction_profile=0,
)
gC[m, n_idx] = cutlass.Float32(
partials.reduce(
cute.ReductionOp.ADD,
init_val=cutlass.Float32(0.0),
reduction_profile=0,
)
).to(gC.element_type)
if const_expr(self.use_pdl):
cute.arch.griddepcontrol_launch_dependents()
@@ -47,7 +47,7 @@ class LLBf16SplitK:
:param ab_dtype: Element type for A and B operands.
:param acc_dtype: Accumulator type used by MMA and reductions.
:param out_dtype: Output element type. The public wrapper uses fp32.
:param out_dtype: Output element type (fp32 or bf16).
:param tile_n: CTA tile size in N. M is fixed to 16 for router batches.
:type tile_n: int
:param tile_k: K tile size staged through shared memory.
@@ -580,6 +580,6 @@ class LLBf16SplitK:
reduction_profile=0,
)
)
gC[local_coord] = acc
gC[local_coord] = cutlass.Float32(acc).to(gC.element_type)
cute.arch.sync_threads()
@@ -76,8 +76,8 @@ def make_fake_gemm_tensors(
a_dtype: torch.dtype,
b_dtype: torch.dtype,
divisibility: int,
output_dtype: torch.dtype = torch.float32,
):
from cutlass import Float32
from quack.compile_utils import make_fake_tensor
hidden_states = make_fake_tensor(
@@ -86,7 +86,7 @@ def make_fake_gemm_tensors(
router_weight = make_fake_tensor(
cutlass_dtype(b_dtype), (N, K), divisibility=divisibility
)
output = make_fake_tensor(Float32, (M, N), divisibility=1)
output = make_fake_tensor(cutlass_dtype(output_dtype), (M, N), divisibility=1)
return hidden_states, router_weight, output
@@ -97,6 +97,7 @@ def validate_common_gemm_inputs(
*,
op_name: str,
k_multiple: int | None = None,
output_dtypes: tuple[torch.dtype, ...] = (torch.float32,),
) -> None:
if hidden_states.dim() != 2 or router_weight.dim() != 2:
raise ValueError("hidden_states and router_weight must be 2D tensors")
@@ -106,8 +107,11 @@ def validate_common_gemm_inputs(
raise ValueError(
"hidden_states and router_weight must be on the same CUDA device"
)
if output_dtype != torch.float32:
raise ValueError(f"{op_name} only supports output_dtype=torch.float32")
if output_dtype not in output_dtypes:
raise ValueError(
f"{op_name} only supports output_dtype="
+ " or ".join(str(dtype) for dtype in output_dtypes)
)
if hidden_states.shape[1] != router_weight.shape[1]:
raise ValueError(
"hidden_states and router_weight must have matching K dimensions"
@@ -47,6 +47,8 @@ _SM100F_TUNED_SPLITK_CONFIGS: dict[tuple[int, int], dict[int, tuple[int, int]]]
**{M: (8, 5) for M in (5, 8)},
9: (8, 2),
},
(5120, 1792): {M: (2, 4) for M in range(5, 17)},
(5120, 3584): {M: (2, 4) for M in range(5, 17)},
(7168, 256): {14: (8, 2)},
(6144, 256): {M: (8, 2) for M in (9, 12, 16)},
(7168, 384): {M: (7, 5) for M in (13, 16)},
@@ -101,39 +103,52 @@ class LLBf16Gemm:
bs: int = 0
split_k: int = 0
num_stages: int = 0
output_dtype: torch.dtype = torch.float32
def __init__(self, *, prefetch_pdl_weights: bool = False) -> None:
self._prefetch_pdl_weights = prefetch_pdl_weights
# Dot-prod: keyed on (M, K, bs), because M and K are Constexpr.
self._compiled_cache: dict[tuple[int, int, int], Any] = {}
# Split-K: keyed on (split_k, num_stages), fully shape-dynamic.
self._splitk_cache: dict[tuple[int, int], Any] = {}
# M and K are compile-time constants for dot-product.
self._compiled_cache: dict[tuple[int, int, int, torch.dtype], Any] = {}
# Split-K is shape-dynamic; both caches distinguish output dtype.
self._splitk_cache: dict[tuple[int, int, torch.dtype], Any] = {}
def dispatch(self, *, M: int, K: int, N: int) -> CompileKey:
def dispatch(
self, *, M: int, K: int, N: int, output_dtype: torch.dtype = torch.float32
) -> CompileKey:
tuned_bs, tuned_splitk = _arch_tuned_configs()
if M <= _DEFAULT_DOTPROD_MAX_M or K < 2048:
bs = tuned_bs.get((K, N), {}).get(M, _DEFAULT_DOTPROD_BS)
return self.CompileKey(backend="dotprod", M=M, K=K, bs=bs)
return self.CompileKey(
backend="dotprod", M=M, K=K, bs=bs, output_dtype=output_dtype
)
split_k, num_stages = tuned_splitk.get((K, N), {}).get(
M, _DEFAULT_SPLITK_CONFIG
)
return self.CompileKey(backend="splitk", split_k=split_k, num_stages=num_stages)
return self.CompileKey(
backend="splitk",
split_k=split_k,
num_stages=num_stages,
output_dtype=output_dtype,
)
def get_warmup_keys(
self,
*,
shapes: Iterable[tuple[int, int]],
m_values: Iterable[int],
output_dtype: torch.dtype = torch.float32,
) -> list[CompileKey]:
return list(
dict.fromkeys(
self.dispatch(M=M, K=K, N=N) for K, N in shapes for M in m_values
self.dispatch(M=M, K=K, N=N, output_dtype=output_dtype)
for K, N in shapes
for M in m_values
)
)
@staticmethod
def _fake_gemm_tensors(*, M, K, N, divisibility: int):
def _fake_gemm_tensors(*, M, K, N, divisibility: int, output_dtype: torch.dtype):
return make_fake_gemm_tensors(
M=M,
K=K,
@@ -141,10 +156,13 @@ class LLBf16Gemm:
a_dtype=torch.bfloat16,
b_dtype=torch.bfloat16,
divisibility=divisibility,
output_dtype=output_dtype,
)
def _compile_splitk(self, compile_key: CompileKey) -> None:
cute, _ = cute_context()
from cutlass import BFloat16, Float32
from ._ll_bf16_splitk import LLBf16SplitK
hidden_states, router_weight, output = self._fake_gemm_tensors(
@@ -152,8 +170,12 @@ class LLBf16Gemm:
K=cute.sym_int(),
N=cute.sym_int(),
divisibility=8,
output_dtype=compile_key.output_dtype,
)
gemm = LLBf16SplitK(
out_dtype=BFloat16
if compile_key.output_dtype == torch.bfloat16
else Float32,
tile_n=16,
tile_k=256,
num_stages=compile_key.num_stages,
@@ -169,7 +191,9 @@ class LLBf16Gemm:
current_cuda_stream(),
options="--enable-tvm-ffi",
)
self._splitk_cache[(compile_key.split_k, compile_key.num_stages)] = compiled
self._splitk_cache[
(compile_key.split_k, compile_key.num_stages, compile_key.output_dtype)
] = compiled
logger.debug(
"Compiled ll_bf16_splitk: sk=%d ns=%d",
compile_key.split_k,
@@ -187,6 +211,7 @@ class LLBf16Gemm:
K=compile_key.K,
N=N,
divisibility=stride_divisibility,
output_dtype=compile_key.output_dtype,
)
gemm = LLBf16Dotprod(
k=compile_key.K,
@@ -205,7 +230,9 @@ class LLBf16Gemm:
current_cuda_stream(),
options="--enable-tvm-ffi --ptxas-options -maxrregcount=64",
)
self._compiled_cache[(compile_key.M, compile_key.K, compile_key.bs)] = compiled
self._compiled_cache[
(compile_key.M, compile_key.K, compile_key.bs, compile_key.output_dtype)
] = compiled
logger.debug(
"Compiled ll_bf16_dotprod: M=%d, K=%d, bs=%d",
compile_key.M,
@@ -215,12 +242,21 @@ class LLBf16Gemm:
def compile(self, compile_key: CompileKey) -> None:
if compile_key.backend == "splitk":
splitk_cache_key = (compile_key.split_k, compile_key.num_stages)
splitk_cache_key = (
compile_key.split_k,
compile_key.num_stages,
compile_key.output_dtype,
)
if splitk_cache_key not in self._splitk_cache:
self._compile_splitk(compile_key)
return
dotprod_cache_key = (compile_key.M, compile_key.K, compile_key.bs)
dotprod_cache_key = (
compile_key.M,
compile_key.K,
compile_key.bs,
compile_key.output_dtype,
)
if dotprod_cache_key not in self._compiled_cache:
self._compile_dotprod(compile_key)
@@ -229,8 +265,11 @@ class LLBf16Gemm:
*,
shapes: Iterable[tuple[int, int]],
m_values: Iterable[int],
output_dtype: torch.dtype = torch.float32,
) -> None:
for compile_key in self.get_warmup_keys(shapes=shapes, m_values=m_values):
for compile_key in self.get_warmup_keys(
shapes=shapes, m_values=m_values, output_dtype=output_dtype
):
self.compile(compile_key)
@staticmethod
@@ -251,6 +290,7 @@ class LLBf16Gemm:
output_dtype,
op_name="ll_bf16_gemm",
k_multiple=8,
output_dtypes=(torch.float32, torch.bfloat16),
)
def __call__(
@@ -258,19 +298,28 @@ class LLBf16Gemm:
hidden_states: torch.Tensor, # [M, K] bf16
router_weight: torch.Tensor, # [N, K] bf16
output_dtype: torch.dtype = torch.float32,
) -> torch.Tensor: # [M, N] fp32
) -> torch.Tensor: # [M, N]
self._validate_inputs(hidden_states, router_weight, output_dtype)
M, K = hidden_states.shape
N = router_weight.shape[0]
compile_key = self.dispatch(M=M, K=K, N=N)
compile_key = self.dispatch(M=M, K=K, N=N, output_dtype=output_dtype)
if compile_key.backend == "splitk":
splitk_cache_key = (compile_key.split_k, compile_key.num_stages)
splitk_cache_key = (
compile_key.split_k,
compile_key.num_stages,
compile_key.output_dtype,
)
if splitk_cache_key not in self._splitk_cache:
self.compile(compile_key)
kernel = self._splitk_cache[splitk_cache_key]
else:
dotprod_cache_key = (compile_key.M, compile_key.K, compile_key.bs)
dotprod_cache_key = (
compile_key.M,
compile_key.K,
compile_key.bs,
compile_key.output_dtype,
)
if dotprod_cache_key not in self._compiled_cache:
self.compile(compile_key)
kernel = self._compiled_cache[dotprod_cache_key]
@@ -6,19 +6,6 @@ from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import direct_register_custom_op
def _torch_hc_prenorm_gemm(
x: torch.Tensor,
fn: torch.Tensor,
out: torch.Tensor,
sqrsum: torch.Tensor,
) -> None:
assert out.shape[0] == 1
assert sqrsum.shape[0] == 1
x_float = x.float()
out[0].copy_(x_float @ fn.t())
sqrsum[0].copy_(x_float.square().sum(dim=-1))
def _hc_prenorm_gemm_outputs(
x: torch.Tensor,
fn: torch.Tensor,
@@ -2024,6 +2024,7 @@ def build_mla_chunked_context_metadata(
dcp_local_block_size: int,
dcp_virtual_block_size: int,
dcp_manager: MLADCPManager | None = None,
align_indptr: bool = False,
) -> "MLACommonPrefillMetadata.ChunkedContextMetadata | None":
"""Build chunked-context metadata for an MLA prefill.
@@ -2043,6 +2044,7 @@ def build_mla_chunked_context_metadata(
dcp_local_block_size: Per-rank interleave block size for DCP.
dcp_virtual_block_size: ``dcp_local_block_size * dcp_world_size``.
dcp_manager: Shared MLA DCP collective manager.
align_indptr: Align packed int32 indptr slices to 16 bytes for CuTe DSL.
Returns:
The chunked-context metadata, or None when no prefill has any context.
@@ -2123,6 +2125,15 @@ def build_mla_chunked_context_metadata(
num_tokens = sum(plan.seq_lens)
num_local_tokens = num_tokens
if align_indptr:
# CuTe DSL requires 16-byte-aligned int32 indptrs.
padding = -boundary_offset % 4
cu_seq_lens_flat.extend([0] * padding)
cu_seqlens_q_flat.extend([0] * padding)
if use_dcp:
padded_local_cu_seq_lens_flat.extend([0] * padding)
boundary_offset += padding
seq_lens_flat.extend(plan.seq_lens)
cu_seq_lens_flat.extend(itertools.accumulate(plan.seq_lens, initial=0))
query_base = query_start_loc[plan.request_start]
@@ -2616,6 +2627,14 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
query_start_loc_cpu[reqs_start:] - query_start_loc_cpu[reqs_start]
)
align_indptr = False
if (
self._prefill_backend.get_name() == "TRTLLM_RAGGED"
and self.vllm_config.attention_config.trtllm_ragged_backend
== "cute-dsl"
):
align_indptr = True
chunked_context_metadata = build_mla_chunked_context_metadata(
context_lens_cpu=context_lens_cpu,
prefill_query_start_loc_cpu=prefill_query_start_loc_cpu,
@@ -2628,6 +2647,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
dcp_local_block_size=self.dcp_local_block_size,
dcp_virtual_block_size=self.dcp_virtual_block_size,
dcp_manager=self.dcp_manager,
align_indptr=align_indptr,
)
prefill_metadata = MLACommonPrefillMetadata(
@@ -359,13 +359,6 @@ class ExpertMapManager:
return True
return self._expert_map[global_id] != -1
def get_local_expert_ids(self) -> list[int]:
"""Get list of global IDs for experts on this rank."""
if self._expert_map is None:
return list(range(self.global_num_experts))
return torch.where(self._expert_map != -1)[0].tolist()
def update(
self,
moe_parallel_config: FusedMoEParallelConfig,
@@ -545,7 +545,12 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
return (workspace1, workspace2, output)
def _act_mul_quant(
self, input: torch.Tensor, output: torch.Tensor, activation: MoEActivation
self,
input: torch.Tensor,
output: torch.Tensor,
activation: MoEActivation,
expert_ends: torch.Tensor | None = None,
expert_alignment: int = 0,
) -> tuple[torch.Tensor, torch.Tensor]:
block_k = self._ACT_BLOCK_K
scale_fmt = DeepGemmQuantScaleFMT.from_oracle()
@@ -561,6 +566,8 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
output_q=output,
group_size=block_k,
clamp_limit=self.gemm1_clamp_limit,
expert_ends=expert_ends,
expert_alignment=expert_alignment,
)
use_ue8m0 = scale_fmt == DeepGemmQuantScaleFMT.FLOAT32_CEIL_UE8M0
return silu_mul_per_token_group_quant_fp8_colmajor(
@@ -634,7 +641,8 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
a1q_perm = _resize_cache(
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, K)
)
a1q, a1q_scale, expert_ids, inv_perm, align_used = deepgemm_moe_permute(
use_psum_layout = current_platform.is_device_capability_family(100)
a1q, a1q_scale, grouped_layout, inv_perm, align_used = deepgemm_moe_permute(
aq=a1q,
aq_scale=a1q_scale,
topk_ids=topk_ids,
@@ -642,6 +650,7 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
expert_map=expert_map,
expert_tokens_meta=expert_tokens_meta,
aq_out=a1q_perm,
use_psum_layout=use_psum_layout,
)
assert a1q.size(0) == M_sum
@@ -655,18 +664,23 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
(a1q, a1q_scale),
(w1.view(torch.int8), self.w1_scale),
mm1_out,
expert_ids,
grouped_layout,
use_psum_layout=use_psum_layout,
recipe_a=(1, self._ACT_BLOCK_K),
recipe_b=(1, self._WEIGHT_BLOCK_K),
)
# SwiGLU activation + FP8 requant
# SiLU+mul and FP8 requant use the same live expert ranges as GEMM.
activation_out_dim = self.adjust_N_for_activation(N, activation)
quant_out = _resize_cache(
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, activation_out_dim)
)
a2q, a2q_scale = self._act_mul_quant(
input=mm1_out.view(-1, N), output=quant_out, activation=activation
input=mm1_out.view(-1, N),
output=quant_out,
activation=activation,
expert_ends=grouped_layout if use_psum_layout else None,
expert_alignment=align_used if use_psum_layout else 0,
)
# FC2: FP8 activations x FP4 weights
@@ -675,7 +689,8 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
(a2q, a2q_scale),
(w2.view(torch.int8), self.w2_scale),
mm2_out,
expert_ids,
grouped_layout,
use_psum_layout=use_psum_layout,
recipe_a=(1, self._ACT_BLOCK_K),
recipe_b=(1, self._WEIGHT_BLOCK_K),
)
@@ -517,6 +517,7 @@ _TILE_CONFIG_OVERRIDES = {
256: (128, 32),
512: (128, 19),
},
(5120, 256): {64: (64, 64)}, # GB200
(6144, 128): { # MiniMax-M3
64: (32, 64),
128: (64, 64),
@@ -533,7 +534,7 @@ def _pick_tile_config(N: int, K: int, M: int, num_sms: int) -> tuple[int, int]:
if 32 < N <= 512:
token_bucket = triton.next_power_of_2(N)
overrides = _TILE_CONFIG_OVERRIDES.get((K, M))
if overrides is not None:
if overrides is not None and token_bucket in overrides:
BN, split_k = overrides[token_bucket]
return BN, min(split_k, k_tiles)
@@ -609,6 +610,8 @@ def _bf16x3_router_gemm(X: torch.Tensor, W: torch.Tensor) -> torch.Tensor:
"""Return ``X @ W.T`` using the SM100 BF16x3 router GEMM kernel."""
N, K = X.shape
M, _ = W.shape
if N == 0:
return X.new_empty((N, M), dtype=torch.float32)
num_sms = torch.cuda.get_device_properties(X.device).multi_processor_count
BN, split_k = _pick_tile_config(N, K, M, num_sms)
@@ -24,8 +24,8 @@ class GateLinear(ReplicatedLinear):
1. cuteDSL ll_bf16_gemm (SM90+, M<=16, bf16 in, fp32 out,
any shape w/ K divisible by 8)
2. fp32 specialized kernel (bf16/fp32 in, fp32 out, M<=32; SM90+
model-specific shapes, gfx950 any shape)
2. fp32 specialized kernel (bf16/fp32 in, fp32 out; SM90+
model-specific shapes and batch limits, gfx950 any shape with M<=32)
3. cuteDSL ll_fp32w_gemm (SM90+, M<=32, bf16/fp32 in, fp32 out,
any shape)
4. bf16x3 CuteDSL kernel (SM100, bf16 in, fp32 weight)
@@ -46,9 +46,14 @@ class GateLinear(ReplicatedLinear):
split-K from the token count and skip the batch-invariant linear path.
"""
# (hidden_size, num_experts) pairs with an instantiated fp32 kernel:
# (3072, 256) -> MiniMax-M2/M2.5, (6144, 128) -> MiniMax-M3
FP32_SUPPORTED_SHAPES = {(3072, 256), (6144, 128)}
# (hidden_size, num_experts) pairs with an instantiated fp32 kernel.
FP32_SUPPORTED_SHAPES = {
(3072, 256),
(6144, 128),
(5120, 256),
(4096, 352),
(8192, 256),
}
FP32_MAX_TOKENS = 32
# Largest batch for which tier 1 beats the cuBLAS epilogue. Subclasses
# whose router shape stays ahead further up the range may raise it.
@@ -454,9 +459,9 @@ def fp32_router_gemm_dispatch_impl(
weight: torch.Tensor,
allow_bf16x3_router_gemm: bool,
) -> torch.Tensor:
"""Dynamically run fp32 specialized gemm if num_tokens <= FP32_MAX_TOKENS,
otherwise optionally run the experimental BF16x3 kernel for medium/large
SM100 router batches, then fall back to F.linear.
"""Run the FP32 kernel for small batches, with shape-specific limits.
Larger SM100 BF16 batches use BF16x3; other inputs fall back to F.linear.
This must be wrapped in a custom op because our torch.compile integration
does not support runtime dispatching on num_tokens.
"""
@@ -473,7 +478,15 @@ def fp32_router_gemm_dispatch_impl(
if weight.dtype == torch.bfloat16:
return torch.mm(x, weight.T, out_dtype=torch.float32)
return torch.nn.functional.linear(x.float(), weight)
if x.shape[0] <= _FP32_ROUTER_GEMM_MAX_TOKENS:
max_tokens = _FP32_ROUTER_GEMM_MAX_TOKENS
if (
allow_bf16x3_router_gemm
and x.dtype == torch.bfloat16
and weight.shape == (256, 5120)
):
# GB200: four token groups win through 48 for multiples of eight.
max_tokens = 48 if x.shape[0] % 8 == 0 else 16
if x.shape[0] <= max_tokens:
return ops.fp32_router_gemm(x, weight)
if allow_bf16x3_router_gemm and x.dtype == torch.bfloat16:
@@ -9,139 +9,162 @@
from packaging import version
from vllm.model_executor.layers.mamba.ops.triton_helpers import fast_exp
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
TRITON_22 = version.parse(triton.__version__) >= version.parse("2.2.0")
def _drop_racy_wgmma_configs(configs: list[triton.Config]) -> list[triton.Config]:
"""Drop the configs that Triton miscompiles on sm90.
With whole warpgroups (num_warps % 4 == 0) and BLOCK_SIZE_M % 64 == 0,
`tl.dot(cb, x)` lowers to wgmma with `cb` in registers. With
num_stages > 1, Triton leaves that wgmma running across the loop
back-edge while the next iteration overwrites the registers of `cb`,
which writes NaN/Inf into whole output rows.
"""
if not (current_platform.is_cuda() and current_platform.is_device_capability(90)):
return configs
return [
c
for c in configs
if c.num_stages == 1
or c.num_warps % 4 != 0
or c.kwargs["BLOCK_SIZE_M"] % 64 != 0
]
@triton.autotune(
configs=[
# =================================================================
# Higher warp count configs for better latency hiding
# More warps = more instructions in flight = better memory latency hiding
# =================================================================
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=8,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=8,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=8,
),
# Smaller tiles with more stages for software pipelining
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=3,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 64},
num_stages=2,
num_warps=4,
),
# =================================================================
# Low register pressure configs (num_stages=1) for large dstate
# =================================================================
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
num_stages=1,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=1,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=1,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=1,
num_warps=4,
),
# num_stages=2 configs - moderate register pressure
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
num_stages=2,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=4,
),
# Original configs for larger dstate values
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 64},
num_stages=3,
num_warps=8,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=5,
num_warps=2,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=5,
num_warps=2,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=2,
),
],
configs=_drop_racy_wgmma_configs(
[
# =================================================================
# Higher warp count configs for better latency hiding
# More warps = more instructions in flight = better memory latency hiding
# =================================================================
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=8,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=8,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=8,
),
# Smaller tiles with more stages for software pipelining
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=3,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 64},
num_stages=2,
num_warps=4,
),
# =================================================================
# Low register pressure configs (num_stages=1) for large dstate
# =================================================================
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
num_stages=1,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=1,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=1,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=1,
num_warps=4,
),
# num_stages=2 configs - moderate register pressure
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
num_stages=2,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=4,
),
# Original configs for larger dstate values
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 64},
num_stages=3,
num_warps=8,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=5,
num_warps=2,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=5,
num_warps=2,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=2,
),
]
),
key=["chunk_size", "hdim", "dstate", "IS_CAUSAL"],
)
@triton.jit
@@ -12,35 +12,6 @@ from vllm.triton_utils import tl, triton
logger = logging.getLogger(__name__)
def block_dequant(
x_q_block: torch.Tensor,
x_s: torch.Tensor,
block_size: list[int],
) -> torch.Tensor:
"""This function conducts block-wise dequantization.
The inputs are block-wise quantization tensor `x_q_block`,
block-wise quantization scale and the block size.
The outputs are dequantized tensor.
"""
block_n, block_k = block_size[0], block_size[1]
n, k = x_q_block.shape
n_tiles = (n + block_n - 1) // block_n
k_tiles = (k + block_k - 1) // block_k
assert n_tiles == x_s.shape[0]
assert k_tiles == x_s.shape[1]
x_dq_block = x_q_block.to(torch.float32)
for i in range(k_tiles):
for j in range(n_tiles):
x_dq_block[
j * block_n : min((j + 1) * block_n, n),
i * block_k : min((i + 1) * block_k, k),
] *= x_s[j][i]
return x_dq_block
if current_platform.is_rocm():
@triton.jit
@@ -1404,14 +1404,17 @@ class DiffusionSampler:
# unaffected; only the canvas exploration is constrained. Applied
# before canvas padding so phantom positions stay uniform.
if num_decode > 0:
top_k, top_p = self.sampling_states.get_top_k_top_p(
top_k, top_p, has_topp_only_rows = self.sampling_states.get_top_k_top_p(
decode_slots.repeat_interleave(
valid_canvas_len, output_size=int(valid_canvas_len_np.sum())
),
decode_slots_np,
logits.shape[-1],
)
if top_k is not None or top_p is not None:
logits = apply_top_k_top_p(logits.float(), top_k, top_p)
logits = apply_top_k_top_p(
logits.float(), top_k, top_p, has_topp_only_rows=has_topp_only_rows
)
# Where each decode request's rows start in the flat logits. Tiles
# below gather and pad a request to the tile's width.
+42
View File
@@ -64,6 +64,7 @@ if TYPE_CHECKING:
from vllm.tasks import ScoreType
from vllm.tokenizers import TokenizerLike
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, KVCacheSpec
from vllm.v1.worker.encoder_cudagraph_defs import (
EncoderCudaGraphCaptureInputs,
EncoderCudaGraphConfig,
@@ -670,6 +671,47 @@ def supports_score_template(
return getattr(model, "supports_score_template", False)
@runtime_checkable
class SupportsKVCacheGrouping(Protocol):
"""The interface required for models that own their KV cache grouping.
.. note::
Experimental. The classmethod contract may change as model-owned
grouping stabilizes.
Implementing it lets a model replace the default KV cache grouping (which
layers share a block table) while Core keeps owning placement, byte
accounting, and admission.
"""
@classmethod
def get_kv_cache_groups(
cls,
vllm_config: "VllmConfig",
kv_cache_spec: dict[str, "KVCacheSpec"],
) -> list["KVCacheGroupSpec"]:
"""Split the model's layers into logical KV cache groups."""
...
@overload
def supports_kv_cache_grouping(
model: type[object],
) -> TypeIs[type[SupportsKVCacheGrouping]]: ...
@overload
def supports_kv_cache_grouping(
model: object,
) -> TypeIs[SupportsKVCacheGrouping]: ...
def supports_kv_cache_grouping(
model: type[object] | object,
) -> TypeIs[type[SupportsKVCacheGrouping]] | TypeIs[SupportsKVCacheGrouping]:
return isinstance(model, SupportsKVCacheGrouping)
@runtime_checkable
class SupportsLoRA(Protocol):
"""The interface required for all models that support LoRA."""
+3
View File
@@ -50,6 +50,7 @@ from .interfaces import (
is_hybrid,
requires_raw_input_tokens,
supports_hc_sp,
supports_kv_cache_grouping,
supports_multimodal,
supports_multimodal_encoder_tp_data,
supports_multimodal_raw_input_only,
@@ -882,6 +883,7 @@ class _ModelInfo:
supports_replayssm: bool
supports_transcription: bool
supports_transcription_only: bool
supports_kv_cache_grouping: bool
supported_video_pruning_methods: tuple[str, ...]
supports_mm_device_do_normalize: bool
@@ -913,6 +915,7 @@ class _ModelInfo:
supports_transcription_only=(
supports_transcription(model) and model.supports_transcription_only
),
supports_kv_cache_grouping=supports_kv_cache_grouping(model),
has_noops=has_noops(model),
supported_video_pruning_methods=getattr(
model, "supported_video_pruning_methods", ()
@@ -11,11 +11,14 @@ from typing import TYPE_CHECKING
import vllm.envs as envs
from vllm.config import CacheConfig, VllmConfig, replace
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.distributed.parallel_state import GroupCoordinator
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
logger = init_logger(__name__)
def _normalize_cache_config_for_hash(cache_config: CacheConfig) -> CacheConfig:
"""Normalize cache config for hashing"""
@@ -55,6 +58,53 @@ def resolve_flashinfer_autotune_file(runner: "GPUModelRunner") -> Path:
return output_dir / "autotune_configs.json"
def use_flashinfer_autotune_v2(runner: "GPUModelRunner") -> bool:
from vllm.distributed.parallel_state import get_node_count, get_world_group
from vllm.utils.flashinfer import has_flashinfer_autotune_v2
# Elastic EP transfers tuning state to joining workers with v1
# save_configs/load_configs, which do not export managed-store winners.
if runner.vllm_config.parallel_config.enable_elastic_ep:
return False
# PP stages profile independently. Keep their group-scoped v1 files until
# managed-cache reads and publishes can be isolated across tuning groups.
if runner.vllm_config.parallel_config.pipeline_parallel_size > 1:
return False
if not has_flashinfer_autotune_v2():
return False
# Use the initialized topology: Ray may span nodes with config.nnodes=1.
# A cache hit skips profiling collectives, so node-local stores with
# different contents cannot safely participate in synchronized tuning.
if get_node_count() != 1:
logger.info_once(
"Using legacy FlashInfer autotune cache synchronization for "
"multi-node deployments; managed-cache synchronization is "
"currently supported only within one node."
)
return False
# Lift this gate after pinning a release with the fix below and validating
# distributed managed-cache tuning: https://github.com/flashinfer-ai/flashinfer/pull/6185
if get_world_group().world_size > 1:
logger.info_once(
"Using legacy FlashInfer autotune cache synchronization for "
"multi-rank deployments until FlashInfer coordinates cache hits."
)
return False
return True
def resolve_flashinfer_autotune_v2_root() -> Path | None:
"""Placement root for FlashInfer's managed autotune store.
Unlike :func:`resolve_flashinfer_autotune_file`, no vLLM-side identity
(config hash, flashinfer version, arch) is folded into the path: the
managed store keys entries per op and hashes the software/hardware
environment itself. ``None`` selects FlashInfer's default root.
"""
override_dir = envs.VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR
return Path(override_dir).expanduser() if override_dir else None
def sync_flashinfer_autotune_cache(
runner: "GPUModelRunner",
group: "GroupCoordinator",
@@ -9,6 +9,8 @@ import torch
from vllm.logger import init_logger
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
resolve_flashinfer_autotune_file,
resolve_flashinfer_autotune_v2_root,
use_flashinfer_autotune_v2,
write_flashinfer_autotune_cache,
)
from vllm.platforms import current_platform
@@ -145,7 +147,6 @@ def _run_flashinfer_sparse_mla_decode_autotune(
world = get_world_group()
is_leader = world.rank_in_group == 0
cache_path = resolve_flashinfer_autotune_file(runner)
dummy_run_kwargs = dict(
num_tokens=num_tokens,
@@ -155,6 +156,13 @@ def _run_flashinfer_sparse_mla_decode_autotune(
create_mixed_batch=True,
)
if use_flashinfer_autotune_v2(runner):
return _run_sparse_mla_decode_autotune_v2(
worker, num_tokens, log_label, dummy_run_kwargs
)
cache_path = resolve_flashinfer_autotune_file(runner)
if is_leader:
logger.info(
"Autotuning FlashInfer SM120 sparse MLA %s decode with cache: %s",
@@ -223,6 +231,58 @@ def _run_flashinfer_sparse_mla_decode_autotune(
return True
def _run_sparse_mla_decode_autotune_v2(
worker: "Worker",
num_tokens: int,
log_label: str,
dummy_run_kwargs: dict,
) -> bool:
"""Run every rank's sparse warmup in the managed autotune context.
FlashInfer owns persistence; some sparse kernels use a separate calibration
cache instead of managed winner entries.
"""
from flashinfer import autotune_v2, autotune_v2_reload
from vllm.distributed.parallel_state import get_world_group
runner = worker.model_runner
world = get_world_group()
cache_root = resolve_flashinfer_autotune_v2_root()
if world.rank_in_group == 0:
logger.info(
"Autotuning FlashInfer SM120 sparse MLA %s decode with the managed "
"autotune context (root=%s)",
log_label,
cache_root if cache_root is not None else "flashinfer default",
)
with torch.inference_mode():
if _uses_v2_model_runner(runner) and runner.max_num_reqs >= 2:
v2_runner = cast("V2GPUModelRunner", runner)
warmup_executed = run_mixed_prefill_decode_warmup(
v2_runner,
worker.execute_model,
worker.sample_tokens,
num_tokens,
mixed_step_context=autotune_v2(mode="tune", cache_root=cache_root),
req_id_prefix="_sparse_mla_v2_warmup",
)
else:
warmup_executed = True
with autotune_v2(mode="tune", cache_root=cache_root):
runner._dummy_run(**dummy_run_kwargs)
if not warmup_executed:
return False
if world.world_size > 1:
world.barrier()
autotune_v2_reload()
world.barrier()
return True
def _flashinfer_sparse_mla_decode_autotune(
worker: "Worker",
num_tokens: int,
+33 -6
View File
@@ -1082,6 +1082,10 @@ class JitWarmupRegistry:
from tqdm import tqdm
from vllm.distributed import is_global_first_rank
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
VllmTritonJitKernel,
compile_triton_warmup,
)
kernel_items: list[tuple[VllmJitKernel[Any], dict[Any, None]]] = []
for kernel, registrations in self._registrations.items():
@@ -1102,17 +1106,40 @@ class JitWarmupRegistry:
if not kernel_items:
return
parallel_items: list[tuple[VllmJitKernel[Any], dict[Any, None]]] = []
serial_items = []
for kernel, compile_keys in kernel_items:
if (
isinstance(kernel, VllmTritonJitKernel)
and kernel.supports_parallel_compile
):
parallel_items.append((kernel, compile_keys))
else:
serial_items.append((kernel, compile_keys))
total_keys = sum(len(compile_keys) for _, compile_keys in kernel_items)
with tqdm(
kernel_items,
total=len(kernel_items),
desc=f"JIT kernel warmup ({total_keys} compile keys)",
disable=not is_global_first_rank(),
dynamic_ncols=True,
unit="kernel",
) as progress:
for kernel, compile_keys in progress:
progress.set_postfix_str(
f"{kernel.__class__.__name__} ({len(compile_keys)} keys)",
refresh=False,
def compile_kernels(
items: list[tuple[VllmJitKernel[Any], dict[Any, None]]],
) -> None:
for kernel, compile_keys in items:
progress.set_postfix_str(
f"{kernel.__class__.__name__} ({len(compile_keys)} keys)",
refresh=False,
)
kernel.compile_many(compile_keys)
progress.update()
if parallel_items:
compile_triton_warmup(
lambda: compile_kernels(parallel_items),
sum(len(keys) for _, keys in parallel_items),
)
kernel.compile_many(compile_keys)
compile_kernels(serial_items)
@@ -6,7 +6,7 @@ from abc import abstractmethod
from collections.abc import Callable, Hashable, Iterable, Mapping
from concurrent.futures import ThreadPoolExecutor
from contextlib import nullcontext, suppress
from contextvars import copy_context
from contextvars import ContextVar, copy_context
from dataclasses import dataclass, field
from functools import cache, cached_property, update_wrapper
from typing import Any, Generic, ParamSpec, Protocol, TypeVar, cast, overload
@@ -207,6 +207,31 @@ class TritonWarmupTensor:
return result if dim is None else result[dim]
_shared_compile_pool: ContextVar[bool] = ContextVar(
"shared_triton_warmup_compile_pool", default=False
)
def compile_triton_warmup(compile_all: Callable[[], None], num_keys: int) -> None:
"""Share one compiler pool across all eligible warmup kernels."""
num_threads = min(envs.VLLM_TRITON_JIT_WARMUP_NUM_THREADS, num_keys)
if num_threads <= 1:
compile_all()
return
def compile_in_context() -> None:
_shared_compile_pool.set(True)
with (
ThreadPoolExecutor(max_workers=num_threads) as executor,
triton.AsyncCompileMode(executor),
):
# Dispatch mutates owner state and must stay on this thread.
compile_all()
# Older Triton versions leave async mode active on compilation failure.
copy_context().run(compile_in_context)
class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
"""Triton owner whose runtime launch specification is reused for warmup."""
@@ -230,32 +255,25 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
self._warming = False
self._warming_compile_key = None
@property
def supports_parallel_compile(self) -> bool:
# AMD LLVM code generation can abort under concurrent compilation.
return (
current_platform.is_cuda()
and not self._run_autotune
and getattr(triton, "AsyncCompileMode", None) is not None
and envs.VLLM_TRITON_JIT_WARMUP_NUM_THREADS > 1
)
def compile_many(self, compile_keys: Iterable[CompileKeyT]) -> None:
"""Compile CUDA startup warmup variants in parallel, then wait."""
keys = list(compile_keys)
num_threads = min(envs.VLLM_TRITON_JIT_WARMUP_NUM_THREADS, len(keys))
async_compile = getattr(triton, "AsyncCompileMode", None)
# AMD LLVM code generation can abort under concurrent compilation.
if (
not current_platform.is_cuda()
or self._run_autotune
or async_compile is None
or num_threads <= 1
):
if not self.supports_parallel_compile or _shared_compile_pool.get():
return super().compile_many(keys)
def compile_all() -> None:
with (
ThreadPoolExecutor(max_workers=num_threads) as executor,
async_compile(executor),
):
# Dispatch stays on this thread: compile() mutates owner state.
# Only Triton's underlying compiler work runs in the pool.
super(VllmTritonJitKernel, self).compile_many(keys)
# Older Triton versions leave async mode active on compilation failure.
# Isolate that state so it cannot affect later runtime JIT compilation.
copy_context().run(compile_all)
compile_triton_warmup(
lambda: super(VllmTritonJitKernel, self).compile_many(keys), len(keys)
)
@cached_property
def _kernel_arg_names(self) -> tuple[str, ...]:
+42 -30
View File
@@ -23,6 +23,8 @@ from vllm.model_executor.warmup.cutedsl_warmup import cutedsl_warmup
from vllm.model_executor.warmup.deep_gemm_warmup import deep_gemm_warmup
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
resolve_flashinfer_autotune_file,
resolve_flashinfer_autotune_v2_root,
use_flashinfer_autotune_v2,
)
from vllm.model_executor.warmup.flashinfer_sparse_mla_warmup import (
autotune_hisparse_flashinfer_attention,
@@ -419,7 +421,7 @@ def flashinfer_autotune(runner: "GPUModelRunner", *, skip_attn: bool = False) ->
otherwise the world group tunes together. Per-tactic timings are
averaged over the tuning group so all its ranks select the same tactic.
Results are persisted per rank: FlashInfer keys MoE entries by tp/ep rank
Legacy results are persisted per rank: FlashInfer keys MoE entries by tp/ep rank
(``MoERunner.get_cache_key_extras``), so one rank's file only hits on that
rank. A rank with a cache hit skips the per-tactic reduce the others block
in, so ranks keep loaded configs only if every rank in the tuning group
@@ -450,38 +452,45 @@ def flashinfer_autotune(runner: "GPUModelRunner", *, skip_attn: bool = False) ->
)
autotune_kwargs["skip_ops"] = skip_ops
cache_path = resolve_flashinfer_autotune_file(runner)
# The world group only spans DP ranks when vLLM folds DP into it.
dp_rank = runner.vllm_config.parallel_config.data_parallel_rank
cache_path = cache_path.with_name(
f"{cache_path.stem}_dp{dp_rank}_rank{world.rank_in_group}{cache_path.suffix}"
)
if is_leader:
logger.info_once("Using FlashInfer autotune cache file: %s", cache_path)
use_autotune_v2 = use_flashinfer_autotune_v2(runner)
if use_autotune_v2:
from flashinfer import autotune_v2, autotune_v2_reload
# We skip EPLB here since we don't want to record dummy metrics.
# Randomize inputs to avoid every token pick the same experts,
# which lead to some EP ranks receiving no tokens and skipping their
# MoE kernel entirely, and cause hang due to all-reduce collective
# during synchronized autotuning.
if _all_ranks_have_matching_cache(cache_path, tune_group):
loaded = tuner.load_configs(str(cache_path))
if tune_group.world_size > 1:
loaded_by_rank: list[bool | None] = [None] * tune_group.world_size
torch.distributed.all_gather_object(
loaded_by_rank, loaded, group=tune_group.cpu_group
cache_root = resolve_flashinfer_autotune_v2_root()
if is_leader:
logger.info_once(
"Using FlashInfer managed autotune cache (root=%s)",
cache_root if cache_root is not None else "flashinfer default",
)
loaded = all(loaded_by_rank)
if not loaded:
tuner.clear_cache()
autotune_context = autotune_v2(
mode="tune", cache_root=cache_root, **autotune_kwargs
)
else:
cache_path = resolve_flashinfer_autotune_file(runner)
# The world group only spans DP ranks when vLLM folds DP into it.
dp_rank = runner.vllm_config.parallel_config.data_parallel_rank
cache_path = cache_path.with_name(
f"{cache_path.stem}_dp{dp_rank}_rank{world.rank_in_group}{cache_path.suffix}"
)
if is_leader:
logger.info_once("Using FlashInfer autotune cache file: %s", cache_path)
if _all_ranks_have_matching_cache(cache_path, tune_group):
loaded = tuner.load_configs(str(cache_path))
if tune_group.world_size > 1:
loaded_by_rank: list[bool | None] = [None] * tune_group.world_size
torch.distributed.all_gather_object(
loaded_by_rank, loaded, group=tune_group.cpu_group
)
loaded = all(loaded_by_rank)
if not loaded:
tuner.clear_cache()
autotune_context = fi_utils.autotune(tune_mode=True, **autotune_kwargs)
group = tune_group.cpu_group if tune_group.world_size > 1 else None
set_autotune_process_group(group)
try:
with (
torch.inference_mode(),
fi_utils.autotune(tune_mode=True, **autotune_kwargs),
):
with torch.inference_mode(), autotune_context:
hisparse_enabled = (
runner.vllm_config.attention_config.hisparse_config is not None
)
@@ -498,7 +507,10 @@ def flashinfer_autotune(runner: "GPUModelRunner", *, skip_attn: bool = False) ->
if world.world_size > 1:
world.barrier()
# Skip the rewrite when nothing was tuned this start (every entry came from
# the file). FlashInfer gates its own autotune(cache=...) save the same way.
if tuner._dirty:
if use_autotune_v2:
if world.world_size > 1:
# All ranks serve the shared store's final state after publishing.
autotune_v2_reload()
world.barrier()
elif tuner._dirty:
tuner.save_configs(str(cache_path))
@@ -19,9 +19,11 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
def _get_kda_layer(worker: Worker) -> KimiK3DeltaAttention | None:
def _get_kda_layer(
worker: Worker, module: str = "vllm.models.kimi_k3.nvidia.kda"
) -> KimiK3DeltaAttention | None:
# Kimi model construction already imports kda. Avoid importing it here.
kda = sys.modules.get("vllm.models.kimi_k3.nvidia.kda")
kda = sys.modules.get(module)
if kda is None:
return None
@@ -172,9 +174,59 @@ def _warm_recurrent_kda(
)
def _warm_gluon_chunk_kda(layer, input_dtype: torch.dtype) -> None:
from vllm.models.kimi_k3.amd.ops.kda_prefill import (
chunk_kda_prefill,
get_gluon_chunk_kda_warmup_profiles,
)
if layer.kda_prefill_backend != "gluon":
return
kv_cache = layer.kv_cache
if not isinstance(kv_cache, (list, tuple)) or len(kv_cache) < 2:
return
state = kv_cache[1]
if not isinstance(state, torch.Tensor) or not state.numel():
return
logger.info("Warming up Kimi-K3 Gluon chunk KDA prefill kernels.")
h, d = int(layer.local_num_heads), int(layer.head_dim)
for num_seqs, seq_len in get_gluon_chunk_kda_warmup_profiles():
num_tokens = num_seqs * seq_len
shape = (1, num_tokens, h, d)
chunk_kda_prefill(
q=torch.zeros(shape, dtype=input_dtype, device=state.device),
k=torch.zeros(shape, dtype=input_dtype, device=state.device),
v=torch.zeros(shape, dtype=input_dtype, device=state.device),
raw_g=torch.zeros(shape, dtype=input_dtype, device=state.device),
raw_beta=torch.zeros(
(1, num_tokens, h), dtype=input_dtype, device=state.device
),
A_log=layer.A_log,
g_bias=layer.dt_bias,
lower_bound=layer.gate_lower_bound,
state_cache=state,
state_indices=torch.zeros(num_seqs, dtype=torch.int32, device=state.device),
has_initial_state=torch.zeros(
num_seqs, dtype=torch.bool, device=state.device
),
use_qk_l2norm_in_kernel=True,
cu_seqlens=torch.arange(
0, num_tokens + 1, seq_len, dtype=torch.int32, device=state.device
),
backend="gluon",
out=torch.empty(shape, dtype=input_dtype, device=state.device),
)
@torch.inference_mode()
def kimi_k3_triton_warmup(worker: Worker) -> None:
"""Warm Kimi-K3 Triton kernels reachable by this server."""
if current_platform.is_rocm():
layer = _get_kda_layer(worker, "vllm.models.kimi_k3.amd.kda")
if layer is not None:
_warm_gluon_chunk_kda(layer, worker.model_config.dtype)
return
if not current_platform.is_cuda():
return
+13 -2
View File
@@ -8,7 +8,7 @@ from itertools import islice
import torch
from vllm.config import VllmConfig
from vllm.distributed import get_pp_group
from vllm.distributed import get_pp_group, get_tensor_model_parallel_rank
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
@@ -71,6 +71,7 @@ class DeepseekV32DecoderLayer(torch.nn.Module):
topk_indices_buffer=topk_indices_buffer,
)
self.zero_mlp_output = False
if (
config.n_routed_experts is not None
and layer_idx >= config.first_k_dense_replace
@@ -82,7 +83,15 @@ class DeepseekV32DecoderLayer(torch.nn.Module):
quant_config=quant_config,
prefix=f"{prefix}.mlp",
)
self.mlp.experts.moe_config.skip_final_all_reduce = True
moe_cfg = self.mlp.experts.moe_config
pc = moe_cfg.moe_parallel_config
# Validated on MoRI only. DeepEP, pplx and the other combines
# keep the previous always-skip behavior.
if pc.use_mori_kernels:
# Combine is already all-reduced; the next fused norm sums TP.
self.zero_mlp_output = get_tensor_model_parallel_rank() > 0
else:
moe_cfg.skip_final_all_reduce = True
else:
self.mlp = DeepseekV2MLP(
hidden_size=config.hidden_size,
@@ -116,6 +125,8 @@ class DeepseekV32DecoderLayer(torch.nn.Module):
hidden_states, residual, self.post_attention_layernorm
)
hidden_states = self.mlp(hidden_states)
if self.zero_mlp_output:
hidden_states = torch.zeros_like(hidden_states)
return hidden_states, residual
+4
View File
@@ -578,6 +578,9 @@ def _validate_aiter_mega_moe_config(vllm_config: VllmConfig) -> None:
class DeepseekV4MoE(nn.Module):
# None: use the padding vLLM added.
experts_intermediate_pad: int | None = None
def __init__(
self,
vllm_config: VllmConfig,
@@ -726,6 +729,7 @@ class DeepseekV4MoE(nn.Module):
top_k=config.num_experts_per_tok,
hidden_size=config.hidden_size,
intermediate_size=config.moe_intermediate_size,
intermediate_pad=self.experts_intermediate_pad,
renormalize=config.norm_topk_prob,
quant_config=quant_config,
prefix=f"{prefix}.experts",
+4
View File
@@ -77,6 +77,10 @@ logger = init_logger(__name__)
class DeepseekV4MoE(DeepseekV4MoEBase):
# TODO: revert once https://github.com/ROCm/aiter/pull/6598 is in the
# pinned AITER; with the padding set, its FlyDSL MoE can output NaN.
experts_intermediate_pad = 0
def __init__(
self,
vllm_config: VllmConfig,
+29 -9
View File
@@ -4,7 +4,7 @@
Mirrors the reference ``mtp_model.py`` shipped with the checkpoint: each MTP
depth ``i`` owns ``hidden_norm`` / ``embed_norm`` RMSNorms, an ``input_proj``
(``2H -> H``) and a full Inkling transformer block (dense bf16 MLP, with the
(``2H -> H``) and a full Inkling transformer block (dense MLP, with the
same short convolutions as the backbone; its attention is full or sliding-window
per depth, selected by ``mtp_config.local_layer_ids``). When enabled, a shared
``chain_norm`` is applied after every depth; its output is both the logits input
@@ -28,18 +28,23 @@ from torch import nn
from vllm.config import VllmConfig
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.model_loader.mtp_validation import (
is_mtp_completeness_check_enabled,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.interfaces import SupportsMultiModalEmbeddings
from vllm.model_executor.models.utils import maybe_prefix
from vllm.model_executor.models.utils import WeightsMapper, maybe_prefix
from vllm.sequence import IntermediateTensors
from ..configs import InklingModelConfig
from .layernorm import InklingRMSNorm
from .model import InklingDecoderLayer, InklingReplicatedEmbedding
from .model import (
InklingDecoderLayer,
InklingReplicatedEmbedding,
_TmlForCausalLMBase,
)
from .ops.norm import embed_dual_rmsnorm_cat, embed_rmsnorm
# Checkpoint attention projections (wq_du/wk_dv/wv_dv/wr_du) -> fused qkvr.
@@ -71,7 +76,13 @@ def _select_mtp_depth_count(n_predict: int, num_spec: int | None) -> int:
class InklingMTPDepthLayer(nn.Module):
"""One MTP depth: norm both inputs, fuse (2H->H), run a Inkling block."""
def __init__(self, config: InklingModelConfig, prefix: str, is_local: bool) -> None:
def __init__(
self,
config: InklingModelConfig,
prefix: str,
is_local: bool,
quant_config: QuantizationConfig | None = None,
) -> None:
super().__init__()
self.hidden_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.embed_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -80,16 +91,17 @@ class InklingMTPDepthLayer(nn.Module):
config.hidden_size,
bias=False,
return_bias=False,
quant_config=quant_config,
prefix=f"{prefix}.input_proj",
)
# A force-dense-MLP bf16 block; ``is_local`` selects sliding-window vs
# A force-dense-MLP block; ``is_local`` selects sliding-window vs
# full attention (the swa_* head config and sliding_window window) to
# match this depth's checkpoint transformer_block weights.
self.transformer_block = InklingDecoderLayer(
config,
layer_id=0,
is_local=is_local,
quant_config=None,
quant_config=quant_config,
prefix=f"{prefix}.transformer_block",
force_dense_mlp=True,
)
@@ -131,7 +143,10 @@ class InklingMultiTokenPredictor(nn.Module):
self.layers = nn.ModuleDict(
{
str(idx): InklingMTPDepthLayer(
config, f"{prefix}.layers.{idx}", idx in local_ids
config,
f"{prefix}.layers.{idx}",
idx in local_ids,
quant_config=vllm_config.quant_config,
)
for idx in range(self.num_mtp_layers)
}
@@ -244,6 +259,11 @@ class InklingMultiTokenPredictor(nn.Module):
class InklingMTP(nn.Module, SupportsMultiModalEmbeddings):
hf_to_vllm_mapper = _TmlForCausalLMBase.hf_to_vllm_mapper | WeightsMapper(
orig_to_new_prefix={"model.mtp.": "model."},
)
packed_modules_mapping = _TmlForCausalLMBase.packed_modules_mapping
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
assert vllm_config.speculative_config is not None
@@ -418,9 +438,9 @@ def _load_inkling_mtp_weights(
continue
# Dense MLP fused gate/up + down.
if ".mlp.w13_dn.weight" in name:
if ".mlp.w13_dn." in name:
loaded_weight = _load(name.replace(".w13_dn.", ".gate_up_proj."), weight)
elif ".mlp.w2_md.weight" in name:
elif ".mlp.w2_md." in name:
loaded_weight = _load(name.replace(".w2_md.", ".down_proj."), weight)
else:
if name.endswith(".bias") and name not in params:
+61 -59
View File
@@ -4,12 +4,15 @@
from __future__ import annotations
import math
from collections.abc import Iterable, Mapping, Sequence
from typing import Any, cast
import numpy as np
import regex as re
import torch
from PIL import Image
from transformers import InklingProcessor
from transformers.feature_extraction_utils import BatchFeature
from vllm.config.multimodal import (
@@ -29,18 +32,19 @@ from vllm.multimodal.processing import (
PromptUpdate,
PromptUpdateDetails,
)
from vllm.transformers_utils.processors.inkling import (
AUDIO_MARKER_ID,
AUDIO_TOKEN_ID,
IMAGE_MARKER_ID,
IMAGE_TOKEN_ID,
InklingAudioFeatureExtractor,
InklingImageProcessor,
InklingProcessor,
)
from ..configs import InklingMMConfig
# Block-start markers (<|content_image|>, <|content_audio_input|>), kept verbatim
IMAGE_MARKER_ID = 200005
AUDIO_MARKER_ID = 200020
# Per-patch / per-frame placeholders (<|unused_200054|>, <|unused_200053|>)
IMAGE_TOKEN_ID = 200054
AUDIO_TOKEN_ID = 200053
# Long-edge upscale applied before patchifying (upstream defaults to no rescale)
DEFAULT_RESCALE_IMAGE_FRAC = 2.0
# Maximum audio tokens accepted per clip. At the dMel rate of 20 tokens/s
# (50 ms hop) this is ~10 minutes of audio. It bounds the persistent per-request
# buffers and the encoder/memory budget; longer clips are rejected up front.
@@ -57,6 +61,24 @@ class InklingMultiModalDataParser(MultiModalDataParser):
return super()._parse_audio_data(data)
def _rescale_image(
image: Image.Image, frac: float | None, max_upscaled_long_edge: int | None
) -> Image.Image:
"""Scale the long edge by `frac` with PIL LANCZOS, as the reference does."""
image = image.convert("RGB")
if frac is None:
return image
long_edge = max(image.size)
target_long_edge = long_edge * frac
if max_upscaled_long_edge is not None:
target_long_edge = min(target_long_edge, max(max_upscaled_long_edge, long_edge))
ratio = target_long_edge / long_edge
if ratio == 1.0:
return image
size = tuple(max(1, math.floor(d * ratio + 0.5)) for d in image.size)
return image.resize(size, resample=Image.Resampling.LANCZOS)
def inkling_vision_enabled(config: InklingMMConfig) -> bool:
return getattr(config.vision_config, "decoder_dmodel", None) is not None
@@ -70,38 +92,13 @@ class InklingProcessingInfo(BaseProcessingInfo):
return self.ctx.get_hf_config(InklingMMConfig)
def get_hf_processor(self, **kwargs: object) -> InklingProcessor:
config = self.get_hf_config()
vision_config = config.vision_config
audio_config = config.audio_config
image_processor = InklingImageProcessor(
patch_size=getattr(vision_config, "patch_size", None) or 40,
)
if inkling_audio_enabled(config):
audio_params = {
"n_mels": audio_config.n_mel_bins,
"num_dmel_bins": audio_config.mel_vocab_size,
"dmel_min_value": audio_config.dmel_min_value,
"dmel_max_value": audio_config.dmel_max_value,
}
else:
audio_params = {}
audio_extractor = InklingAudioFeatureExtractor(params=audio_params)
return InklingProcessor(
image_processor=image_processor,
audio_feature_extractor=audio_extractor,
tokenizer=self.get_tokenizer(),
)
return self.ctx.get_hf_processor(InklingProcessor, **kwargs)
def get_data_parser(self) -> MultiModalDataParser:
# Audio inputs must be resampled to the dMel feature extractor's rate
# before `process_audios` (see InklingAudioFeatureExtractor._decode_one).
# Without a target_sr the default parser raises on any audio input.
extractor = self.get_hf_processor().audio_feature_extractor
# The feature extractor requires audio at its sampling rate
feature_extractor = self.get_hf_processor().feature_extractor
return InklingMultiModalDataParser(
target_sr=extractor.params.sample_rate,
target_sr=feature_extractor.sampling_rate,
target_channels=1,
expected_hidden_size=self._get_expected_hidden_size(),
)
@@ -163,9 +160,8 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]):
if num_audios:
# Size the dummy at the maximum allowed audio so memory/encoder
# budgeting reflects the largest clip we accept (MAX_AUDIO_TOKENS).
params = self.info.get_hf_processor().audio_feature_extractor.params
hop = int(round(params.audio_token_duration_s * params.sample_rate))
audio_len = MAX_AUDIO_TOKENS * hop
feature_extractor = self.info.get_hf_processor().feature_extractor
audio_len = MAX_AUDIO_TOKENS * feature_extractor.hop_length
mm_data["audio"] = self._get_dummy_audios(
length=audio_len,
num_audios=num_audios,
@@ -203,31 +199,37 @@ class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo])
data: dict[str, Any] = {"input_ids": [prompt_ids]}
if images:
img_feat = processor.process_images(images)
data["pixel_values"] = img_feat["vision_patches_bthwc"]
data["num_patches"] = torch.tensor(
img_feat["num_patches"], dtype=torch.int64
image_processor = processor.image_processor
frac = image_processor.rescale_image_frac or DEFAULT_RESCALE_IMAGE_FRAC
max_long_edge = image_processor.rescale_image_max_upscaled_long_edge
# Resize with PIL: torchvision LANCZOS does not match the reference
images = [_rescale_image(img, frac, max_long_edge) for img in images]
img_feat = image_processor(
images, rescale_image_frac=None, return_tensors="pt"
)
data["pixel_values"] = img_feat["pixel_values"].to(torch.bfloat16)
data["num_patches"] = img_feat["num_patches"].to(torch.int64)
if audios:
aud_feat = processor.process_audios(audios)
per_clip = aud_feat["dmel_bins"]
num_audio_tokens = aud_feat["num_audio_tokens"]
for i, n in enumerate(num_audio_tokens):
if int(n) > MAX_AUDIO_TOKENS:
feature_extractor = processor.feature_extractor
aud_feat = feature_extractor(
audios,
sampling_rate=feature_extractor.sampling_rate,
return_tensors="pt",
)
audio_input_ids = processor._extract_dmel_bins(aud_feat["input_features"])
num_audio_tokens = aud_feat["input_features_mask"].sum(-1)
for i, n in enumerate(num_audio_tokens.tolist()):
if n > MAX_AUDIO_TOKENS:
raise ValueError(
f"Audio clip {i} produces {int(n)} tokens, exceeding the "
f"Audio clip {i} produces {n} tokens, exceeding the "
f"maximum of {MAX_AUDIO_TOKENS} (~10 min at 20 tokens/s). "
"Provide a shorter clip."
)
if per_clip:
input_audio_features = torch.cat(
[torch.as_tensor(c) for c in per_clip], dim=0
)
else:
input_audio_features = torch.empty(0)
data["input_audio_features"] = input_audio_features
data["num_audio_tokens"] = torch.tensor(num_audio_tokens, dtype=torch.int64)
data["input_audio_features"] = torch.cat(
[ids[:n] for ids, n in zip(audio_input_ids, num_audio_tokens)]
)
data["num_audio_tokens"] = num_audio_tokens.to(torch.int64)
processed_data = BatchFeature(data=data, tensor_type=None)
return self._finalize_hf_mm_data(
+29 -9
View File
@@ -4,7 +4,7 @@
Mirrors the reference ``mtp_model.py`` shipped with the checkpoint: each MTP
depth ``i`` owns ``hidden_norm`` / ``embed_norm`` RMSNorms, an ``input_proj``
(``2H -> H``) and a full Inkling transformer block (dense bf16 MLP, with the
(``2H -> H``) and a full Inkling transformer block (dense MLP, with the
same short convolutions as the backbone; its attention is full or sliding-window
per depth, selected by ``mtp_config.local_layer_ids``). When enabled, a shared
``chain_norm`` is applied after every depth; its output is both the logits input
@@ -28,18 +28,23 @@ from torch import nn
from vllm.config import VllmConfig
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.model_loader.mtp_validation import (
is_mtp_completeness_check_enabled,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.interfaces import SupportsMultiModalEmbeddings
from vllm.model_executor.models.utils import maybe_prefix
from vllm.model_executor.models.utils import WeightsMapper, maybe_prefix
from vllm.sequence import IntermediateTensors
from ..configs import InklingModelConfig
from .layernorm import InklingRMSNorm
from .model import InklingDecoderLayer, InklingReplicatedEmbedding
from .model import (
InklingDecoderLayer,
InklingReplicatedEmbedding,
_TmlForCausalLMBase,
)
from .ops.norm import embed_dual_rmsnorm_cat, embed_rmsnorm
# Checkpoint attention projections (wq_du/wk_dv/wv_dv/wr_du) -> fused qkvr.
@@ -71,7 +76,13 @@ def _select_mtp_depth_count(n_predict: int, num_spec: int | None) -> int:
class InklingMTPDepthLayer(nn.Module):
"""One MTP depth: norm both inputs, fuse (2H->H), run a Inkling block."""
def __init__(self, config: InklingModelConfig, prefix: str, is_local: bool) -> None:
def __init__(
self,
config: InklingModelConfig,
prefix: str,
is_local: bool,
quant_config: QuantizationConfig | None = None,
) -> None:
super().__init__()
self.hidden_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.embed_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -80,16 +91,17 @@ class InklingMTPDepthLayer(nn.Module):
config.hidden_size,
bias=False,
return_bias=False,
quant_config=quant_config,
prefix=f"{prefix}.input_proj",
)
# A force-dense-MLP bf16 block; ``is_local`` selects sliding-window vs
# A force-dense-MLP block; ``is_local`` selects sliding-window vs
# full attention (the swa_* head config and sliding_window window) to
# match this depth's checkpoint transformer_block weights.
self.transformer_block = InklingDecoderLayer(
config,
layer_id=0,
is_local=is_local,
quant_config=None,
quant_config=quant_config,
prefix=f"{prefix}.transformer_block",
force_dense_mlp=True,
)
@@ -131,7 +143,10 @@ class InklingMultiTokenPredictor(nn.Module):
self.layers = nn.ModuleDict(
{
str(idx): InklingMTPDepthLayer(
config, f"{prefix}.layers.{idx}", idx in local_ids
config,
f"{prefix}.layers.{idx}",
idx in local_ids,
quant_config=vllm_config.quant_config,
)
for idx in range(self.num_mtp_layers)
}
@@ -244,6 +259,11 @@ class InklingMultiTokenPredictor(nn.Module):
class InklingMTP(nn.Module, SupportsMultiModalEmbeddings):
hf_to_vllm_mapper = _TmlForCausalLMBase.hf_to_vllm_mapper | WeightsMapper(
orig_to_new_prefix={"model.mtp.": "model."},
)
packed_modules_mapping = _TmlForCausalLMBase.packed_modules_mapping
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
assert vllm_config.speculative_config is not None
@@ -418,9 +438,9 @@ def _load_inkling_mtp_weights(
continue
# Dense MLP fused gate/up + down.
if ".mlp.w13_dn.weight" in name:
if ".mlp.w13_dn." in name:
loaded_weight = _load(name.replace(".w13_dn.", ".gate_up_proj."), weight)
elif ".mlp.w2_md.weight" in name:
elif ".mlp.w2_md." in name:
loaded_weight = _load(name.replace(".w2_md.", ".down_proj."), weight)
else:
if name.endswith(".bias") and name not in params:
+60 -41
View File
@@ -95,7 +95,9 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
assert isinstance(spec, MambaSpec)
# Only the fused path exports the checkpoint snapshots
can_checkpoint = (
self.use_prefill_checkpoint and self.use_fused_chunk and self.use_safe_gate
self.use_prefill_checkpoint
and self.kda_prefill_backend == "fused"
and self.use_safe_gate
)
if can_checkpoint and vllm_config.cache_config.mamba_cache_mode == "align":
logger.info_once("Kimi-K3 KDA prefill checkpoint enabled.")
@@ -132,6 +134,14 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
self.conv_size = kda_config["short_conv_kernel_size"]
self.use_full_rank_gate = True
self._fused_qkv_conv_fn = None
if rocm_aiter_ops.is_enabled():
from aiter.ops.triton.gated_delta_net import (
causal_conv1d_split_qkv_triton_tile_fn,
)
self._fused_qkv_conv_fn = causal_conv1d_split_qkv_triton_tile_fn
# Keep f_a before the narrow beta shard, then pad each TP-local row to
# select the aligned BF16 GEMM path. The padding also avoids an Inductor
# correctness issue seen with the row-strided G view.
@@ -242,13 +252,15 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
"The fused KDA chunk kernel requires gfx950 and a build that "
"includes it."
)
self.use_fused_chunk = backend == "fused" or (
backend == "auto" and is_fused_kda_chunk_supported()
)
logger.info_once(
"Kimi-K3 KDA prefill backend: %s",
"fused" if self.use_fused_chunk else "triton",
)
self.kda_prefill_backend = backend
if backend == "auto":
if self.use_safe_gate and rocm_aiter_ops.is_kda_gluon_prefill_available():
self.kda_prefill_backend = "gluon"
elif is_fused_kda_chunk_supported():
self.kda_prefill_backend = "fused"
else:
self.kda_prefill_backend = "triton"
logger.info_once("Kimi-K3 KDA prefill backend: %s", self.kda_prefill_backend)
# TODO: Remove the extra config arg once tested
self.use_prefill_checkpoint = (
additional_config.get("kda_prefill_checkpoint", True)
@@ -576,35 +588,48 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
if mixed_qkv_ns is not None:
assert g1_ns is not None and beta_ns is not None
if m.num_prefills > 0:
q_ns, k_ns, v_ns = mixed_qkv_ns.split(
self.local_projection_size, dim=-1
)
# Packed prefill conv would require copying V solely to make
# it dense for KDA. Separate calls accept the strided inputs
# and produce dense Q/K/V without that extra traffic.
# TODO: Use packed conv once every KDA prefill backend accepts
# row-strided Q/K/V directly.
def _prefill_conv(
x: torch.Tensor,
state: torch.Tensor,
weight: torch.Tensor,
) -> torch.Tensor:
return causal_conv1d_fn(
x.transpose(0, 1),
weight,
if self._fused_qkv_conv_fn is not None:
# One launch (fused qkv) instead of 3
q_ns, k_ns, v_ns = self._fused_qkv_conv_fn(
mixed_qkv_ns.transpose(0, 1),
conv_weights,
None,
activation="silu",
conv_states=state,
has_initial_state=has_initial_state,
conv_state,
non_spec_query_start_loc,
self.local_projection_size,
self.local_projection_size,
cache_indices=non_spec_state_indices_tensor,
query_start_loc=non_spec_query_start_loc,
has_initial_state=has_initial_state,
activation="silu",
block_m=8,
metadata=m,
).transpose(0, 1)
preserve_input_dtype=True,
)
else:
q_ns, k_ns, v_ns = mixed_qkv_ns.split(
self.local_projection_size, dim=-1
)
q_ns = _prefill_conv(q_ns, q_conv_state, q_conv_weight)
k_ns = _prefill_conv(k_ns, k_conv_state, k_conv_weight)
v_ns = _prefill_conv(v_ns, v_conv_state, v_conv_weight)
def _prefill_conv(
x: torch.Tensor,
state: torch.Tensor,
weight: torch.Tensor,
) -> torch.Tensor:
return causal_conv1d_fn(
x.transpose(0, 1),
weight,
None,
activation="silu",
conv_states=state,
has_initial_state=has_initial_state,
cache_indices=non_spec_state_indices_tensor,
query_start_loc=non_spec_query_start_loc,
metadata=m,
).transpose(0, 1)
q_ns = _prefill_conv(q_ns, q_conv_state, q_conv_weight)
k_ns = _prefill_conv(k_ns, k_conv_state, k_conv_weight)
v_ns = _prefill_conv(v_ns, v_conv_state, v_conv_weight)
q_ns, k_ns, v_ns = (
rearrange(x, "n (h d) -> 1 n h d", d=self.head_dim)
for x in (q_ns, k_ns, v_ns)
@@ -694,7 +719,7 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
cu_seqlens=prefill_query_start_loc,
chunk_indices=m.chunk_indices,
chunk_offsets=m.chunk_offsets,
use_fused_chunk=self.use_fused_chunk,
backend=self.kda_prefill_backend,
out=core_attn_out[:, nd_tok:num_actual_tokens] if direct else None,
checkpoint_state=checkpoint_state,
checkpoint_offsets=checkpoint_offsets,
@@ -748,15 +773,9 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
# ---------- merge spec and non-spec outputs ----------
if core_attn_out_spec is not None and core_attn_out_non_spec is not None:
# Mixed batches require indexed placement in the original order.
merged = torch.empty(
(1, num_actual_tokens, *core_attn_out_spec.shape[2:]),
dtype=core_attn_out_spec.dtype,
device=core_attn_out_spec.device,
)
merged = core_attn_out[:, :num_actual_tokens]
merged.index_copy_(1, spec_token_indx, core_attn_out_spec)
merged.index_copy_(1, non_spec_token_indx, core_attn_out_non_spec)
core_attn_out[0, :num_actual_tokens] = merged[0, :num_actual_tokens]
elif core_attn_out_non_spec is not None:
if core_attn_out_non_spec.data_ptr() != core_attn_out.data_ptr():
core_attn_out[0, :num_actual_tokens] = core_attn_out_non_spec[
+29
View File
@@ -11,6 +11,7 @@ from dataclasses import replace
import torch
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.model_executor.layers.mamba.checkpoint import (
MambaPrefillCheckpointMetadata,
@@ -18,12 +19,14 @@ from vllm.model_executor.layers.mamba.checkpoint import (
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import next_power_of_2
from vllm.v1.attention.backend import max_decode_query_len
from vllm.v1.attention.backends.gdn_attn import (
GDNAttentionBackend,
GDNAttentionMetadata,
GDNAttentionMetadataBuilder,
)
from vllm.v1.attention.backends.utils import CommonAttentionMetadata
from vllm.v1.kv_cache_interface import KVCacheSpec
logger = init_logger(__name__)
@@ -92,7 +95,24 @@ def prepare_chunk_metadata_device(
return chunk_indices, chunk_offsets
def _adaptive_verification_enabled(vllm_config: VllmConfig) -> bool:
speculative_config = vllm_config.speculative_config
return speculative_config is not None and bool(
getattr(speculative_config, "enable_adaptive_verification", False)
)
class KimiK3ROCmKDAMetadataBuilder(GDNAttentionMetadataBuilder):
@classmethod
def get_varlen_cudagraph_max_query_len(
cls: type["KimiK3ROCmKDAMetadataBuilder"],
vllm_config: VllmConfig,
kv_cache_spec: KVCacheSpec,
) -> int | None:
if not _adaptive_verification_enabled(vllm_config):
return None
return max_decode_query_len(vllm_config)
def build( # type: ignore[override]
self,
common_prefix_len: int,
@@ -108,6 +128,11 @@ class KimiK3ROCmKDAMetadataBuilder(GDNAttentionMetadataBuilder):
num_decode_draft_tokens_cpu,
fast_build,
)
if (
attn_metadata.uniform_spec_sequence_length is not None
and _adaptive_verification_enabled(self.vllm_config)
):
attn_metadata = replace(attn_metadata, uniform_spec_sequence_length=None)
if attn_metadata.checkpoint is None:
return attn_metadata
# The shared builder keys checkpoint rows on every request; the ROCm
@@ -163,3 +188,7 @@ class KimiK3ROCmKDABackend(GDNAttentionBackend):
@staticmethod
def get_builder_cls() -> type[KimiK3ROCmKDAMetadataBuilder]:
return KimiK3ROCmKDAMetadataBuilder
@classmethod
def supports_device_cpu_query_lens_mismatch(cls) -> bool:
return True
+9 -3
View File
@@ -23,8 +23,8 @@ _OPT_MIN_SIZE = 2048
class KimiK3MultiHeadLatentAttentionWrapper(MultiHeadLatentAttentionWrapper):
"""Kimi-K3 MLA wrapper with eager AITER q/kv RMSNorm fusion and a fused
decode Q-prep path."""
"""Kimi-K3 MLA wrapper with eager AITER q/kv RMSNorm and output gate
fusions and a fused decode Q-prep path."""
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
@@ -303,6 +303,12 @@ class KimiK3MultiHeadLatentAttentionWrapper(MultiHeadLatentAttentionWrapper):
)
if self.g_proj is not None:
attn_out = attn_out * self.g_proj(hidden_states)[0].sigmoid()
gate = self.g_proj(hidden_states)[0]
if rocm_aiter_ops.is_enabled():
from aiter.ops.triton.fusions.fused_sigmoid_mul import fused_sigmoid_mul
attn_out = fused_sigmoid_mul(attn_out, gate)
else:
attn_out = attn_out * gate.sigmoid()
return self.o_proj(attn_out)[0]
+39 -5
View File
@@ -9,6 +9,7 @@ path.
import torch
from vllm._aiter_ops import rocm_aiter_ops
from vllm.logger import init_logger
from vllm.model_executor.layers.mamba.ops.gather_initial_states import (
gather_initial_states,
@@ -24,6 +25,11 @@ from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
logger = init_logger(__name__)
# Consumed by kimi_k3_triton_warmup.py during kernel_warmup().
def get_gluon_chunk_kda_warmup_profiles() -> tuple[tuple[int, int], ...]:
return ((1, 64), (2, 64), (4, 64), (8, 64), (16, 64), (1, 4096), (2, 2048))
def chunk_kda_prefill(
q: torch.Tensor,
k: torch.Tensor,
@@ -40,7 +46,7 @@ def chunk_kda_prefill(
cu_seqlens: torch.Tensor | None = None,
chunk_indices: torch.Tensor | None = None,
chunk_offsets: torch.Tensor | None = None,
use_fused_chunk: bool = False,
backend: str = "triton",
out: torch.Tensor | None = None,
checkpoint_state: torch.Tensor | None = None,
checkpoint_offsets: torch.Tensor | None = None,
@@ -68,9 +74,10 @@ def chunk_kda_prefill(
cu_seqlens: int32 cumulative sequence lengths.
chunk_indices: precomputed chunk indices for ``cu_seqlens``.
chunk_offsets: int32 per-sequence first chunk index.
use_fused_chunk: request the two-kernel ROCm path. It is used only when
every one of its preconditions holds; otherwise the Triton path
runs unchanged.
backend: ``"triton"`` (the vendored chunk path), ``"fused"`` (the
two-kernel ROCm path; used only when every one of its
preconditions holds, otherwise the Triton path runs unchanged) or
``"gluon"`` (AITER's ``chunk_kda``).
out: buffer the result must land in. Honoured by both backends, so the
caller can hand in a slice of its own output and skip a copy.
checkpoint_state: destination for mid-prefill recurrent state
@@ -89,6 +96,11 @@ def chunk_kda_prefill(
The output and, when requested, the final recurrent state.
"""
if backend not in ("triton", "fused", "gluon"):
raise ValueError(
"chunk_kda_prefill backend must be 'triton', 'fused' or 'gluon', "
f"got {backend!r}."
)
if scale is None:
scale = k.shape[-1] ** -0.5
@@ -96,7 +108,7 @@ def chunk_kda_prefill(
# needs the raw projections and a bounded gate rather than the pre-normalized
# tensors the Triton path takes.
fused = (
use_fused_chunk
backend == "fused"
and use_qk_l2norm_in_kernel
and cu_seqlens is not None
and lower_bound is not None
@@ -116,6 +128,28 @@ def chunk_kda_prefill(
if state_indices is None or has_initial_state is None:
raise ValueError("state_cache needs state_indices and has_initial_state")
if backend == "gluon":
return rocm_aiter_ops.kda_chunk_prefill(
q=q,
k=k,
v=v,
g=raw_g,
beta=raw_beta,
A_log=A_log,
dt_bias=g_bias,
lower_bound=lower_bound,
cu_seqlens=cu_seqlens,
chunk_indices=chunk_indices,
chunk_offsets=chunk_offsets,
scale=scale,
out=out,
initial_state=initial_state,
output_final_state=output_final_state,
state_cache=state_cache,
state_indices=state_indices,
has_initial_state=has_initial_state,
)
# Only the fused walk addresses the paged rows directly. The Triton path
# gathers the rows it needs and scatters the results back here, so callers
# pass the cache the same way for either backend.
+3 -3
View File
@@ -104,12 +104,12 @@ def convert_image_mode(
to_mode: str,
background_color: tuple[int, int, int] | list[int] = (255, 255, 255),
) -> Image.Image:
if image.mode == to_mode:
return image
if to_mode == "RGB" and _has_transparency(image):
if image.mode != "RGBA":
image = image.convert("RGBA")
return rgba_to_rgb(image, background_color)
if image.mode == to_mode:
return image
return image.convert(to_mode)
+3 -8
View File
@@ -16,7 +16,7 @@ from vllm.utils.sparse_utils import (
safe_to_dense,
)
from ..image import convert_image_mode, normalize_image, open_image, rgba_to_rgb
from ..image import convert_image_mode, normalize_image, open_image
from .base import MediaDecodedBudget, MediaIO, MediaWithBytes
MAGIC_NUMPY_PREFIX = b"\x93NUMPY" # https://numpy.org/devdocs/reference/generated/numpy.lib.format.html#format-version-1-0
@@ -74,14 +74,9 @@ class ImageMediaIO(MediaIO[Image.Image]):
"""Convert image mode with custom background color."""
if isinstance(image, MediaWithBytes):
image = image.media
if self.image_mode is None or image.mode == self.image_mode:
if self.image_mode is None:
return image
elif image.mode == "RGBA" and self.image_mode == "RGB":
return rgba_to_rgb(image, self.rgba_background_color)
else:
return convert_image_mode(
image, self.image_mode, self.rgba_background_color
)
return convert_image_mode(image, self.image_mode, self.rgba_background_color)
def load_bytes(self, data: bytes) -> MediaWithBytes[Image.Image]:
try:
+5
View File
@@ -769,6 +769,10 @@ class DelegatingParser(Parser):
return False
return self._reasoning_parser.is_reasoning_end(input_ids)
def set_prompt_token_ids(self, prompt_token_ids: Sequence[int]) -> None:
if self._reasoning_parser is not None:
self._reasoning_parser.set_prompt_token_ids(prompt_token_ids)
def _is_reasoning_end_streaming(
self, input_ids: list[int], delta_ids: list[int]
) -> bool:
@@ -858,6 +862,7 @@ class DelegatingParser(Parser):
if not state.prompt_reasoning_checked and prompt_token_ids is not None:
state.prompt_reasoning_checked = True
self.set_prompt_token_ids(prompt_token_ids)
if self._reasoning_parser is None or self.is_reasoning_end(
prompt_token_ids
):
+262
View File
@@ -0,0 +1,262 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Parser for Beam checkpoints."""
from __future__ import annotations
import json
from typing import TYPE_CHECKING, Any, cast
from openai.types.responses import ToolChoiceFunction
from vllm import envs
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionNamedToolChoiceParam,
ChatCompletionRequest,
)
from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger
from vllm.parser.abstract_parser import (
_xgrammar_supports,
structured_outputs_to_format,
)
from vllm.parser.chat_parsing import ResponseParser
from vllm.parser.chat_parsing.response_templates import ResponseTemplate
from vllm.parser.response_template import (
CONTENT_FIELD,
THINKING_FIELD,
TOOL_FIELD,
ResponseTemplateParser,
ResponseTemplateReasoningParser,
ResponseTemplateToolParser,
_as_tool_dicts,
_is_named_tool_choice,
_tool_parsing_enabled,
)
from vllm.parser.response_template_grammar import (
_openers,
answer_rest_structural_tag,
message_start_anchors,
tool_call_structural_tag,
)
from vllm.sampling_params import StructuredOutputsParams
from vllm.tool_parsers.structural_tag_registry import resolve_tool_strictness
if TYPE_CHECKING:
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
logger = init_logger(__name__)
def _named_tool(request: ChatCompletionRequest | ResponsesRequest) -> str:
choice = request.tool_choice
if isinstance(choice, ToolChoiceFunction):
return choice.name
return cast(ChatCompletionNamedToolChoiceParam, choice).function.name
def _starts_message(template: ResponseTemplate, prefix: str) -> bool:
"""Whether output after `prefix` starts the assistant message."""
ends = [match.end() for match in template.start_anchor_re.finditer(prefix)]
return bool(ends) and ends[-1] == len(prefix)
def _open_field(template: ResponseTemplate, prefix: str) -> str | None:
"""The field that `prefix` leaves open with its explicit opener, if any."""
field = None
for event in ResponseParser(template, prefix=prefix).initial_events:
if event["type"] == "region_open" and event["end"] > event["start"]:
field = event["field"]
elif event["type"] in {"region_close", "region_malformed"}:
field = None
return field
class BeamReasoningParser(ResponseTemplateReasoningParser):
"""Selects `BeamParser` as the reasoning parser."""
@classmethod
def unified_parser_cls(cls) -> type[ResponseTemplateParser]:
return BeamParser
@property
def reasoning_start_str(self) -> str | None:
if literal := super().reasoning_start_str:
return literal
thinking = self.response_template.fields.get(THINKING_FIELD)
if thinking is None:
return None
openers = _openers(thinking, message_start_anchors(self.response_template))
if not openers:
return None
# Budget tracking needs one marker for both a prompt continuation
# and a complete assistant message in the checkpoint's pattern.
marker = min(openers, key=len)
return marker if all(opener.endswith(marker) for opener in openers) else None
class BeamToolParser(ResponseTemplateToolParser):
"""Selects `BeamParser` as the tool parser."""
@classmethod
def unified_parser_cls(cls) -> type[ResponseTemplateParser]:
return BeamParser
class BeamParser(ResponseTemplateParser):
"""Parse Beam output with the checkpoint's `response_template` metadata,
and constrain it with a grammar derived from that template."""
reasoning_parser_cls = BeamReasoningParser
tool_parser_cls = BeamToolParser
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
chat_template_kwargs = kwargs.get("chat_template_kwargs") or {}
# From the message start, the grammar opens the message with non-empty
# reasoning, unless the request turns thinking off.
self._requires_reasoning = bool(
chat_template_kwargs.get("enable_thinking", True)
)
def _constrain_request(
self,
request: ChatCompletionRequest | ResponsesRequest,
) -> ChatCompletionRequest | ResponsesRequest:
message_start = _starts_message(self.response_template, self._prefix)
tag = self._template_tag(request, message_start=message_start)
if tag is not None:
grammar = StructuredOutputsParams(structural_tag=json.dumps(tag))
if isinstance(request, ChatCompletionRequest):
request.structured_outputs = grammar
request.response_format = None
else:
request._parser_structured_outputs = grammar
# From the message start, the tag also covers the reasoning.
request._grammar_from_parser = message_start
return request
def _template_tag(
self,
request: ChatCompletionRequest | ResponsesRequest,
*,
message_start: bool,
) -> dict[str, Any] | None:
"""The template grammar for `request`, or `None` to leave its output
to the request's own constraint, if any."""
fields = self.response_template.fields
named = _is_named_tool_choice(request)
tools_enabled = (
self._tool_parser is not None
and bool(request.tools)
and _tool_parsing_enabled(
request, enable_auto_tools=self._enable_auto_tools
)
)
forced = tools_enabled and (request.tool_choice == "required" or named)
# The prompt may still list the tools, so only the grammar keeps the
# model from calling them.
tools_forbidden = (
bool(request.tools)
and request.tool_choice == "none"
and TOOL_FIELD in fields
)
constrained_tools = None
if tools_enabled and (forced or envs.VLLM_ENFORCE_STRICT_TOOL_CALLING):
assert self._tool_parser is not None
strict_level = self.tool_strict_level
if strict_level is None:
strict_level = self._tool_parser.default_tool_strict_level
constrained_tools = resolve_tool_strictness(
request.tools or [], request.tool_choice, strict_level
)
output = request.extract_structured_outputs()
if forced and output is not None:
logger.warning_once(
"structured outputs are ignored because tool_choice forces tool call."
)
output = None
answer = fields.get(CONTENT_FIELD)
if output is not None and (
output.structural_tag is not None
or answer is None
or answer.open_re is None
or not _xgrammar_supports(output)
):
if constrained_tools is not None:
logger.warning_once(
"response_template parsing cannot enforce strict tools "
"alongside this structured output."
)
return None
# Without a tool or structured output constraint, the output still
# follows the template.
always_constrain = envs.VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT and not forced
if (
constrained_tools is None
and output is None
and not tools_forbidden
and not always_constrain
):
return None
# Inside an open answer, only the always-on grammar applies, finishing a
# free answer. Inside open reasoning, the grammar needs the reasoning
# gate to defer it to the reasoning's end.
open_field = _open_field(self.response_template, self._prefix)
reasoning_gated = (
self._reasoning_parser is not None and request.include_reasoning
)
tag = None
if open_field == CONTENT_FIELD:
if always_constrain and output is None:
tag = answer_rest_structural_tag(fields[CONTENT_FIELD])
elif open_field != THINKING_FIELD or reasoning_gated:
functions = []
if tools_enabled:
tools = _as_tool_dicts(constrained_tools or request.tools)
functions = [tool.get("function", tool) for tool in tools]
if named:
name = _named_tool(request)
functions = [f for f in functions if f.get("name") == name]
answer_format = (
None if output is None else structured_outputs_to_format(output)
)
try:
tag = tool_call_structural_tag(
fields.get(TOOL_FIELD),
functions,
parallel_tool_calls=(
not named and request.parallel_tool_calls is not False
),
thinking=fields.get(THINKING_FIELD),
answer=None if forced else answer,
answer_format=(
None if answer_format is None else answer_format.model_dump()
),
message_start=(
message_start_anchors(self.response_template)
if message_start
else None
),
requires_reasoning=self._requires_reasoning,
# Beam drifts from schemas the grammar does not enforce,
# and also writes arguments sorted by name.
sorted_arguments=True,
strict_arguments=True,
)
except ValueError as exc:
raise VLLMValidationError(str(exc), parameter="tools") from exc
if tag is None and forced:
kind = "named" if named else "required"
raise VLLMValidationError(
f"response_template parsing cannot guarantee {kind} tool choice "
"for this request",
parameter="tool_choice",
)
if tag is None and constrained_tools is not None:
logger.warning_once(
"response_template parsing cannot enforce strict tools for this "
"request."
)
return tag
+8
View File
@@ -68,6 +68,11 @@ class ParserEngineReasoningAdapter(ReasoningParser):
def find_reasoning_end(self, token_ids: Sequence[int]) -> ReasoningEnd | None:
return self._parser_engine.find_reasoning_end(token_ids)
def set_prompt_token_ids(self, prompt_token_ids: Sequence[int]) -> None:
self._parser_engine.set_prompt_token_ids(prompt_token_ids)
if self._counting_parser_engine is not None:
self._counting_parser_engine.set_prompt_token_ids(prompt_token_ids)
def adjust_initial_state_from_prompt(self, prompt_token_ids: Sequence[int]) -> None:
self._parser_engine.adjust_initial_state_from_prompt(prompt_token_ids)
@@ -148,6 +153,9 @@ class ParserEngineReasoningAdapter(ReasoningParser):
self._counting_parser_engine = self._parser_engine_cls(
self.model_tokenizer, **self._parser_engine_kwargs
) # type: ignore[call-arg]
self._counting_parser_engine._prompt_closes_reasoning = (
self._parser_engine._prompt_closes_reasoning
)
self._counting_parser_engine._single_pass_parse(
self.model_tokenizer.decode(token_ids), token_ids
)
+18 -2
View File
@@ -147,6 +147,7 @@ class ParserEngine(Parser):
or parser_engine_config.initial_state == ParserState.REASONING
)
self._reasoning_ended: bool = not self._has_reasoning
self._prompt_closes_reasoning: bool = False
self._streaming_initialized: bool = False
self._prompt_streaming_prepared: bool = False
@@ -231,6 +232,18 @@ class ParserEngine(Parser):
self._streaming_initialized = True
self._reset(initial_state=initial_state)
def set_prompt_token_ids(self, prompt_token_ids: Sequence[int]) -> None:
"""Start parsing in content when the prompt already closes reasoning,
as with continue_final_message or a template that switches thinking
off. Only configs with turn boundaries can tell a closed block in
this turn from one in an earlier turn, so others are left as is.
"""
self._prompt_closes_reasoning = (
self.parser_engine_config.initial_state is ParserState.REASONING
and bool(self._turn_boundary_token_ids)
and self.is_reasoning_end(list(prompt_token_ids))
)
def adjust_initial_state_from_prompt(self, prompt_token_ids: Sequence[int]) -> None:
"""See :meth:`ReasoningParser.adjust_initial_state_from_prompt`."""
return
@@ -242,8 +255,11 @@ class ParserEngine(Parser):
return None
def _reset(self, initial_state: ParserState | None = None) -> None:
self._engine.reset(initial_state=initial_state)
self._reasoning_ended = not self._has_reasoning
prompt_closed = initial_state is None and self._prompt_closes_reasoning
self._engine.reset(
initial_state=ParserState.CONTENT if prompt_closed else initial_state
)
self._reasoning_ended = not self._has_reasoning or prompt_closed
self._tool_slots.clear()
self._deferred_content = ""
self._deferred_reasoning = ""
+64 -14
View File
@@ -22,6 +22,7 @@ import regex as re
from vllm.parser.engine.events import EventType
from vllm.parser.engine.parser_engine import ParserEngine
from vllm.parser.engine.parser_engine_config import (
ArgValueExtender,
ParserEngineConfig,
ParserState,
Transition,
@@ -57,37 +58,85 @@ _PARTIAL_PARAM_RE = re.compile(
)
def _minimax_m2_arg_converter(raw_args: str, partial: bool) -> str:
# What ends an open value in ``_PARAM_RE``: a closing tag or the next opener.
_PARAM_VALUE_END_RE = re.compile(r"<\s*/\s*parameter\s*>|<\s*parameter\s+name\s*=")
def _param_name(match: re.Match) -> str:
return (
match.group("dq_name")
or match.group("sq_name")
or match.group("bare_name")
or ""
).strip()
def _minimax_m2_parse_args(
raw_args: str, partial: bool
) -> tuple[dict[str, object], re.Match | None]:
"""Return the arguments and the match of the open trailing value, if any."""
params: dict[str, object] = {}
for match in _PARAM_RE.finditer(raw_args):
name = (
match.group("dq_name")
or match.group("sq_name")
or match.group("bare_name")
or ""
).strip()
name = _param_name(match)
if not name:
continue
# Keep the value verbatim (like the glm47 converter): it is inline
# between `>` and `</parameter>`, so surrounding whitespace is data.
params[name] = match.group("value")
open_match = None
if partial:
remaining = _PARAM_RE.sub("", raw_args)
match = _PARTIAL_PARAM_RE.search(remaining)
if match:
name = (
match.group("dq_name")
or match.group("sq_name")
or match.group("bare_name")
or ""
).strip()
name = _param_name(match)
if name:
# Verbatim, same as the complete-match loop above.
params[name] = match.group("value")
open_match = match
return json.dumps(params, ensure_ascii=False)
return params, open_match
def _minimax_m2_arg_converter(raw_args: str, partial: bool) -> str:
return json.dumps(_minimax_m2_parse_args(raw_args, partial)[0], ensure_ascii=False)
def _value_end_window(text: str) -> str:
"""Return the suffix of *text* that may still grow into a value end."""
start = text.rfind("<")
if start == -1:
return ""
tail = text[start:]
return tail if _PARAM_VALUE_END_RE.fullmatch(tail, partial=True) else ""
def _minimax_m2_arg_value_extender(raw_args: str) -> ArgValueExtender | None:
# The open value runs to the end of raw_args and is the last JSON value,
# so a delta appends its JSON-escaped text unless it closes the value.
params, match = _minimax_m2_parse_args(raw_args, partial=True)
if match is None or next(reversed(params)) != _param_name(match):
return None
# Later text could still re-parse the opener: an earlier unmatched one, or
# a bare name that starts a quote, may close into a different name.
bare_name = match.group("bare_name")
if _PARAM_VALUE_END_RE.search(match.string, 0, match.start()) or (
bare_name is not None and bare_name[0] in "\"'"
):
return None
window = _value_end_window(match.group("value"))
def extend(delta: str) -> str | None:
nonlocal window
text = window + delta
if _PARAM_VALUE_END_RE.search(text):
return None
window = _value_end_window(text)
return json.dumps(delta, ensure_ascii=False)[1:-1]
return extend
@functools.cache
@@ -191,6 +240,7 @@ def minimax_m2_config() -> ParserEngineConfig:
},
},
arg_converter=_minimax_m2_arg_converter,
arg_value_extender=_minimax_m2_arg_value_extender,
stream_arg_deltas=True,
tool_args_json=False,
validate_tool_names=True,
+33 -32
View File
@@ -16,8 +16,6 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
HF_PARSER = "hf"
class ParserManager:
"""Provides a unified Parser by composing reasoning and tool parser adapters."""
@@ -122,8 +120,10 @@ class ParserManager:
tags (``--tool-strict-level``), or ``None`` for the tool
parser's default level.
tokenizer: Tokenizer the composed parser is validated against
at startup (see :meth:`validate_parser_tokenizer`). ``None``
skips the check (e.g. ``skip_tokenizer_init``).
at startup (see :meth:`validate_parser_tokenizer`), including
its `response_template` metadata when a response-template
parser is selected. ``None`` skips the check (e.g.
``skip_tokenizer_init``).
Returns:
A Parser class, or None if neither parser is specified.
@@ -184,39 +184,40 @@ class ParserManager:
HarmonyParser.tool_strict_level = strict_level
return HarmonyParser
if HF_PARSER in (reasoning_parser_name, tool_parser_name):
if {reasoning_parser_name, tool_parser_name} - {
HF_PARSER,
None,
"",
}:
raise TypeError(
"The hf parser cannot be combined with other "
"reasoning or tool call parsers"
)
from vllm.parser.response_template import (
ResponseTemplateParser,
validate_tokenizer_response_template,
)
from vllm.parser.response_template import (
response_template_parser_cls,
validate_tokenizer_response_template,
)
template_parsers = {
response_template_parser_cls(parser_cls)
for parser_cls in (reasoning_parser_cls, tool_parser_cls)
if parser_cls is not None
}
template_parser_cls = next(iter(template_parsers - {None}), None)
if template_parser_cls is not None:
if len(template_parsers) > 1:
raise TypeError(
f"--reasoning-parser {reasoning_parser_name} and "
f"--tool-call-parser {tool_parser_name} cannot be combined"
)
if tokenizer is not None:
validate_tokenizer_response_template(
tokenizer,
reasoning=reasoning_parser_cls is not None,
tools=tool_parser_cls is not None,
reasoning_parser=reasoning_parser_name,
# Without auto tool choice, no tool parser is selected.
tool_parser=tool_parser_name if tool_parser_cls else None,
)
r_cls = reasoning_parser_cls
t_cls = tool_parser_cls
auto_tools = enable_auto_tools
class _ResponseTemplateParser(ResponseTemplateParser):
reasoning_parser_cls = r_cls
tool_parser_cls = t_cls
tool_strict_level = strict_level
_enable_auto_tools = auto_tools
return _ResponseTemplateParser
return type(
template_parser_cls.__name__,
(template_parser_cls,),
{
"reasoning_parser_cls": reasoning_parser_cls,
"tool_parser_cls": tool_parser_cls,
"tool_strict_level": strict_level,
"_enable_auto_tools": enable_auto_tools,
},
)
if reasoning_parser_name == "kimi_k3" or tool_parser_name == "kimi_k3":
from vllm.parser.kimi_k3 import KimiK3Parser
+151 -15
View File
@@ -7,6 +7,7 @@ from __future__ import annotations
import copy
import json
from collections.abc import Callable, Iterable, Sequence
from math import inf
from typing import TYPE_CHECKING, Any, cast
from openai.types.responses import ToolChoiceCustom, ToolChoiceFunction
@@ -28,6 +29,7 @@ from vllm.parser.chat_parsing.response_templates import (
ResponseTemplate,
load_response_template,
)
from vllm.parser.response_template_grammar import message_start_anchors
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
from vllm.renderers.chat_utils import make_tool_call_id
from vllm.tool_parsers.abstract_tool_parser import ToolParser
@@ -135,27 +137,38 @@ def validate_response_template_for_serving(
def validate_tokenizer_response_template(
tokenizer: Any,
*,
reasoning: bool,
tools: bool,
reasoning_parser: str | None,
tool_parser: str | None,
) -> None:
"""Fail at startup if the tokenizer cannot serve the requested parsing."""
"""Fail at startup if the tokenizer cannot serve the selected parsers,
named as given on the command line."""
template = resolve_response_template(tokenizer)
if template is None:
selected = " ".join(
f"{flag} {name}"
for flag, name in (
("--reasoning-parser", reasoning_parser),
("--tool-call-parser", tool_parser),
)
if name
)
raise TypeError(
"The hf parser requires `response_template` metadata "
f"{selected} requires `response_template` metadata "
"in the tokenizer configuration"
)
try:
fields = validate_response_template_for_serving(template).fields
except (TypeError, ValueError) as exc:
raise TypeError(f"Invalid response_template metadata: {exc}") from exc
if reasoning and THINKING_FIELD not in fields:
if reasoning_parser and THINKING_FIELD not in fields:
raise TypeError(
"--reasoning-parser hf requires a response_template with a thinking field"
f"--reasoning-parser {reasoning_parser} requires a response_template "
"with a thinking field"
)
if tools and TOOL_FIELD not in fields:
if tool_parser and TOOL_FIELD not in fields:
raise TypeError(
"--tool-call-parser hf requires a response_template with a tool_calls field"
f"--tool-call-parser {tool_parser} requires a response_template "
"with a tool_calls field"
)
@@ -292,7 +305,7 @@ def _serving_template(
template = resolve_response_template(tokenizer, template)
if template is None:
raise ValueError(
"The hf parser requires `response_template` "
"Parsing with a response template requires `response_template` "
"metadata in the tokenizer configuration"
)
validate_response_template_for_serving(template)
@@ -319,12 +332,17 @@ def _to_tool_calls(value: Any) -> list[tuple[str, str]]:
class ResponseTemplateReasoningParser(ReasoningParser):
"""Reasoning end detection for `--reasoning-parser hf`.
"""Reasoning end detection for a response-template parser.
`ResponseTemplateParser` parses the output; this parser tells it and the
structured-output engine where reasoning ends.
"""
@classmethod
def unified_parser_cls(cls) -> type[ResponseTemplateParser]:
"""The unified parser that selecting this reasoning parser stands for."""
return ResponseTemplateParser
def __init__(
self,
tokenizer: TokenizerLike,
@@ -400,8 +418,26 @@ class ResponseTemplateReasoningParser(ReasoningParser):
class ResponseTemplateToolParser(ToolParser):
"""Registers `--tool-call-parser hf`; `ResponseTemplateParser` extracts the
tool calls."""
"""Selects a response-template parser as the tool parser;
`ResponseTemplateParser` extracts the tool calls."""
@classmethod
def unified_parser_cls(cls) -> type[ResponseTemplateParser]:
"""The unified parser that selecting this tool parser stands for."""
return ResponseTemplateParser
def response_template_parser_cls(
split_parser_cls: type[ReasoningParser] | type[ToolParser],
) -> type[ResponseTemplateParser] | None:
"""The response-template parser that a registered reasoning or tool parser
stands for, or `None` for any other parser."""
if issubclass(
split_parser_cls,
(ResponseTemplateReasoningParser, ResponseTemplateToolParser),
):
return split_parser_cls.unified_parser_cls()
return None
class _DeltaBuilder:
@@ -441,23 +477,50 @@ class _ResponseStream:
self.include_reasoning = include_reasoning
self.parse_tools = parse_tools
self.new_tool_call_id = new_tool_call_id
self.template = template
self.parser = ResponseParser(template, prefix=prefix, tools=tools)
self._prefix_end = len(self.parser.input_text)
self._next_tool_index = 0
self._finalized = False
self._open_region: dict[str, Any] | None = None
self._track(self.parser.initial_events)
def feed(self, text: str, *, finished: bool) -> DeltaMessage | None:
if self._finalized:
return None
delta = _DeltaBuilder()
self._route(self.parser.feed(text), delta)
self._route(self._track(self.parser.feed(text)), delta)
if finished:
self._finalized = True
if len(self.parser.input_text) > self._prefix_end:
_, events = self.parser.finalize()
self._route(events, delta)
events = self.parser.feed(self._closer_rest())
self._route(events + self.parser.finalize()[1], delta)
return delta.build()
def _track(self, events: list[dict[str, Any]]) -> list[dict[str, Any]]:
for event in events:
if event["type"] == "region_open":
self._open_region = event
elif event["type"] in ("region_close", "region_malformed"):
self._open_region = None
return events
def _closer_rest(self) -> str:
"""The rest of a closer the output stopped in after ending its region,
when only the next message's start anchor is missing."""
if self._open_region is None:
return ""
field = self.template.fields[self._open_region["field"]]
text = self.parser.input_text[self._open_region["end"] :]
for closer in field.close_literals or ():
for anchor in message_start_anchors(self.template):
if not anchor or not closer.endswith(anchor) or closer == anchor:
continue
for cut in range(len(closer) - 1, len(closer) - len(anchor) - 1, -1):
if text.endswith(closer[:cut]):
return closer[cut:]
return ""
def _route(self, events: Sequence[dict[str, Any]], delta: _DeltaBuilder) -> None:
for event in events:
field = event.get("field")
@@ -499,6 +562,54 @@ class _ResponseStream:
self._next_tool_index += 1
class _ReasoningTokenCounter:
"""Counts the reasoning tokens of one growing generation, parsing each
token once."""
def __init__(
self, template: ResponseTemplate, prefix: str, tokenizer: TokenizerLike
) -> None:
self.tokenizer = tokenizer
self.parser = ResponseParser(template, prefix=prefix)
self.start = len(self.parser.input_text)
self.token_ids: list[int] = []
self.token_ends: list[int] = []
# Input spans between the delimiters of thinking regions.
self.spans: list[tuple[int, float]] = []
self.open: int | None = None
self._track(self.parser.initial_events)
def continues(self, token_ids: Sequence[int]) -> bool:
return list(token_ids[: len(self.token_ids)]) == self.token_ids
def count(self, token_ids: Sequence[int]) -> int:
new_ids = list(token_ids[len(self.token_ids) :])
texts = [_decode(self.tokenizer, [token_id]) for token_id in new_ids]
end = self.token_ends[-1] if self.token_ends else self.start
for text in texts:
end += len(text)
self.token_ends.append(end)
self.token_ids.extend(new_ids)
self._track(self.parser.feed("".join(texts)))
# Text held back as a possible closer counts until a closer arrives.
spans = self.spans if self.open is None else [*self.spans, (self.open, inf)]
count, start = 0, self.start
for end in self.token_ends:
count += any(begin < end and start < close for begin, close in spans)
start = end
return count
def _track(self, events: Iterable[dict[str, Any]]) -> None:
for event in events:
if event["field"] != THINKING_FIELD or event["type"] == "region_chunk":
continue
if event["type"] == "region_open":
self.open = event["end"]
elif self.open is not None:
self.spans.append((self.open, event["start"]))
self.open = None
class ResponseTemplateParser(Parser):
"""Parse reasoning, content, and tool calls with one response template."""
@@ -526,6 +637,7 @@ class ResponseTemplateParser(Parser):
self._prompt_token_ids: list[int] | None = None
self._prefix = ""
self._stream: _ResponseStream | None = None
self._reasoning_counter: _ReasoningTokenCounter | None = None
def set_prompt_token_ids(self, prompt_token_ids: Sequence[int]) -> None:
prompt_token_ids = list(prompt_token_ids)
@@ -534,6 +646,7 @@ class ResponseTemplateParser(Parser):
self._prompt_token_ids = prompt_token_ids
self._prefix = _decode(self.model_tokenizer, prompt_token_ids)
self._stream = None
self._reasoning_counter = None
def adjust_request(
self,
@@ -547,6 +660,14 @@ class ResponseTemplateParser(Parser):
request.skip_special_tokens = False
if hasattr(request, "spaces_between_special_tokens"):
request.spaces_between_special_tokens = False
return self._constrain_request(request)
def _constrain_request(
self,
request: ChatCompletionRequest | ResponsesRequest,
) -> ChatCompletionRequest | ResponsesRequest:
"""Constrain `request` to the tool guarantees it asks for, or reject
those this parser cannot enforce."""
if (
self._tool_parser is not None
and request.tools
@@ -567,6 +688,21 @@ class ResponseTemplateParser(Parser):
or self._reasoning_parser.is_reasoning_end(input_ids)
)
def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
"""Count the generated tokens whose text the template parses as
reasoning, continuing reasoning left open by the prompt."""
if (
self._reasoning_parser is None
or THINKING_FIELD not in self.response_template.fields
):
return 0
counter = self._reasoning_counter
if counter is None or not counter.continues(token_ids):
counter = self._reasoning_counter = _ReasoningTokenCounter(
self.response_template, self._prefix, self.model_tokenizer
)
return counter.count(token_ids)
def parse(
self,
model_output: str,
File diff suppressed because it is too large Load Diff
+16 -8
View File
@@ -289,10 +289,9 @@ class CpuPlatform(Platform):
cache_config.block_size,
)
cache_config.block_size = 256
elif not cache_config.user_specified_block_size:
cache_config.block_size = 128
if not reference_cpu_mla_enabled and cache_config.block_size % 32 != 0:
elif (
cache_config.user_specified_block_size and cache_config.block_size % 32 != 0
):
logger.warning(
"CPU backend prefers block_size is multiples of 32, "
"otherwise the performance is not optimized."
@@ -532,15 +531,24 @@ class CpuPlatform(Platform):
@classmethod
def update_block_size_for_backend(cls, vllm_config: "VllmConfig") -> None:
model_config = vllm_config.model_config
if model_config is None or not model_config.is_hybrid:
if model_config is None:
return
# reconcile attention and mamba page sizes
backend_classes = cls._find_non_ssm_backends(vllm_config)
if not backend_classes:
return
pre_block_size = vllm_config.cache_config.block_size
# Not in check_and_update_config(), which reruns with every
# VllmConfig.__post_init__ and would undo the hybrid alignment below.
cache_config = vllm_config.cache_config
if not cache_config.user_specified_block_size:
cache_config.block_size = cls._preferred_block_size_for_backends(
backend_classes, 128, vllm_config
)
if not model_config.is_hybrid:
return
# reconcile attention and mamba page sizes
pre_block_size = cache_config.block_size
cls._align_hybrid_block_size(vllm_config, backend_classes[0])
cls._check_aligned_block_size(vllm_config, backend_classes, pre_block_size)
+22
View File
@@ -1467,6 +1467,28 @@ class Platform:
"Unknown vLLM environment variable detected: %s", env
)
@classmethod
def get_kv_cache_config_builder_cls(cls, vllm_config: "VllmConfig") -> str:
"""Resolve the fully-qualified class path of the platform's KV cache
config builder.
.. note::
Experimental, like the ``KVCacheConfigBuilder`` interface it selects.
The builder supplies grouping (``get_kv_cache_groups``) and tensor
placement (``get_kv_cache_config_from_groups``). Vendors may override
this to return a platform-specific builder. The default builder honors a
model implementing
:class:`~vllm.model_executor.models.interfaces.SupportsKVCacheGrouping`;
a builder that overrides ``get_kv_cache_groups`` owns grouping and
bypasses that model hook unless it calls ``super()``.
Returns:
Fully-qualified class path of a ``KVCacheConfigBuilder`` subclass.
"""
return "vllm.v1.core.kv_cache_planning.DefaultKVCacheConfigBuilder"
class UnspecifiedPlatform(Platform):
_enum = PlatformEnum.UNSPECIFIED
+4
View File
@@ -84,6 +84,10 @@ _REASONING_PARSERS_TO_REGISTER = {
"response_template_reasoning_parser",
"ResponseTemplateReasoningParser",
),
"beam": (
"beam_reasoning_parser",
"BeamReasoningParser",
),
"holo2": (
"deepseek_v3_reasoning_parser",
"DeepSeekV3ReasoningWithThinkingParser",
+9
View File
@@ -192,6 +192,15 @@ class ReasoningParser:
"""Adjust request parameters; override in subclasses as needed."""
return request
def set_prompt_token_ids(self, prompt_token_ids: Sequence[int]) -> None:
"""Provide the rendered prompt before the output is parsed.
Parsers that can tell whether the prompt already closes reasoning
(e.g. continue_final_message) use it to classify the output.
Default is a no-op; override in subclasses as needed.
"""
return
def adjust_initial_state_from_prompt(self, prompt_token_ids: Sequence[int]) -> None:
"""Hook called once at the start of streaming with the prompt tokens.
+6
View File
@@ -0,0 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.parser.beam import BeamReasoningParser
__all__ = ["BeamReasoningParser"]
+10 -2
View File
@@ -687,13 +687,17 @@ class BaseRenderer(ABC, Generic[_T]):
prompt = self._tokenize_prompt(prompt, params)
prompt = self._apply_prompt_char_offset(prompt, prompt_char_offset)
prompt = params.apply_post_tokenization(self.tokenizer, prompt) # type: ignore[arg-type]
# Detokenize after truncation so the text matches the tokens sent
# to the engine.
if params.needs_detokenization and "prompt" not in prompt:
if "prompt_token_ids" not in prompt:
raise RuntimeError("Cannot run detokenization on embeddings")
prompt = self._detokenize_prompt(prompt) # type: ignore[arg-type]
return params.apply_post_tokenization(self.tokenizer, prompt) # type: ignore[arg-type]
return prompt
@overload
async def _tokenize_singleton_prompt_async(
@@ -727,13 +731,17 @@ class BaseRenderer(ABC, Generic[_T]):
prompt = await self._tokenize_prompt_async(prompt, params)
prompt = self._apply_prompt_char_offset(prompt, prompt_char_offset)
prompt = params.apply_post_tokenization(self.tokenizer, prompt) # type: ignore[arg-type]
# Detokenize after truncation so the text matches the tokens sent
# to the engine.
if params.needs_detokenization and "prompt" not in prompt:
if "prompt_token_ids" not in prompt:
raise RuntimeError("Cannot run detokenization on embeddings")
prompt = await self._detokenize_prompt_async(prompt) # type: ignore[arg-type]
return params.apply_post_tokenization(self.tokenizer, prompt) # type: ignore[arg-type]
return prompt
def _tokenize_enc_dec_prompt(
self,
+44 -16
View File
@@ -47,14 +47,16 @@ from vllm.entrypoints.serve.utils.tool_calls_utils import (
from vllm.logger import init_logger
from vllm.parser import Parser, ParserManager
from vllm.renderers import BaseRenderer
from vllm.renderers.chat_utils import ChatTemplateContentFormatOption
from vllm.renderers.chat_utils import (
ChatTemplateContentFormatOption,
make_tool_call_id,
)
from vllm.tokenizers import TokenizerLike
from vllm.tokenizers.detokenizer_utils import (
convert_prompt_ids_to_tokens,
detokenize_incrementally,
get_leading_space_marker,
)
from vllm.utils import random_uuid
from vllm.utils.async_utils import make_async
from vllm.v1.engine.detokenizer import uses_fast_detokenizer
@@ -202,13 +204,21 @@ class OnlineDerenderer:
if has_parser and chat_request is not None
else {}
)
# Like the coupled chat endpoint, hide per-token metadata (logprobs,
# token ids) when a parser is configured and reasoning is hidden:
# either would otherwise leak the reasoning text.
hide_metadata = (
has_parser
and chat_request is not None
and not chat_request.include_reasoning
)
return_token_ids = (
chat_request is not None
and chat_request.return_token_ids
and (not has_parser or chat_request.include_reasoning)
and not hide_metadata
)
top_limit = _chat_top_logprobs_limit(chat_request)
for choice in generate_response.choices:
if not choice.token_ids:
raise ValueError(f"choice {choice.index} has empty or null token_ids")
@@ -219,7 +229,7 @@ class OnlineDerenderer:
tokenizer,
top_limit=top_limit,
)
if choice.logprobs is not None
if choice.logprobs is not None and not hide_metadata
else None
)
@@ -326,7 +336,10 @@ class OnlineDerenderer:
reasoning = None
tc_items = (
[ToolCall(id=random_uuid(), function=tc) for tc in tool_calls]
[
ToolCall(id=tc.id or make_tool_call_id(), function=tc)
for tc in tool_calls
]
if tool_calls
else []
)
@@ -508,13 +521,6 @@ class OnlineDerenderer:
spaces_between_special_tokens=spaces_between,
)
# NOTE: parser-configured servers dispatch to
# _derender_chat_stream_parsed above and never reach this plain
# path. That parsed path does not resolve logprobs yet; when it
# does, mirror the generate chat streaming path, which suppresses
# logprobs entirely when a parser is configured and reasoning is
# hidden, because decoded logprob token text would leak hidden
# reasoning.
resolved_logprobs = None
if choice.logprobs is not None:
resolved_logprobs = _resolve_logprobs(
@@ -595,8 +601,17 @@ class OnlineDerenderer:
Text comes from a fresh incremental detokenizer with special tokens
preserved (``skip_special_tokens=False``), seeded from the prompt
tail on every call since replay starts from scratch.
Logprobs need no replay: a generate chunk's ``logprobs`` cover exactly
that chunk's tokens, so they are resolved once and attached to
whatever delta the parser emits for the chunk, as the coupled chat
endpoint does. Only the byte-fallback context crosses chunks, via
``state.logprob_context_token_ids``. With ``include_reasoning=False``
logprobs are dropped on every chunk, again like the coupled endpoint.
"""
tokenizer = self.renderer.get_tokenizer()
hide_metadata = not chat_request.include_reasoning
top_limit = _chat_top_logprobs_limit(chat_request)
parser = parser_cls(
tokenizer,
@@ -651,10 +666,9 @@ class OnlineDerenderer:
role_sent = state.role_sent
tools_streamed = state.tools_streamed
last_tool_call_ids = list(state.last_tool_call_ids)
logprob_context = list(state.logprob_context_token_ids)
return_token_ids = (
chat_request.return_token_ids and chat_request.include_reasoning
)
return_token_ids = chat_request.return_token_ids and not hide_metadata
# At most one choice: the caller (derender_chat_stream) already
# rejects >1 before dispatching here. role_sent/tools_streamed/
@@ -707,6 +721,18 @@ class OnlineDerenderer:
output_token_ids.extend(delta_tids)
output_chunk_lens.append(len(delta_tids))
resolved_logprobs = None
if choice.logprobs is not None and not hide_metadata:
resolved_logprobs = _resolve_logprobs(
choice.logprobs,
tokenizer,
initial_context_token_ids=state.logprob_context_token_ids,
top_limit=top_limit,
)
logprob_context = _logprob_context_tail(
state.logprob_context_token_ids, delta_tids
)
if delta_message is None:
delta_message = DeltaMessage()
@@ -735,6 +761,7 @@ class OnlineDerenderer:
stream_choice = ChatCompletionResponseStreamChoice(
index=choice.index,
delta=delta_message,
logprobs=resolved_logprobs,
finish_reason=finish_reason,
stop_reason=choice.stop_reason,
token_ids=choice.token_ids if return_token_ids else None,
@@ -750,6 +777,7 @@ class OnlineDerenderer:
"role_sent": role_sent,
"tools_streamed": tools_streamed,
"last_tool_call_ids": last_tool_call_ids,
"logprob_context_token_ids": logprob_context,
}
)
+8 -3
View File
@@ -60,6 +60,7 @@ from vllm.renderers.chat_utils import (
has_non_text_content,
)
from vllm.renderers.inputs.preprocess import (
extract_prompt_components,
parse_model_prompt,
prompt_to_seq,
)
@@ -844,13 +845,17 @@ class OnlineRenderer:
f"but got {type(request).__name__}"
)
raise NotImplementedError(msg)
request = parser(
request_parser = parser(
tokenizer,
request.tools,
model_config=self.model_config,
chat_template_kwargs=chat_params.chat_template_kwargs,
).adjust_request(
request=request,
)
prompt_token_ids = extract_prompt_components(
self.model_config, engine_input
).token_ids
if prompt_token_ids is not None:
request_parser.set_prompt_token_ids(prompt_token_ids)
request = request_parser.adjust_request(request=request)
return conversation, [engine_input]
+4
View File
@@ -90,6 +90,10 @@ _TOOL_PARSERS_TO_REGISTER = {
"response_template_tool_parser",
"ResponseTemplateToolParser",
),
"beam": (
"beam_tool_parser",
"BeamToolParser",
),
"poolside_v1": (
"poolside_v1_tool_parser",
"PoolsideV1ToolParser",
+6
View File
@@ -0,0 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.parser.beam import BeamToolParser
__all__ = ["BeamToolParser"]
@@ -39,9 +39,6 @@ __all__ = [
"OvisProcessor",
"Ovis2_5Processor",
"Step3VLProcessor",
"InklingProcessor",
"InklingImageProcessor",
"InklingAudioFeatureExtractor",
]
_CLASS_TO_MODULE: dict[str, str] = {
@@ -74,9 +71,6 @@ _CLASS_TO_MODULE: dict[str, str] = {
"OvisProcessor": "vllm.transformers_utils.processors.ovis",
"Ovis2_5Processor": "vllm.transformers_utils.processors.ovis2_5",
"Step3VLProcessor": "vllm.transformers_utils.processors.step3_vl",
"InklingProcessor": "vllm.transformers_utils.processors.inkling",
"InklingImageProcessor": "vllm.transformers_utils.processors.inkling",
"InklingAudioFeatureExtractor": "vllm.transformers_utils.processors.inkling",
}
@@ -1,504 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Vendored HuggingFace-convention processors for the Inkling Titan model.
Implements the image processor (numba patchifier), the audio feature extractor
(STFT/dMel path), the composite processor, and the MM token-id constants.
Besides raw bytes / file paths, the extractors also accept the dummy inputs
vLLM generates during profiling (PIL images / numpy audio arrays).
"""
from __future__ import annotations
import io
import math
from collections.abc import Sequence
from dataclasses import dataclass
import numpy as np
import torch
import torch.nn.functional as F
from numba import njit
from transformers.feature_extraction_utils import (
BatchFeature,
FeatureExtractionMixin,
)
from transformers.image_processing_utils import BaseImageProcessor
from transformers.image_utils import ImageInput
# ---------------------------------------------------------------------------
# MM token-id constants
# ---------------------------------------------------------------------------
# Block-start marker token ids. These are real tokens the model was trained on
# (``<|content_image|>`` / ``<|content_audio_input|>``) that mark the start of an
# image/audio embedding block; they are kept verbatim in ``input_ids``.
IMAGE_MARKER_ID = 200005 # <|content_image|>
AUDIO_MARKER_ID = 200020 # <|content_audio_input|>
# Per-patch / per-frame placeholder ids marking where tower embeddings are
# scattered in. These are unused slots in the padded vocabulary and the
# corresponding positions are always overwritten by tower embeddings.
IMAGE_TOKEN_ID = 200054 # <|unused_200054|>
AUDIO_TOKEN_ID = 200053 # <|unused_200053|>
# ---------------------------------------------------------------------------
# Image processing
# ---------------------------------------------------------------------------
IMAGE_MEAN = np.array([0.48145466, 0.4578275, 0.40821073], dtype=np.float32)
IMAGE_STD = np.array([0.26862954, 0.2613026, 0.2757771], dtype=np.float32)
PAD_RAW_VALUE = np.float32(-1.0 / 255.0)
PAD_NORM = (np.full((3,), PAD_RAW_VALUE, dtype=np.float32) - IMAGE_MEAN) / IMAGE_STD
def _validate_image_rescale(
rescale_image_frac: float | None,
rescale_image_max_upscaled_long_edge: int | None,
) -> None:
if rescale_image_frac is not None and (
not math.isfinite(rescale_image_frac) or rescale_image_frac <= 0
):
raise ValueError(
"rescale_image_frac must be positive and finite or None, "
f"got {rescale_image_frac}"
)
if rescale_image_max_upscaled_long_edge is None:
return
if rescale_image_max_upscaled_long_edge <= 0:
raise ValueError(
"rescale_image_max_upscaled_long_edge must be positive or None, "
f"got {rescale_image_max_upscaled_long_edge}"
)
if rescale_image_frac is None or rescale_image_frac <= 1.0:
raise ValueError(
"rescale_image_max_upscaled_long_edge requires rescale_image_frac > 1, "
f"got {rescale_image_frac}"
)
def _scaled_image_dimensions(
width: int,
height: int,
rescale_image_frac: float | None,
rescale_image_max_upscaled_long_edge: int | None,
) -> tuple[int, int]:
"""Return the long-edge-scaled ``(width, height)``."""
if rescale_image_frac is None:
return width, height
long_edge = max(width, height)
if long_edge == 0:
return width, height
target_long_edge = float(long_edge) * rescale_image_frac
if rescale_image_max_upscaled_long_edge is not None:
effective_cap = max(rescale_image_max_upscaled_long_edge, long_edge)
target_long_edge = min(target_long_edge, float(effective_cap))
ratio = target_long_edge / float(long_edge)
if ratio == 1.0:
return width, height
def scale(value: int) -> int:
return max(1, math.floor(float(value) * ratio + 0.5))
return scale(width), scale(height)
def _load_image_bytes(image) -> bytes:
"""Encode a PIL image as raw PNG bytes for preprocessing.
The HF processor is always handed ``PIL.Image`` instances by vLLM, so no
other input types need to be supported here.
"""
if image.mode != "RGB":
image = image.convert("RGB")
buf = io.BytesIO()
image.save(buf, format="PNG")
return buf.getvalue()
@njit(cache=True)
def _fill_patches_numba(
arr: np.ndarray,
patch_size: int,
patches: np.ndarray,
mean: np.ndarray,
std: np.ndarray,
pad_norm: np.ndarray,
) -> None:
h = arr.shape[0]
w = arr.shape[1]
nph = (h + patch_size - 1) // patch_size
npw = w // patch_size + 1
inv255 = np.float32(1.0 / 255.0)
for k in range(nph * npw):
i = k // npw
j = k - i * npw
y_base = i * patch_size
x_base = j * patch_size
for y in range(patch_size):
iy = y_base + y
for x in range(patch_size):
ix = x_base + x
if iy < h and ix < w:
for c in range(3):
raw = np.float32(arr[iy, ix, c]) * inv255
patches[k, y, x, c] = (raw - mean[c]) / std[c]
else:
for c in range(3):
patches[k, y, x, c] = pad_norm[c]
def _encode_image_bytes(
image_bytes: bytes,
*,
patch_size: int,
rescale_image_frac: float | None,
rescale_image_max_upscaled_long_edge: int | None,
) -> torch.Tensor:
if patch_size <= 0:
raise ValueError("patch_size must be greater than zero")
_validate_image_rescale(
rescale_image_frac,
rescale_image_max_upscaled_long_edge,
)
from PIL import Image
image = Image.open(io.BytesIO(image_bytes)).convert("RGB")
scaled_size = _scaled_image_dimensions(
image.width,
image.height,
rescale_image_frac=rescale_image_frac,
rescale_image_max_upscaled_long_edge=rescale_image_max_upscaled_long_edge,
)
if scaled_size != image.size:
image = image.resize(scaled_size, resample=Image.Resampling.LANCZOS)
arr = np.array(image, dtype=np.uint8, copy=True)
height, width, _ = arr.shape
nph = (height + patch_size - 1) // patch_size
npw = width // patch_size + 1
num_patches = nph * npw
patches = np.empty((num_patches, patch_size, patch_size, 3), dtype=np.float32)
_fill_patches_numba(arr, patch_size, patches, IMAGE_MEAN, IMAGE_STD, PAD_NORM)
return (
torch.from_numpy(patches)
.to(torch.bfloat16)
.view(num_patches, 1, patch_size, patch_size, 3)
.expand(num_patches, 2, patch_size, patch_size, 3)
)
class InklingImageProcessor(BaseImageProcessor):
r"""Turn raw images into ``vision_patches_bthwc`` for Inkling hMLP.
``rescale_image_frac`` scales the long edge while preserving aspect ratio.
``rescale_image_max_upscaled_long_edge`` optionally caps only upscaling and
therefore requires a scale factor greater than one. The defaults, ``2.0`` and
``2048``, grow images toward a 2048-pixel long edge by at most 2x, while leaving
images already at or above 2048 unchanged.
"""
model_input_names = ["vision_patches_bthwc"]
def __init__(
self,
patch_size: int = 40,
rescale_image_frac: float | None = 2.0,
rescale_image_max_upscaled_long_edge: int | None = 2048,
**kwargs,
):
if patch_size <= 0:
raise ValueError("patch_size must be greater than zero")
_validate_image_rescale(
rescale_image_frac,
rescale_image_max_upscaled_long_edge,
)
super().__init__(**kwargs)
self.patch_size = patch_size
self.rescale_image_frac = rescale_image_frac
self.rescale_image_max_upscaled_long_edge = rescale_image_max_upscaled_long_edge
def _encode_one(self, image) -> torch.Tensor:
return _encode_image_bytes(
_load_image_bytes(image),
patch_size=self.patch_size,
rescale_image_frac=self.rescale_image_frac,
rescale_image_max_upscaled_long_edge=self.rescale_image_max_upscaled_long_edge,
)
def preprocess(
self,
images: ImageInput | list,
return_tensors: str | None = "pt",
**kwargs,
) -> BatchFeature:
del return_tensors, kwargs
if not isinstance(images, (list, tuple)):
images = [images]
per_image_patches: list[torch.Tensor] = []
num_patches: list[int] = []
num_tokens: list[int] = []
for img in images:
vp = self._encode_one(img)
n_patches = int(vp.shape[0])
per_image_patches.append(vp)
num_patches.append(n_patches)
num_tokens.append(n_patches)
if len(per_image_patches) == 1:
vision_patches_bthwc = per_image_patches[0]
elif per_image_patches:
vision_patches_bthwc = torch.cat(per_image_patches, dim=0)
else:
vision_patches_bthwc = torch.empty(0)
data = {
"vision_patches_bthwc": vision_patches_bthwc,
"num_patches": num_patches,
"num_tokens": num_tokens,
}
return BatchFeature(data=data, tensor_type=None)
# ---------------------------------------------------------------------------
# Audio feature extraction
# ---------------------------------------------------------------------------
@dataclass
class InklingAudioEncoderParams:
"""Audio preprocessing parameters used to convert raw audio into dMel bins."""
sample_rate: int = 16_000
window_size_multiplier: float = 2.0
n_fft: int | None = None
n_mels: int = 80
num_dmel_bins: int = 16
dmel_min_value: float = -7.0
dmel_max_value: float = 2.0
audio_token_duration_s: float = 0.05
def _to_exact_int(value: float, name: str, tolerance: float = 1e-6) -> int:
rounded = round(value)
if abs(value - rounded) > tolerance:
raise ValueError(f"{name} must resolve to an integer sample count, got {value}")
return int(rounded)
def _hz_to_mel(frequencies: np.ndarray) -> np.ndarray:
"""Slaney mel scale, matching the librosa/torchaudio convention."""
frequencies = np.asarray(frequencies, dtype=np.float64)
f_sp = 200.0 / 3.0
min_log_hz = 1000.0
min_log_mel = min_log_hz / f_sp
logstep = np.log(6.4) / 27.0
linear = frequencies / f_sp
log = (
min_log_mel + np.log(np.maximum(frequencies, min_log_hz) / min_log_hz) / logstep
)
return np.where(frequencies >= min_log_hz, log, linear)
def _mel_to_hz(mels: np.ndarray) -> np.ndarray:
mels = np.asarray(mels, dtype=np.float64)
f_sp = 200.0 / 3.0
min_log_hz = 1000.0
min_log_mel = min_log_hz / f_sp
logstep = np.log(6.4) / 27.0
linear = mels * f_sp
log = min_log_hz * np.exp(logstep * (mels - min_log_mel))
return np.where(mels >= min_log_mel, log, linear)
_MEL_BASIS_CACHE: dict[tuple[int, int, int], torch.Tensor] = {}
def _mel_basis(sample_rate: int, n_fft: int, n_mels: int) -> torch.Tensor:
key = (sample_rate, n_fft, n_mels)
cached = _MEL_BASIS_CACHE.get(key)
if cached is not None:
return cached
fft_bins = n_fft // 2 + 1
fft_freqs = np.arange(fft_bins, dtype=np.float64) * sample_rate / n_fft
mel_edges = _mel_to_hz(
np.linspace(
_hz_to_mel(np.array([0.0]))[0],
_hz_to_mel(np.array([sample_rate / 2.0]))[0],
n_mels + 2,
dtype=np.float64,
)
)
mel_widths = np.diff(mel_edges)
lower = (fft_freqs[None, :] - mel_edges[:-2, None]) / mel_widths[:-1, None]
upper = (mel_edges[2:, None] - fft_freqs[None, :]) / mel_widths[1:, None]
weights = np.maximum(0.0, np.minimum(lower, upper))
# Slaney area normalization.
weights *= (2.0 / (mel_edges[2:] - mel_edges[:-2]))[:, None]
basis = torch.from_numpy(weights.astype(np.float32, copy=False)).contiguous()
_MEL_BASIS_CACHE[key] = basis
return basis
def _dmel_bins(audio: torch.Tensor, params: InklingAudioEncoderParams) -> torch.Tensor:
hop_length = _to_exact_int(
params.audio_token_duration_s * params.sample_rate,
"audio_token_duration_s * sample_rate",
)
window_size = _to_exact_int(
params.audio_token_duration_s
* params.window_size_multiplier
* params.sample_rate,
"audio_token_duration_s * window_size_multiplier * sample_rate",
)
n_fft = params.n_fft or window_size
if hop_length <= 0 or window_size <= 0 or n_fft <= 0:
raise ValueError("audio hop length, window size, and n_fft must be positive")
if audio.numel() == 0:
return torch.empty((0, params.n_mels), dtype=torch.int32)
right_pad = math.ceil(audio.numel() / hop_length) * hop_length - audio.numel()
left_pad = max(n_fft - hop_length, 0)
audio = F.pad(audio, (left_pad, right_pad))
window = torch.hann_window(window_size, periodic=True, dtype=torch.float32)
spec = torch.stft(
audio.unsqueeze(0),
n_fft=n_fft,
hop_length=hop_length,
win_length=window_size,
window=window,
center=False,
normalized=False,
onesided=True,
return_complex=True,
)
spec_ri = torch.view_as_real(spec)
magnitude = (
(spec_ri[..., 0].square() + spec_ri[..., 1].square())
.clamp_min(1e-10)
.sqrt()
.squeeze(0)
)
mel = (
_mel_basis(params.sample_rate, n_fft, params.n_mels)
.matmul(magnitude)
.clamp_min(1e-10)
.log10()
)
mel = mel.to(torch.float64).clamp(
min=params.dmel_min_value, max=params.dmel_max_value
)
bin_centers = torch.linspace(
params.dmel_min_value,
params.dmel_max_value,
params.num_dmel_bins,
dtype=torch.float64,
)
dmel_bins = (mel.unsqueeze(-1) - bin_centers).abs().argmin(dim=-1)
return dmel_bins.to(torch.int32).T.contiguous()
class InklingAudioFeatureExtractor(FeatureExtractionMixin):
"""Convert raw audio into Inkling dMel bins in the HF feature-extractor API."""
model_input_names = ["dmel_bins"]
def __init__(self, params: dict | None = None, **kwargs):
super().__init__(**kwargs)
merged = InklingAudioEncoderParams()
if params:
for k, v in params.items():
if hasattr(merged, k):
setattr(merged, k, v)
# also accept flat kwargs (HF config style)
for k in list(kwargs.keys()):
if hasattr(merged, k):
setattr(merged, k, kwargs[k])
self.params = merged
def _decode_one(self, audio) -> torch.Tensor:
# vLLM hands the feature extractor numpy arrays (the dummy-input builder
# during profiling, and MultiModalDataParser after resampling to the
# target sample rate), so no other input types need to be supported.
return torch.from_numpy(
np.ascontiguousarray(audio.astype(np.float32, copy=False))
).flatten()
def _encode_one(self, audio) -> torch.Tensor:
return _dmel_bins(self._decode_one(audio), self.params)
def __call__(
self,
audios: Sequence | None,
return_tensors: str | None = None,
**kwargs,
) -> BatchFeature:
del return_tensors, kwargs
if audios is None:
audios = []
if not isinstance(audios, (list, tuple)):
audios = [audios]
dmel_bins = [self._encode_one(a) for a in audios]
data = {
# per-clip feature: dmel bins as float32 [T, n_mels]
"dmel_bins": [bins.to(torch.float32) for bins in dmel_bins],
"num_audio_tokens": [int(bins.shape[0]) for bins in dmel_bins],
}
# return_tensors intentionally ignored: per-clip features have ragged T.
return BatchFeature(data=data, tensor_type=None)
# ---------------------------------------------------------------------------
# Composite processor
# ---------------------------------------------------------------------------
class InklingProcessor:
"""Bundle Inkling image + audio preprocessing with the MM token ids."""
def __init__(
self,
image_processor: InklingImageProcessor | None = None,
audio_feature_extractor: InklingAudioFeatureExtractor | None = None,
tokenizer=None,
):
self.image_processor = image_processor or InklingImageProcessor()
self.audio_feature_extractor = (
audio_feature_extractor or InklingAudioFeatureExtractor()
)
self.tokenizer = tokenizer
def process_images(self, images: list):
"""Raw images -> BatchFeature(vision_patches_bthwc, num_patches, num_tokens)."""
return self.image_processor.preprocess(images, return_tensors="pt")
def process_audios(self, audios: list):
"""Raw audios -> BatchFeature(dmel_bins, num_audio_tokens)."""
return self.audio_feature_extractor(audios)
__all__ = [
"InklingImageProcessor",
"InklingAudioFeatureExtractor",
"InklingAudioEncoderParams",
"InklingProcessor",
"IMAGE_MARKER_ID",
"AUDIO_MARKER_ID",
"IMAGE_TOKEN_ID",
"AUDIO_TOKEN_ID",
]
+2 -6
View File
@@ -16,10 +16,8 @@ def _compute_slot_mapping_kernel_impl(
positions: torch.Tensor, # [num_tokens], int64
block_table: torch.Tensor, # [max_num_reqs, max_num_blocks_per_req], int32
block_table_stride: int, # max_num_blocks_per_req
block_size: int,
slot_mapping: torch.Tensor, # [max_num_tokens], int64
KV_CACHE_BLOCK_SIZE: int | None = None,
BLOCKS_PER_KV_BLOCK: int = 1,
KV_CACHE_BLOCK_SIZE: int,
TOTAL_CP_WORLD_SIZE: int = 1,
TOTAL_CP_RANK: int = 0,
CP_KV_CACHE_INTERLEAVE_SIZE: int = 1,
@@ -27,14 +25,12 @@ def _compute_slot_mapping_kernel_impl(
BLOCK_SIZE: int = 1024,
) -> None:
assert TOTAL_CP_WORLD_SIZE == 1, "Context Parallelism is not supported on CPU."
if BLOCKS_PER_KV_BLOCK != 1:
assert block_size * BLOCKS_PER_KV_BLOCK == KV_CACHE_BLOCK_SIZE
torch.ops._C.compute_slot_mapping_kernel_impl(
query_start_loc,
positions,
block_table,
slot_mapping,
block_size,
KV_CACHE_BLOCK_SIZE,
)
+14
View File
@@ -358,6 +358,19 @@ autotune = _lazy_import_wrapper(
)
@functools.cache
def has_flashinfer_autotune_v2() -> bool:
"""Return ``True`` if FlashInfer ships the managed-cache autotuner
(``flashinfer.autotune_v2``, flashinfer-ai/flashinfer#3861)."""
if not has_flashinfer():
return False
mod = _get_submodule("flashinfer")
return mod is not None and all(
callable(getattr(mod, name, None))
for name in ("autotune_v2", "autotune_v2_reload")
)
@functools.cache
def has_flashinfer_comm() -> bool:
"""Return `True` if FlashInfer comm module is available."""
@@ -1269,6 +1282,7 @@ __all__ = [
"flashinfer_fused_kda_decode",
"autotune",
"has_flashinfer_moe",
"has_flashinfer_autotune_v2",
"has_flashinfer_comm",
"has_flashinfer_nvlink_two_sided",
"has_flashinfer_nvlink_one_sided",
+19 -2
View File
@@ -6,7 +6,7 @@ from typing import Any
import torch
from vllm.config import VllmConfig
from vllm.config import VllmConfig, get_current_vllm_config_or_none
from vllm.model_executor.layers.mamba.checkpoint import (
MambaPrefillCheckpointBuilder,
MambaPrefillCheckpointMetadata,
@@ -15,12 +15,13 @@ from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backend import (
AttentionBackend,
CommonAttentionMetadata,
max_decode_query_len,
)
from vllm.v1.attention.backends.mamba_attn import (
BaseMambaAttentionMetadata,
BaseMambaAttentionMetadataBuilder,
)
from vllm.v1.kv_cache_interface import MambaSpec
from vllm.v1.kv_cache_interface import KVCacheSpec, MambaSpec
def compute_varlen_chunk_metadata(
@@ -104,6 +105,14 @@ class Mamba2AttentionBackend(AttentionBackend):
def is_ssm(cls) -> bool:
return True
@classmethod
def supports_device_cpu_query_lens_mismatch(cls) -> bool:
# Decode kernels take device offsets; ReplaySSM plans off CPU counts.
vllm_config = get_current_vllm_config_or_none()
if vllm_config is None:
return True
return not vllm_config.cache_config.use_replayssm
@dataclass
class Mamba2AttentionMetadata(BaseMambaAttentionMetadata):
@@ -124,6 +133,14 @@ class Mamba2AttentionMetadataBuilder(
):
metadata_cls = Mamba2AttentionMetadata
@classmethod
def get_varlen_cudagraph_max_query_len(
cls,
vllm_config: VllmConfig,
kv_cache_spec: KVCacheSpec,
) -> int | None:
return max_decode_query_len(vllm_config)
def __init__(
self,
kv_cache_spec: MambaSpec,
@@ -22,6 +22,17 @@ if TYPE_CHECKING:
from vllm.platforms.interface import DeviceCapability
def is_cute_dsl_mla_prefill_supported(
qk_head_dim: int, v_head_dim: int, query_dtype: torch.dtype
) -> bool:
# FlashInfer has no public compatibility API; replace these checks when available.
return (
query_dtype in (torch.float16, torch.bfloat16, torch.float8_e4m3fn)
and (qk_head_dim, v_head_dim) in ((128, 128), (192, 128))
and (qk_head_dim != 192 or query_dtype == torch.float8_e4m3fn)
)
class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
"""TRT-LLM Ragged backend for MLA prefill."""
@@ -79,6 +90,7 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
v_head_dim=v_head_dim,
vllm_config=vllm_config,
)
self._context_backend = vllm_config.attention_config.trtllm_ragged_backend
self._workspace_spec = (
(envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE,),
torch.uint8,
@@ -198,6 +210,16 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
"TRTLLM ragged context prefill contains an empty query or KV row"
)
if self._context_backend == "cute-dsl":
if not is_cute_dsl_mla_prefill_supported(q.shape[-1], v.shape[-1], q.dtype):
raise ValueError(
"CuTe DSL context prefill does not support "
f"Q/K head dimension {q.shape[-1]}, "
f"V head dimension {v.shape[-1]}, and query dtype {q.dtype}."
)
# Generic MLA supplies V as a projection slice; CuTe requires packed V.
v = v.contiguous()
if out is None:
out = torch.empty(
q.shape[0],
@@ -225,6 +247,7 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
cum_seq_lens_kv=chunk.cu_seq_lens,
enable_pdl=False,
is_causal=False,
backend=self._context_backend,
return_lse=True,
out=out,
skip_all_rows_active_check=True,
+145 -10
View File
@@ -35,6 +35,7 @@ from vllm.v1.attention.backend import (
AttentionLayer,
CommonAttentionMetadata,
MultipleOf,
max_decode_query_len,
)
from vllm.v1.attention.backends.mla.triton_mla import _compute_num_kv_splits
from vllm.v1.attention.backends.utils import (
@@ -49,7 +50,11 @@ from vllm.v1.attention.ops.rocm_aiter_mla_prefill import (
gather_compressed_context,
)
from vllm.v1.attention.ops.triton_decode_attention import decode_attention_fwd
from vllm.v1.kv_cache_interface import AttentionSpec, is_quantized_kv_cache
from vllm.v1.kv_cache_interface import (
AttentionSpec,
KVCacheSpec,
is_quantized_kv_cache,
)
from vllm.v1.worker.workspace import (
current_workspace_manager,
is_workspace_manager_initialized,
@@ -457,6 +462,16 @@ def _aiter_mla_small_head_mode() -> str:
return mode
def _adaptive_varlen_verify_enabled(vllm_config: VllmConfig) -> bool:
speculative_config = vllm_config.speculative_config
return (
speculative_config is not None
and bool(getattr(speculative_config, "enable_adaptive_verification", False))
and vllm_config.parallel_config.decode_context_parallel_size == 1
and max_decode_query_len(vllm_config) > 1
)
class AiterMLABackend(MLACommonBackend):
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16]
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
@@ -620,6 +635,8 @@ class AiterMLADecodeMetadata(MLACommonDecodeMetadata):
use_gluon_verify: bool = False
# Whether persistent MLA metadata was computed
has_persistent_metadata: bool = False
varlen_q_rows: torch.Tensor | None = None
varlen_o_rows: torch.Tensor | None = None
@dataclass
@@ -720,6 +737,21 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
return uniform_qo_len
@classmethod
def get_varlen_cudagraph_max_query_len(
cls: type["AiterMLAMetadataBuilder"],
vllm_config: VllmConfig,
kv_cache_spec: KVCacheSpec,
) -> int | None:
if not _adaptive_varlen_verify_enabled(vllm_config):
return None
if (
cls.get_cudagraph_support(vllm_config, kv_cache_spec)
!= AttentionCGSupport.UNIFORM_BATCH
):
return None
return max_decode_query_len(vllm_config)
def __init__(
self,
kv_cache_spec: AttentionSpec,
@@ -823,6 +855,26 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
# persistent gate below and makes aiter raise a KeyError mid-run.
self._mtp_decode_qlen = self.reorder_batch_threshold or 1
self._varlen_decode_qlen = 0
if _adaptive_varlen_verify_enabled(vllm_config):
assert self._mtp_decode_qlen == max_decode_query_len(vllm_config), (
self._mtp_decode_qlen,
max_decode_query_len(vllm_config),
)
self._varlen_decode_qlen = self._mtp_decode_qlen
self.query_len_support = QueryLenSupport.VARLEN # type: ignore[misc]
max_rows = vllm_config.scheduler_config.max_num_seqs * (
self._varlen_decode_qlen
)
row_ids = torch.arange(max_rows, dtype=torch.int64, device=device)
self._varlen_row_req = row_ids // self._varlen_decode_qlen
self._varlen_row_pos = row_ids % self._varlen_decode_qlen
self._varlen_row_ids = row_ids
self._varlen_q_rows = torch.zeros_like(row_ids)
self._varlen_o_rows = torch.zeros(
max_rows + 1, dtype=torch.int64, device=device
)
# Store the kernel block size from the spec. When kernel_block_size=1
# (no spec-dec), behavior is identical to the original. When > 1
# (e.g. 16 with Eagle3), we expand block-level indices into per-token
@@ -850,6 +902,12 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
max_num_pages_per_req = vllm_config.model_config.max_model_len
max_num_reqs = vllm_config.scheduler_config.max_num_seqs
max_num_pages = max_num_reqs * max_num_pages_per_req
speculative_config = vllm_config.speculative_config
if _adaptive_varlen_verify_enabled(vllm_config):
assert speculative_config is not None
max_num_pages += max_num_reqs * (
speculative_config.num_speculative_tokens or 0
)
# Preparing persistent buffers
# TODO: we can disambiguate between decode and mixed-prefill decode here
@@ -1437,13 +1495,18 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
device = self.device
num_reqs = seq_lens_device.size(0)
qo_len = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1]
max_qo_len = qo_len.max().item()
padded_mtp_qo_len = self._uniform_padded_mtp_qo_len(
qo_len, max_qo_len, num_decode_tokens
)
if padded_mtp_qo_len > 0:
max_qo_len = padded_mtp_qo_len
pad_uniform_mtp = padded_mtp_qo_len > 0
varlen_decode = self._varlen_decode_qlen > 0 and causal
if varlen_decode:
max_qo_len = self._varlen_decode_qlen
pad_uniform_mtp = True
else:
max_qo_len = qo_len.max().item()
padded_mtp_qo_len = self._uniform_padded_mtp_qo_len(
qo_len, max_qo_len, num_decode_tokens
)
if padded_mtp_qo_len > 0:
max_qo_len = padded_mtp_qo_len
pad_uniform_mtp = padded_mtp_qo_len > 0
seq_lens_for_kernel = seq_lens_device
# the global lengths need the same dummy-row
@@ -1477,6 +1540,13 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
g_tot_seq_lens.new_full((), max_qo_len * self.dcp_world_size),
)
varlen_q_rows = varlen_o_rows = None
if varlen_decode:
seq_lens_for_kernel = seq_lens_device + (max_qo_len - qo_lens_device)
varlen_q_rows, varlen_o_rows = self._fill_varlen_rows(
query_start_loc_device, qo_lens_device, num_reqs
)
if self._graph_seq_lens is not None:
self._graph_seq_lens[:num_kernel_reqs].copy_(
seq_lens_for_kernel, non_blocking=True
@@ -1537,6 +1607,8 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
# device synchronisation. Programs whose chunk lies past
# num_tokens mask out entirely.
max_tokens_per_req = block_table_tensor.shape[1] * self.kernel_block_size
if varlen_decode:
max_tokens_per_req += max_qo_len - 1
num_chunks = max(1, cdiv(max_tokens_per_req, 1024))
_expand_page_indices_kernel[(num_reqs, num_chunks)](
self.paged_kv_indices,
@@ -1544,8 +1616,10 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
block_table_tensor.stride(0),
block_table_tensor.stride(1),
paged_kv_indptr,
seq_lens_device,
KERNEL_BLOCK_SIZE=self.kernel_block_size,
BLOCK_SIZE=1024,
CLAMP_TAIL=varlen_decode,
)
paged_kv_indices = self.paged_kv_indices
@@ -1740,10 +1814,38 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
use_gluon_verify=use_gluon_verify,
attn_out_dtype=self.decode_attn_out_dtype,
has_persistent_metadata=has_persistent_metadata,
varlen_q_rows=varlen_q_rows,
varlen_o_rows=varlen_o_rows,
)
return attn_metadata
def _fill_varlen_rows(
self,
query_start_loc_device: torch.Tensor,
qo_lens_device: torch.Tensor,
num_reqs: int,
) -> tuple[torch.Tensor, torch.Tensor]:
width = self._varlen_decode_qlen
num_rows = num_reqs * width
row_req = self._varlen_row_req[:num_rows]
row_pos = self._varlen_row_pos[:num_rows]
row_qlen = qo_lens_device.to(torch.int64).index_select(0, row_req)
row_start = query_start_loc_device.to(torch.int64).index_select(0, row_req)
is_real = row_pos < row_qlen
packed_row = row_start + row_pos
q_rows = self._varlen_q_rows[:num_rows]
q_rows.copy_(torch.where(is_real, packed_row, 0))
o_rows = self._varlen_o_rows
o_rows.zero_()
dump_slot = o_rows.size(0) - 1
o_rows.scatter_(
0,
torch.where(is_real, packed_row, dump_slot),
self._varlen_row_ids[:num_rows],
)
return q_rows, o_rows
def build(
self,
common_prefix_len: int,
@@ -1795,8 +1897,10 @@ def _expand_page_indices_kernel(
block_table_stride_0,
block_table_stride_1,
cu_num_tokens,
num_real_tokens,
KERNEL_BLOCK_SIZE: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
CLAMP_TAIL: tl.constexpr = False,
):
"""Expand block table entries into per-token flat page indices.
@@ -1810,6 +1914,10 @@ def _expand_page_indices_kernel(
When KERNEL_BLOCK_SIZE=K: block table entry b (covering K tokens)
is expanded to flat indices b*K, b*K+1, ..., b*K+(K-1).
With CLAMP_TAIL, entries past num_real_tokens[req] (padding a request's
page list out to its kernel length) repeat the request's first page, so
they never index past its block-table row.
"""
# One program per (request, token-chunk). Parallelising over requests
# alone degenerates at low concurrency: with num_reqs == 1 a single
@@ -1819,6 +1927,8 @@ def _expand_page_indices_kernel(
row_ptr = block_table + req_idx * block_table_stride_0
start_idx = tl.load(cu_num_tokens + req_idx)
num_tokens = tl.load(cu_num_tokens + req_idx + 1) - start_idx
if CLAMP_TAIL:
real_tokens = tl.load(num_real_tokens + req_idx)
# The grid is sized from the block table width, an upper bound over all
# requests, so a ragged batch launches chunks past a short request's end.
@@ -1830,11 +1940,14 @@ def _expand_page_indices_kernel(
token_offsets = chunk_start + tl.arange(0, BLOCK_SIZE)
mask = token_offsets < num_tokens
source_offsets = token_offsets
if CLAMP_TAIL:
source_offsets = tl.where(token_offsets < real_tokens, token_offsets, 0)
# Which block in the block table does this token belong to?
block_idx = token_offsets // KERNEL_BLOCK_SIZE
block_idx = source_offsets // KERNEL_BLOCK_SIZE
# Offset within that block
offset_in_block = token_offsets % KERNEL_BLOCK_SIZE
offset_in_block = source_offsets % KERNEL_BLOCK_SIZE
# Load the block ID from the block table
# Both strides are taken from the caller: the block table is a view owned
@@ -2600,6 +2713,28 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
kv_c_and_k_pe_cache: torch.Tensor,
attn_metadata: AiterMLAMetadata,
layer: AttentionLayer,
) -> tuple[torch.Tensor, torch.Tensor | None]:
decode = attn_metadata.decode
if decode is None or decode.varlen_q_rows is None:
return self._forward_mqa(q, kv_c_and_k_pe_cache, attn_metadata, layer)
assert decode.varlen_o_rows is not None
q_rows = decode.varlen_q_rows
if isinstance(q, tuple):
num_tokens = q[0].shape[0]
q = (q[0].index_select(0, q_rows), q[1].index_select(0, q_rows))
else:
num_tokens = q.shape[0]
q = q.index_select(0, q_rows)
o, lse = self._forward_mqa(q, kv_c_and_k_pe_cache, attn_metadata, layer)
assert lse is None
return o.index_select(0, decode.varlen_o_rows[:num_tokens]), None
def _forward_mqa(
self,
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
kv_c_and_k_pe_cache: torch.Tensor,
attn_metadata: AiterMLAMetadata,
layer: AttentionLayer,
) -> tuple[torch.Tensor, torch.Tensor | None]:
assert kv_c_and_k_pe_cache.numel() > 0
assert attn_metadata.decode is not None
+103
View File
@@ -0,0 +1,103 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Interface and resolution for pluggable KV cache config builders."""
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING
from vllm.utils.import_utils import resolve_obj_by_qualname
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
KVCacheSpec,
)
class KVCacheConfigBuilder(ABC):
"""Interface for platform KV cache placement.
.. note::
This interface is experimental and subject to change. The exact hooks
and the ``num_blocks`` contract may evolve before it stabilizes.
Subclasses normally inherit from ``DefaultKVCacheConfigBuilder`` and
override only the hooks they need. The cross-worker planning flow
(:func:`vllm.v1.core.kv_cache_planning.get_kv_cache_configs`) is Core-owned;
this builder supplies the placement decisions: how layers are split into
groups and how a group list is materialized into tensors for a given block
count. Grouping coordination lives in ``DefaultKVCacheConfigBuilder`` (a
model implementing :class:`~vllm.model_executor.models.interfaces.
SupportsKVCacheGrouping` owns its grouping there); a platform that replaces
the builder fully owns its own grouping.
"""
@abstractmethod
def get_kv_cache_groups(
self,
vllm_config: "VllmConfig",
kv_cache_spec: dict[str, "KVCacheSpec"],
) -> list["KVCacheGroupSpec"]:
"""Split the given layers into KV cache groups.
Planning passes all workers' layers; profiling passes one worker's.
Core rejects a result that does not put each layer in exactly one group.
The default builder honors a model's
:class:`~vllm.model_executor.models.interfaces.SupportsKVCacheGrouping`
declaration here before falling back to the default grouping; a
platform-specific builder decides its own grouping.
"""
raise NotImplementedError
@abstractmethod
def get_kv_cache_config_from_groups(
self,
vllm_config: "VllmConfig",
kv_cache_groups: list["KVCacheGroupSpec"],
num_blocks: int,
) -> "KVCacheConfig":
"""Materialize the groups for exactly ``num_blocks`` blocks.
Each device backing's size is ``bytes_per_block * num_blocks``. All
device tensors share one backing allocation unless the builder
overrides ``_get_pool_bytes_per_block``. Host-resident tensors, such as
HiSparse source storage, are excluded from the device byte accounting
and sized independently.
"""
raise NotImplementedError
def _get_pool_bytes_per_block(
self,
vllm_config: "VllmConfig",
kv_cache_groups: list["KVCacheGroupSpec"],
) -> int | None:
"""Return the device bytes one block ID consumes, or ``None``.
Private and transitional; expected to go away with worker-driven
allocation (RFC #42449). With ``None`` (the default), Core derives the
value from a one-block placement with one device backing. Override it
only for placements with several device allocations.
"""
return None
def get_kv_cache_config_builder(
vllm_config: "VllmConfig",
) -> KVCacheConfigBuilder:
"""Resolve the builder selected by the current platform.
Resolution priority is owned by the platform hook
(:meth:`vllm.platforms.interface.Platform.get_kv_cache_config_builder_cls`).
Builders are stateless (every method takes ``vllm_config``), so a fresh
instance is returned per call and engines with different configs in the
same process always get the correct builder.
"""
from vllm.platforms import current_platform
builder_cls = resolve_obj_by_qualname(
current_platform.get_kv_cache_config_builder_cls(vllm_config)
)
return builder_cls()
+190 -24
View File
@@ -14,6 +14,7 @@ from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv, round_up
from vllm.utils.mem_utils import format_gib
from vllm.utils.torch_utils import get_dtype_size
from vllm.v1.core.kv_cache_config_builder import KVCacheConfigBuilder
from vllm.v1.core.kv_cache_utils import _kv_cache_device_name
from vllm.v1.hisparse.layout import (
get_hisparse_host_pool_bytes,
@@ -261,6 +262,48 @@ def _pool_bytes_per_block(
)
def _pool_bytes_per_block_from_builder(
builder: KVCacheConfigBuilder,
vllm_config: VllmConfig,
kv_cache_groups: list[KVCacheGroupSpec],
) -> int:
"""Bytes consumed by one block in the worker's device KV cache pool.
Core places one block with the builder and reads the size of its single
device backing. A placement with several device allocations declares the
value through ``KVCacheConfigBuilder._get_pool_bytes_per_block``. HiSparse
keeps the pure computation, since probing its placement would check the host
pool against ``max_model_len`` before auto-fit shortens it.
``num_gpu_blocks_override`` is suppressed so it cannot inflate the probe.
"""
if not kv_cache_groups:
return 0
if vllm_config.attention_config.hisparse_config is not None:
return _pool_bytes_per_block(vllm_config, kv_cache_groups)
pool_bytes = builder._get_pool_bytes_per_block(vllm_config, kv_cache_groups)
if pool_bytes is not None:
return pool_bytes
saved_override = vllm_config.cache_config.num_gpu_blocks_override
vllm_config.cache_config.num_gpu_blocks_override = None
try:
probe = builder.get_kv_cache_config_from_groups(
vllm_config, kv_cache_groups, num_blocks=1
)
finally:
vllm_config.cache_config.num_gpu_blocks_override = saved_override
backing_sizes = {
tensor.size for tensor in probe.kv_cache_tensors if not tensor.host_resident
}
if len(backing_sizes) != 1:
raise ValueError(
"KV cache tensors must share one device backing allocation, but got "
f"sizes {sorted(backing_sizes)}. Override "
"KVCacheConfigBuilder._get_pool_bytes_per_block() for placements "
"with several device allocations."
)
return backing_sizes.pop()
def get_uniform_page_size(kv_cache_specs: Iterable[KVCacheSpec]) -> int:
"""Get the page size of the KV cache."""
page_sizes = {layer.page_size_bytes for layer in kv_cache_specs}
@@ -1264,6 +1307,36 @@ def get_kv_cache_groups(
return groups
def get_kv_cache_groups_from_builder(
builder: KVCacheConfigBuilder,
vllm_config: VllmConfig,
kv_cache_spec: dict[str, KVCacheSpec],
) -> list[KVCacheGroupSpec]:
"""Group the layers with the builder and check the result.
Core checks every grouping, whether it comes from the default grouping, a
model's ``SupportsKVCacheGrouping`` or a platform builder.
"""
groups = builder.get_kv_cache_groups(vllm_config, kv_cache_spec)
_check_exact_partition(groups, kv_cache_spec)
return groups
def _check_exact_partition(
groups: list[KVCacheGroupSpec],
kv_cache_spec: dict[str, KVCacheSpec],
) -> None:
"""Ensure the groups partition every input layer exactly once."""
grouped_layers: set[str] = set()
for group in groups:
assert not set(group.layer_names) & grouped_layers, (
f"KV cache group overlaps: {set(group.layer_names) & grouped_layers}."
)
grouped_layers |= set(group.layer_names)
missing = set(kv_cache_spec) - grouped_layers
assert not missing, f"KV cache groups miss layers: {sorted(missing)}."
def _layer_tp_replicas(spec: KVCacheSpec, tp_size: int, dcp_size: int) -> int:
if not isinstance(spec, AttentionSpec) or spec.max_tp_shards is None:
return 1
@@ -1285,6 +1358,7 @@ def kv_cache_groups_tp_replicas(
def _max_memory_usage_bytes_from_groups(
builder: KVCacheConfigBuilder,
vllm_config: VllmConfig,
kv_cache_groups: list[KVCacheGroupSpec],
) -> int:
@@ -1301,7 +1375,9 @@ def _max_memory_usage_bytes_from_groups(
if not kv_cache_groups:
return 0
bytes_per_block = _pool_bytes_per_block(vllm_config, kv_cache_groups)
bytes_per_block = _pool_bytes_per_block_from_builder(
builder, vllm_config, kv_cache_groups
)
total_blocks = 0
for group in kv_cache_groups:
spec = group.kv_cache_spec
@@ -1317,6 +1393,7 @@ def _max_memory_usage_bytes_from_groups(
def _estimate_max_model_len_from_groups(
builder: KVCacheConfigBuilder,
vllm_config: VllmConfig,
kv_cache_groups: list[KVCacheGroupSpec],
available_memory: int,
@@ -1334,13 +1411,16 @@ def _estimate_max_model_len_from_groups(
vllm_config.model_config.max_model_len = model_len
if hisparse_enabled:
try:
get_kv_cache_config_from_groups(
vllm_config, kv_cache_groups, available_memory
num_blocks = available_memory // _pool_bytes_per_block_from_builder(
builder, vllm_config, kv_cache_groups
)
builder.get_kv_cache_config_from_groups(
vllm_config, kv_cache_groups, num_blocks
)
except ValueError:
return False
return (
_max_memory_usage_bytes_from_groups(vllm_config, kv_cache_groups)
_max_memory_usage_bytes_from_groups(builder, vllm_config, kv_cache_groups)
<= available_memory
)
@@ -1362,6 +1442,7 @@ def _estimate_max_model_len_from_groups(
def _auto_fit_max_model_len(
builder: KVCacheConfigBuilder,
vllm_config: VllmConfig,
projected_groups_per_worker: list[list[KVCacheGroupSpec]],
available_memory: list[int],
@@ -1372,6 +1453,7 @@ def _auto_fit_max_model_len(
workers.
Args:
builder: The builder whose hooks drive block accounting.
vllm_config: The global VllmConfig (will be modified in-place)
projected_groups_per_worker: KV cache groups projected to each worker.
available_memory: Memory available for KV cache in bytes for each
@@ -1395,7 +1477,9 @@ def _auto_fit_max_model_len(
for groups, avail_mem in zip(projected_groups_per_worker, available_memory):
if not groups:
continue
worker_max = _estimate_max_model_len_from_groups(vllm_config, groups, avail_mem)
worker_max = _estimate_max_model_len_from_groups(
builder, vllm_config, groups, avail_mem
)
if worker_max < auto_fit_max:
auto_fit_max = worker_max
limiting_worker_mem = avail_mem
@@ -1473,6 +1557,9 @@ def _project_kv_cache_groups_to_worker(
worker_layer_names,
group_spec,
is_eagle_group=group.is_eagle_group and bool(worker_layer_names),
host_resident=group.host_resident,
enable_kv_transfer=group.enable_kv_transfer,
role=group.role,
)
)
return projected_groups
@@ -1512,6 +1599,10 @@ def get_kv_cache_configs(
The generated KVCacheConfigs for each worker.
"""
from vllm.v1.core.kv_cache_config_builder import get_kv_cache_config_builder
builder = get_kv_cache_config_builder(vllm_config)
# Merge the KV cache specs of all workers. Different PP stages may have
# different layer names, and different TP ranks of the same PP stage should
# have the same KV cache spec.
@@ -1542,7 +1633,9 @@ def get_kv_cache_configs(
# Get global KV cache groups. This also handles spec unification for
# hybrid models when disable_hybrid_kv_cache_manager is enabled.
# After this call, merged_kv_cache_specs may be modified in-place.
global_kv_cache_groups = get_kv_cache_groups(vllm_config, merged_kv_cache_specs)
global_kv_cache_groups = get_kv_cache_groups_from_builder(
builder, vllm_config, merged_kv_cache_specs
)
# If original_max_model_len was -1, automatically
# determine the maximum model length that fits in available GPU memory.
@@ -1553,11 +1646,10 @@ def get_kv_cache_configs(
]
# If `num_gpu_blocks_override` is set, the cache size that will actually
# be allocated is decoupled from the profiled `available_memory`:
# `may_override_num_blocks` in `get_kv_cache_config_from_groups` clamps
# `num_blocks` to the override. Reflect that in `available_memory` here so
# auto-fit, the admission check, and the per-worker config builder all
# plan against the same effective capacity.
# be allocated is decoupled from the profiled `available_memory`. Reflect
# that in `available_memory` here so auto-fit, the admission check, and
# the per-worker config builder all plan against the same effective
# capacity.
override = vllm_config.cache_config.num_gpu_blocks_override
if override is not None:
adjusted_memory: list[int] = []
@@ -1565,7 +1657,9 @@ def get_kv_cache_configs(
if not groups:
adjusted_memory.append(avail_mem)
continue
bytes_per_block = _pool_bytes_per_block(vllm_config, groups)
bytes_per_block = _pool_bytes_per_block_from_builder(
builder, vllm_config, groups
)
logger.info(
"Overriding num_gpu_blocks=%d with num_gpu_blocks_override=%d",
avail_mem // bytes_per_block,
@@ -1578,12 +1672,18 @@ def get_kv_cache_configs(
# the capacity check both plan against usable blocks. Allocation below
# still uses the full memory.
check_memory = [
avail_mem - _pool_bytes_per_block(vllm_config, groups) if groups else avail_mem
(
avail_mem - _pool_bytes_per_block_from_builder(builder, vllm_config, groups)
if groups
else avail_mem
)
for groups, avail_mem in zip(projected_groups_per_worker, available_memory)
]
if vllm_config.model_config.original_max_model_len == -1:
_auto_fit_max_model_len(vllm_config, projected_groups_per_worker, check_memory)
_auto_fit_max_model_len(
builder, vllm_config, projected_groups_per_worker, check_memory
)
# Check if the available memory is enough per worker.
for groups, avail_mem in zip(projected_groups_per_worker, check_memory):
@@ -1591,9 +1691,9 @@ def get_kv_cache_configs(
continue
_check_enough_kv_cache_memory(
avail_mem,
partial(_max_memory_usage_bytes_from_groups, vllm_config, groups),
partial(_max_memory_usage_bytes_from_groups, builder, vllm_config, groups),
vllm_config.model_config.max_model_len,
partial(_estimate_max_model_len_from_groups, vllm_config, groups),
partial(_estimate_max_model_len_from_groups, builder, vllm_config, groups),
)
kv_cache_configs: list[KVCacheConfig] = []
@@ -1603,9 +1703,15 @@ def get_kv_cache_configs(
assert sum(len(group.layer_names) for group in projected_groups) == len(
kv_cache_spec_one_worker
), "Some layers are not assigned to any group."
bytes_per_block = _pool_bytes_per_block_from_builder(
builder, vllm_config, projected_groups
)
num_blocks = (
available_memory_one_worker // bytes_per_block if projected_groups else 1
)
kv_cache_configs.append(
get_kv_cache_config_from_groups(
vllm_config, projected_groups, available_memory_one_worker
builder.get_kv_cache_config_from_groups(
vllm_config, projected_groups, num_blocks
)
)
@@ -1618,13 +1724,12 @@ def get_kv_cache_configs(
for i, kv_cache_config in enumerate(kv_cache_configs):
if kv_cache_config.num_blocks == min_num_blocks:
continue
# Re-plan with exactly the memory the smallest rank can afford, so
# strides and offsets stay consistent with the shrunken allocation.
groups = kv_cache_config.kv_cache_groups
kv_cache_configs[i] = get_kv_cache_config_from_groups(
# Re-plan with exactly the smallest rank's block count, so strides and
# offsets stay consistent with the shrunken allocation.
kv_cache_configs[i] = builder.get_kv_cache_config_from_groups(
vllm_config,
groups,
min_num_blocks * _pool_bytes_per_block(vllm_config, groups),
kv_cache_config.kv_cache_groups,
min_num_blocks,
)
for kv_cache_config in kv_cache_configs:
@@ -1635,3 +1740,64 @@ def get_kv_cache_configs(
)
return kv_cache_configs
class DefaultKVCacheConfigBuilder(KVCacheConfigBuilder):
"""Default grouping and placement, backed by the module-level functions here.
.. note::
This class is part of the experimental ``KVCacheConfigBuilder``
interface and is subject to change.
Subclasses override only the hooks they need. The planning flow and its
invariants (``num_gpu_blocks_override``, null-block reservation, auto-fit,
admission checks, cross-rank convergence) stay in the Core-owned
:func:`get_kv_cache_configs`.
"""
def get_kv_cache_groups(
self,
vllm_config: VllmConfig,
kv_cache_spec: dict[str, KVCacheSpec],
) -> list[KVCacheGroupSpec]:
"""See ``KVCacheConfigBuilder.get_kv_cache_groups``.
A model implementing ``SupportsKVCacheGrouping`` owns the grouping
decision; otherwise the default grouping is used.
"""
if not vllm_config.model_config.supports_kv_cache_grouping:
return get_kv_cache_groups(vllm_config, kv_cache_spec)
if vllm_config.attention_config.hisparse_config is not None:
raise ValueError(
"A model-declared get_kv_cache_groups cannot be combined "
"with HiSparse, which owns its own KV cache grouping. "
"Remove one of them."
)
model_cls, _ = vllm_config.model_config.registry.resolve_model_cls(
vllm_config.model_config.architectures,
vllm_config.model_config,
)
return model_cls.get_kv_cache_groups(vllm_config, kv_cache_spec)
def get_kv_cache_config_from_groups(
self,
vllm_config: VllmConfig,
kv_cache_groups: list[KVCacheGroupSpec],
num_blocks: int,
) -> KVCacheConfig:
"""See ``KVCacheConfigBuilder.get_kv_cache_config_from_groups``."""
if len(kv_cache_groups) == 0:
# Attention free models do not have KV cache.
# Return num_blocks=1 as BlockPool always needs a null_block.
return KVCacheConfig(
num_blocks=1,
kv_cache_tensors=[],
kv_cache_groups=kv_cache_groups,
prefix_cache_retention_interval=(
vllm_config.cache_config.prefix_cache_retention_interval
),
)
bytes_per_block = _pool_bytes_per_block(vllm_config, kv_cache_groups)
return get_kv_cache_config_from_groups(
vllm_config, kv_cache_groups, num_blocks * bytes_per_block
)
+10 -32
View File
@@ -41,12 +41,9 @@ class DPCoordinator:
DPEngineCoreProc._has_global_unfinished_reqs method.
* Broadcasts the START_DP_WAVE message to engines to move them from paused
to running state when one engine receives a new request. This can happen
in two cases:
1) A front-end sending a new request while the engines are paused will
concurrently notify the coordinator.
2) An engine receiving a request for a stale request wave while in paused
state will notify the coordinator.
to running state when an idle engine receives a new request and notifies
the coordinator. Only the engines start waves: a front-end's view of the
running state can be stale, and a paused engine discards START_DP_WAVE.
Engines will move into running state when receiving a new request or
START_DP_WAVE message.
@@ -353,27 +350,9 @@ class DPCoordinatorProc:
)
continue # Skip normal engine notification processing
# Wave coordination: handle new-request messages from front-end.
# Only process these when wave coordination is enabled
if self.enable_wave_coordination:
# We received a message on the front-end XPUB socket,
# from an API server sending a new request while the
# engines are paused, so that we can wake the other
# engines.
engine_to_exclude, wave = decoded
if not engines_running:
if wave < current_wave:
# If the wave number is stale, ensure the message
# is handled by all the engines.
engine_to_exclude = None
# engines_running is only set from the engines'
# own notifications; a paused engine discards
# START_DP_WAVE, so sending it is not evidence
# that the engines are running.
self._send_start_wave(
publish_back, current_wave, engine_to_exclude
)
logger.error(
"DP Coordinator received unexpected message from front-end"
)
if output_back in events:
# We received a message from one of the engines.
@@ -448,13 +427,12 @@ class DPCoordinatorProc:
wave > current_wave
or (wave == current_wave and not engines_running)
):
# 3. The engine received request for a non-current wave
# so we must ensure that other engines progress to the
# next wave (race condition handling).
# 3. An engine started a wave (e.g. an idle engine
# received a request), so the others must join it.
logger.debug(
"Starting wave %d after notification of "
"stale wave request from engine.",
"Starting wave %d on notification from engine %d.",
wave,
eng_index,
)
current_wave = wave
engines_running = True
+3 -4
View File
@@ -2269,15 +2269,14 @@ class DPEngineCoreProc(EngineCoreProc):
def add_request(self, request: Request, request_wave: int = 0):
super().add_request(request, request_wave)
if self.has_coordinator and request_wave != self.current_wave:
if self.has_coordinator:
if request_wave > self.current_wave:
self.current_wave = request_wave
elif (
if (
not self.engines_running
and self.scheduler.pause_state == PauseState.UNPAUSED
):
# Request received for an already-completed wave, notify
# front-end that we need to start the next one.
# Engines alone start waves: have the coordinator wake the others.
self.engines_running = True
self.output_queue.put_nowait(
(-1, EngineCoreOutputs(start_wave=self.current_wave))
+43 -75
View File
@@ -506,8 +506,8 @@ class BackgroundResources:
coordinator: DPCoordinator | None = None
output_socket: zmq.Socket | zmq.asyncio.Socket | None = None
input_socket: zmq.Socket | zmq.asyncio.Socket | None = None
first_req_send_socket: zmq.asyncio.Socket | None = None
first_req_rcv_socket: zmq.asyncio.Socket | None = None
scale_send_socket: zmq.asyncio.Socket | None = None
scale_rcv_socket: zmq.asyncio.Socket | None = None
stats_update_socket: zmq.asyncio.Socket | None = None
output_queue_task: asyncio.Task | None = None
stats_update_task: asyncio.Task | None = None
@@ -535,8 +535,8 @@ class BackgroundResources:
sockets = (
self.output_socket,
self.input_socket,
self.first_req_send_socket,
self.first_req_rcv_socket,
self.scale_send_socket,
self.scale_rcv_socket,
self.stats_update_socket,
)
@@ -1493,9 +1493,9 @@ class DPAsyncMPClient(AsyncMPClient):
self.eep_scaling_cache: ElasticScalingCache | None = None
self.first_req_sock_addr = get_open_zmq_inproc_path()
self.first_req_send_socket = self.resources.first_req_send_socket = (
make_zmq_socket(self.ctx, self.first_req_sock_addr, zmq.PAIR, bind=True)
self.scale_sock_addr = get_open_zmq_inproc_path()
self.scale_send_socket = self.resources.scale_send_socket = make_zmq_socket(
self.ctx, self.scale_sock_addr, zmq.PAIR, bind=True
)
try:
# If we are running in an asyncio event loop, start the stats task.
@@ -1518,75 +1518,48 @@ class DPAsyncMPClient(AsyncMPClient):
with (
make_zmq_socket(self.ctx, stats_addr, zmq.XSUB, linger=0) as socket,
make_zmq_socket(
self.ctx, self.first_req_sock_addr, zmq.PAIR, bind=False, linger=0
) as first_req_rcv_socket,
self.ctx, self.scale_sock_addr, zmq.PAIR, bind=False, linger=0
) as scale_rcv_socket,
):
assert isinstance(socket, zmq.asyncio.Socket)
assert isinstance(first_req_rcv_socket, zmq.asyncio.Socket)
assert isinstance(scale_rcv_socket, zmq.asyncio.Socket)
self.resources.stats_update_socket = socket
self.resources.first_req_rcv_socket = first_req_rcv_socket
self.resources.scale_rcv_socket = scale_rcv_socket
# Send subscription message.
await socket.send(b"\x01")
poller = zmq.asyncio.Poller()
poller.register(socket, zmq.POLLIN)
poller.register(first_req_rcv_socket, zmq.POLLIN)
poller.register(scale_rcv_socket, zmq.POLLIN)
while True:
events = await poller.poll()
if (
not self.engines_running
and len(events) == 2
or (events[0][0] == first_req_rcv_socket)
):
# Check if this is a regular request notification or
# scale up notification
buf = first_req_rcv_socket.recv(flags=zmq.NOBLOCK).result()
decoded = msgspec.msgpack.decode(buf)
if (
isinstance(decoded, (list, tuple))
and len(decoded) == 2
and decoded[0] == "SCALE_ELASTIC_EP"
):
# Extract new engine count from the decoded message
new_engine_count = decoded[1]
# Update engine_ranks_managed and count_slice
parallel_config = self.vllm_config.parallel_config
dp_size = parallel_config.data_parallel_size
dp_rank = parallel_config.data_parallel_rank
assert dp_rank == 0
assert dp_size == new_engine_count
assert not parallel_config.data_parallel_hybrid_lb
# External LB clients continue to manage only local ranks.
if not parallel_config.data_parallel_external_lb:
self.engine_ranks_managed = list(range(dp_size))
if len(self.lb_engines) < new_engine_count:
self.lb_engines += [
[0, 0, 0.0]
for _ in range(
new_engine_count - len(self.lb_engines)
)
]
else:
self.lb_engines = self.lb_engines[:new_engine_count]
# Send scale up notification to coordinator
scale_msg = msgspec.msgpack.encode(
("SCALE_ELASTIC_EP", new_engine_count)
)
await socket.send(scale_msg)
continue
# we're sending a request while the engines are
# paused, so that it can wake the others up
# (to run dummy EP loop).
assert decoded[0] == "FIRST_REQ"
target_eng_index = decoded[1]
self.engines_running = True
msg = msgspec.msgpack.encode(
(target_eng_index, self.current_wave)
)
await socket.send(msg)
events = dict(await poller.poll())
if scale_rcv_socket in events:
buf = scale_rcv_socket.recv(flags=zmq.NOBLOCK).result()
msg_type, new_engine_count = msgspec.msgpack.decode(buf)
assert msg_type == "SCALE_ELASTIC_EP"
# Update engine_ranks_managed and count_slice
parallel_config = self.vllm_config.parallel_config
dp_size = parallel_config.data_parallel_size
dp_rank = parallel_config.data_parallel_rank
assert dp_rank == 0
assert dp_size == new_engine_count
assert not parallel_config.data_parallel_hybrid_lb
# External LB clients continue to manage only local ranks.
if not parallel_config.data_parallel_external_lb:
self.engine_ranks_managed = list(range(dp_size))
if len(self.lb_engines) < new_engine_count:
self.lb_engines += [
[0, 0, 0.0]
for _ in range(
new_engine_count - len(self.lb_engines)
)
]
else:
self.lb_engines = self.lb_engines[:new_engine_count]
# Forward the scale notification to the coordinator.
await socket.send(buf)
continue
buf = None
while True:
@@ -1624,14 +1597,9 @@ class DPAsyncMPClient(AsyncMPClient):
request.current_wave = self.current_wave
request.client_index = self.client_index
# The engine that receives it wakes the other ranks if they are idle.
chosen_engine = self.get_core_engine_for_request(request)
to_await = self._send_input(EngineCoreRequestType.ADD, request, chosen_engine)
if not self.engines_running:
# Notify coordinator that we're sending a request
req_msg = msgspec.msgpack.encode(("FIRST_REQ", chosen_engine))
await self.first_req_send_socket.send(req_msg)
await to_await
await self._send_input(EngineCoreRequestType.ADD, request, chosen_engine)
self._ensure_output_queue_task()
@@ -2100,7 +2068,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
scale_up_marker = msgspec.msgpack.encode(
("SCALE_ELASTIC_EP", new_data_parallel_size)
)
await self.first_req_send_socket.send(scale_up_marker)
await self.scale_send_socket.send(scale_up_marker)
logger.info(
"[Elastic EP] Scale up completed, new data parallel size: %s",
@@ -2176,7 +2144,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
scale_down_marker = msgspec.msgpack.encode(
("SCALE_ELASTIC_EP", new_data_parallel_size)
)
await self.first_req_send_socket.send(scale_down_marker)
await self.scale_send_socket.send(scale_down_marker)
await wait_future
await self.resume_scheduler_async()
except Exception:
+12 -12
View File
@@ -483,24 +483,24 @@ class TopKTopPSampler(nn.Module):
return xpu_sample(logits, k, p, self.logprobs_mode)
# Note: this is a workaround for
# https://github.com/pytorch/pytorch/pull/151218
@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend)
def compiled_random_sample(logits: torch.Tensor) -> torch.Tensor:
probs = logits.softmax(dim=-1, dtype=torch.float32)
q = torch.empty_like(probs)
q.exponential_()
return probs.div(q).argmax(dim=-1).view(-1)
def apply_top_k_top_p(
logits: torch.Tensor, k: torch.Tensor | None, p: torch.Tensor | None
logits: torch.Tensor,
k: torch.Tensor | None,
p: torch.Tensor | None,
has_topp_only_rows: bool = True,
) -> torch.Tensor:
"""Apply top-k and/or top-p masking to logits.
has_topp_only_rows must be True if any row has p < 1 and
k >= logits.shape[-1] (no active top-k).
"""
if p is None and k is None:
return logits
if HAS_TRITON:
return apply_top_k_top_p_triton(logits, k, p)
return apply_top_k_top_p_triton(
logits, k, p, has_topp_only_rows=has_topp_only_rows
)
is_cpu = current_platform.is_cpu()
return apply_top_k_top_p_pytorch(logits, k, p, allow_cpu_sync=is_cpu)
+17 -6
View File
@@ -987,13 +987,19 @@ _SPLIT_MAX_BATCH = 64
_SPLIT_MAX_SPLITS = 32 # power of two
_SPLIT_FANOUT = 8
_SPLIT_ROUNDS = 5
# The step kernels are latency-bound at one program per SM, so keep splitting
# past that while slices span at least this many entries; shorter slices no
# longer amortize each program's row stats and combine replay.
_SPLIT_MIN_SLICE = 7168
_TRITON_SPLIT_CACHE: dict[torch.device, dict[str, torch.Tensor]] = {}
def _topp_split_count(batch_size: int, num_sm: int) -> int:
def _topp_split_count(batch_size: int, vocab_size: int, num_sm: int) -> int:
splits = 1
while splits < _SPLIT_MAX_SPLITS and splits * 2 * batch_size <= num_sm:
while splits < _SPLIT_MAX_SPLITS and (
splits * 2 * batch_size <= num_sm or splits * 2 * _SPLIT_MIN_SLICE <= vocab_size
):
splits *= 2
return splits
@@ -1079,7 +1085,7 @@ def _topp_split_stats(
num_sm: int,
) -> DispatchSpec:
batch_size, vocab_size = logits.shape
splits = _topp_split_count(batch_size, num_sm)
splits = _topp_split_count(batch_size, vocab_size, num_sm)
return (logits.shape[0] * splits,), dict(
K=k if k is not None else logits,
HAS_K=k is not None,
@@ -1373,7 +1379,7 @@ def _topp_split_step(
num_sm: int,
) -> DispatchSpec:
batch_size, vocab_size = logits.shape
splits = _topp_split_count(batch_size, num_sm)
splits = _topp_split_count(batch_size, vocab_size, num_sm)
return (logits.shape[0] * splits,), dict(
K=k if k is not None else logits,
HAS_K=k is not None,
@@ -1513,6 +1519,7 @@ def _topp_sb_mask_kernel(
cum = tl.cumsum(dmask.to(tl.int32)).to(tl.float32) + num_preceding
keep_dup = dmask & (cum <= numkeep)
keep = keep & (~dmask | keep_dup)
num_preceding += tl.sum(dmask.to(tl.float32))
tl.store(ROW + i + offs, tl.where(keep, x, MASK_VALUE), mask=mask_n)
@@ -1554,7 +1561,7 @@ def _topp_split_mask(
num_sm: int,
) -> DispatchSpec:
batch_size, vocab_size = logits.shape
splits = _topp_split_count(batch_size, num_sm)
splits = _topp_split_count(batch_size, vocab_size, num_sm)
return (logits.shape[0] * splits,), dict(
K=k if k is not None else logits,
HAS_K=k is not None,
@@ -1618,6 +1625,7 @@ def apply_top_k_top_p_triton(
k: torch.Tensor | None,
p: torch.Tensor | None,
mask_value: float = float("-inf"),
has_topp_only_rows: bool = True,
) -> torch.Tensor:
"""Apply combined top-k and top-p masking using Triton.
@@ -1632,6 +1640,9 @@ def apply_top_k_top_p_triton(
p: [batch_size] float32 tensor of top-p values per row (0 to 1),
or None to disable top-p
mask_value: Value for masked positions (default: -inf)
has_topp_only_rows: Whether any row has p < 1 and k >= vocab_size.
False lets small batches skip the split top-p pipeline; pass it
only if no such row exists, as such a row would get no top-p.
Returns:
The masked logits tensor. It may or may not be modified in-place.
@@ -1714,7 +1725,7 @@ def apply_top_k_top_p_triton(
mask_value,
num_sm,
)
if use_split:
if use_split and has_topp_only_rows:
_apply_topp_split(logits, k_ptr, p_ptr, mask_value, num_sm)
return logits
+15 -3
View File
@@ -151,6 +151,7 @@ class SimpleCPUOffloadScheduler:
lazy_offload: bool = False,
disk_capacity_bytes: int = 0,
use_page_cache: bool = False,
num_cpu_blocks: int | None = None,
):
self.vllm_config = vllm_config
self.kv_cache_config = kv_cache_config
@@ -170,7 +171,7 @@ class SimpleCPUOffloadScheduler:
# Derive a CPU KVCacheConfig from the GPU config and build a coordinator
assert kv_cache_config is not None
self.cpu_kv_cache_config = self._derive_cpu_config(
kv_cache_config, offload_capacity
kv_cache_config, offload_capacity, num_cpu_blocks
)
self.num_cpu_blocks = self.cpu_kv_cache_config.num_blocks
self.prefix_cacheable_group_ids = (
@@ -278,7 +279,9 @@ class SimpleCPUOffloadScheduler:
@staticmethod
def _derive_cpu_config(
gpu_config: "KVCacheConfig", cpu_capacity_bytes: int
gpu_config: "KVCacheConfig",
cpu_capacity_bytes: int,
num_cpu_blocks: int | None = None,
) -> "KVCacheConfig":
"""Derive a CPU KVCacheConfig from the GPU config.
Same kv_cache_groups, num_blocks scaled by CPU/GPU memory ratio."""
@@ -291,7 +294,16 @@ class SimpleCPUOffloadScheduler:
# so its size is the total GPU KV cache size.
gpu_total_bytes = gpu_config.kv_cache_tensors[0].size
num_gpu_blocks = gpu_config.num_blocks
num_cpu_blocks = max(1, num_gpu_blocks * cpu_capacity_bytes // gpu_total_bytes)
local_capacity = num_gpu_blocks * cpu_capacity_bytes // gpu_total_bytes
if num_cpu_blocks is None:
num_cpu_blocks = max(1, local_capacity)
elif (
type(num_cpu_blocks) is not int or not 0 < num_cpu_blocks <= local_capacity
):
raise ValueError(
f"num_cpu_blocks={num_cpu_blocks!r} must be between 1 and "
f"local offload capacity {local_capacity}"
)
# Create CPU kv_cache_tensors mirroring GPU by scaling size proportionally.
cpu_tensors = [
KVCacheTensor(
+21 -2
View File
@@ -36,10 +36,12 @@ class SimpleCPUOffloadWorker:
disk_capacity_bytes: int = 0,
disk_buffer_slots: int = 2,
use_page_cache: bool = False,
num_cpu_blocks: int | None = None,
):
self.vllm_config = vllm_config
self.kv_cache_config = kv_cache_config
self.cpu_capacity_bytes = cpu_capacity_bytes
self._num_cpu_blocks_override = num_cpu_blocks
self.disk_path = disk_path
self.disk_capacity_bytes = disk_capacity_bytes
self.disk_buffer_slots = disk_buffer_slots
@@ -159,7 +161,10 @@ class SimpleCPUOffloadWorker:
]
total_bytes_per_block = sum(per_tensor_bpb)
self.num_cpu_blocks = max(1, self.cpu_capacity_bytes // total_bytes_per_block)
self.num_cpu_blocks = self._resolve_num_cpu_blocks(
self.disk_capacity_bytes if self.disk_mode else self.cpu_capacity_bytes,
total_bytes_per_block,
)
# Use lowest priority so KV cache I/O yields to compute streams.
low_pri, _ = torch.cuda.Stream.priority_range()
@@ -173,13 +178,27 @@ class SimpleCPUOffloadWorker:
else:
self._init_cpu_mode(unique_gpu_caches, total_bytes_per_block, self.device)
def _resolve_num_cpu_blocks(self, capacity_bytes: int, bytes_per_block: int) -> int:
local_capacity = capacity_bytes // bytes_per_block
requested = self._num_cpu_blocks_override
if requested is None:
return max(1, local_capacity)
if type(requested) is not int or not 0 < requested <= local_capacity:
raise ValueError(
f"num_cpu_blocks={requested!r} must be between 1 and "
f"local offload capacity {local_capacity}"
)
return requested
def _init_disk_mode(
self,
unique_gpu_caches: dict[str, torch.Tensor],
total_bytes_per_block: int,
device: torch.device,
) -> None:
num_disk_slots = max(1, self.disk_capacity_bytes // total_bytes_per_block)
num_disk_slots = self._resolve_num_cpu_blocks(
self.disk_capacity_bytes, total_bytes_per_block
)
self.num_cpu_blocks = num_disk_slots
logger.info(
+5 -1
View File
@@ -58,6 +58,7 @@ class GPUWatermarkSampler(Sampler):
pos: torch.Tensor,
top_k: torch.Tensor | None,
top_p: torch.Tensor | None,
has_topp_only_rows: bool,
use_fused_sampler: bool,
) -> tuple[torch.Tensor, torch.Tensor]:
enabled = self.watermarking.np[idx_mapping_np] & (
@@ -71,10 +72,13 @@ class GPUWatermarkSampler(Sampler):
pos,
top_k,
top_p,
has_topp_only_rows,
use_fused_sampler,
)
processed_logits = apply_top_k_top_p(processed_logits, top_k, top_p)
processed_logits = apply_top_k_top_p(
processed_logits, top_k, top_p, has_topp_only_rows=has_topp_only_rows
)
contexts = self._get_contexts(expanded_idx_mapping)
repeated_contexts = None
if self.deduplicate_contexts != "none":
+9 -13
View File
@@ -1036,26 +1036,22 @@ def _extrapolate_full_graph_memory(mem_samples: list[int], total_graphs: int) ->
def _init_minimal_kv_cache_for_profiling(runner: "GPUModelRunner") -> None:
"""Allocate the smallest KV cache that still lets every graph be captured."""
from vllm.v1.core.kv_cache_planning import (
get_kv_cache_config_from_groups,
get_kv_cache_groups,
)
from vllm.v1.core.kv_cache_config_builder import get_kv_cache_config_builder
from vllm.v1.core.kv_cache_planning import get_kv_cache_groups_from_builder
kv_cache_spec = runner.get_kv_cache_spec()
kv_cache_groups = get_kv_cache_groups(runner.vllm_config, kv_cache_spec)
builder = get_kv_cache_config_builder(runner.vllm_config)
kv_cache_groups = get_kv_cache_groups_from_builder(
builder, runner.vllm_config, kv_cache_spec
)
# At least one block per sequence is required to capture the graphs.
min_blocks = (
min(runner.max_num_reqs, runner.compilation_config.max_cudagraph_capture_size)
or 1
)
saved_override = runner.cache_config.num_gpu_blocks_override
runner.cache_config.num_gpu_blocks_override = min_blocks
try:
minimal_config = get_kv_cache_config_from_groups(
runner.vllm_config, kv_cache_groups, available_memory=0
)
finally:
runner.cache_config.num_gpu_blocks_override = saved_override
minimal_config = builder.get_kv_cache_config_from_groups(
runner.vllm_config, kv_cache_groups, num_blocks=min_blocks
)
runner.initialize_kv_cache(minimal_config, is_profiling=True)
runner.cache_config.num_gpu_blocks = minimal_config.num_blocks
+7 -3
View File
@@ -298,8 +298,8 @@ class Sampler:
seq_lens_upper_bound_np,
skip_top_k_top_p=True,
)
top_k, top_p = self.sampling_states.get_top_k_top_p(
expanded_idx_mapping, idx_mapping_np
top_k, top_p, has_topp_only_rows = self.sampling_states.get_top_k_top_p(
expanded_idx_mapping, idx_mapping_np, logits.shape[-1]
)
# Don't use a fused sampler if no requests use top_k/top_p, if there are
# any greedy requests or per-request seeds, or if post-processed
@@ -321,6 +321,7 @@ class Sampler:
pos,
top_k,
top_p,
has_topp_only_rows,
use_fused_sampler,
)
@@ -332,6 +333,7 @@ class Sampler:
pos: torch.Tensor,
top_k: torch.Tensor | None,
top_p: torch.Tensor | None,
has_topp_only_rows: bool,
use_fused_sampler: bool,
) -> tuple[torch.Tensor, torch.Tensor]:
if use_fused_sampler:
@@ -342,7 +344,9 @@ class Sampler:
else: # Use XPU sampler
sampled, _ = xpu_sample(processed_logits, top_k, top_p)
else:
processed_logits = apply_top_k_top_p(processed_logits, top_k, top_p)
processed_logits = apply_top_k_top_p(
processed_logits, top_k, top_p, has_topp_only_rows=has_topp_only_rows
)
sampled = gumbel_sample(
processed_logits,
expanded_idx_mapping,

Some files were not shown because too many files have changed in this diff Show More