Merge branch 'main' into patches
This commit is contained in:
+160
-33
@@ -21,7 +21,7 @@ from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from aiter import ActivationType, QuantType
|
||||
from aiter import ActivationType
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -1492,6 +1492,74 @@ def _rocm_aiter_rmsnorm_fp8_group_quant_fake(
|
||||
)
|
||||
|
||||
|
||||
def _rocm_aiter_fused_rms_fp8_per_tensor_static_quant_impl(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
epsilon: float,
|
||||
scale: torch.Tensor,
|
||||
quant_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
from aiter.ops.triton.fused_fp8_quant import fused_rms_fp8_per_tensor_static_quant
|
||||
|
||||
x_quant, _, _, _ = fused_rms_fp8_per_tensor_static_quant(
|
||||
x,
|
||||
weight,
|
||||
epsilon,
|
||||
scale,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
dtype_quant=quant_dtype,
|
||||
res1=None,
|
||||
)
|
||||
return x_quant
|
||||
|
||||
|
||||
def _rocm_aiter_fused_rms_fp8_per_tensor_static_quant_fake(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
epsilon: float,
|
||||
scale: torch.Tensor,
|
||||
quant_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty_like(x, dtype=quant_dtype)
|
||||
|
||||
|
||||
def _rocm_aiter_fused_add_rms_fp8_per_tensor_static_quant_impl(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
epsilon: float,
|
||||
scale: torch.Tensor,
|
||||
quant_dtype: torch.dtype,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
from aiter.ops.triton.fused_fp8_quant import fused_rms_fp8_per_tensor_static_quant
|
||||
|
||||
x_quant, _, _, residual_out = fused_rms_fp8_per_tensor_static_quant(
|
||||
x,
|
||||
weight,
|
||||
epsilon,
|
||||
scale,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
dtype_quant=quant_dtype,
|
||||
res1=residual,
|
||||
)
|
||||
return x_quant, residual_out
|
||||
|
||||
|
||||
def _rocm_aiter_fused_add_rms_fp8_per_tensor_static_quant_fake(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
epsilon: float,
|
||||
scale: torch.Tensor,
|
||||
quant_dtype: torch.dtype,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return torch.empty_like(x, dtype=quant_dtype), torch.empty_like(residual)
|
||||
|
||||
|
||||
def _rocm_aiter_fused_rms_gated_fp8_group_quant_impl(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
@@ -2202,38 +2270,6 @@ class rocm_aiter_ops:
|
||||
}
|
||||
return mapping.get(name)
|
||||
|
||||
@staticmethod
|
||||
def get_aiter_quant_type(quant_type_str: str) -> "QuantType | None":
|
||||
"""Given a quantization type as a string, returns the corresponding aiter QuantType enum.
|
||||
Supported quantization types: "no", "per_tensor", "per_token", "per_1x32", "per_1x128", "per_128x128".
|
||||
Returns None if the mapping fails.
|
||||
|
||||
Args:
|
||||
quant_type_str (str): Quantization type as string.
|
||||
|
||||
Returns:
|
||||
Aiter QuantType enum value, or None if not found.
|
||||
|
||||
"""
|
||||
try:
|
||||
from aiter import QuantType
|
||||
except ImportError:
|
||||
return None
|
||||
|
||||
if not isinstance(quant_type_str, str):
|
||||
return None
|
||||
|
||||
name = quant_type_str.strip().lower()
|
||||
mapping = {
|
||||
"no": QuantType.No,
|
||||
"per_tensor": QuantType.per_Tensor,
|
||||
"per_token": QuantType.per_Token,
|
||||
"per_1x32": QuantType.per_1x32,
|
||||
"per_1x128": QuantType.per_1x128,
|
||||
"per_128x128": QuantType.per_128x128,
|
||||
}
|
||||
return mapping.get(name)
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
def is_enabled(cls) -> bool:
|
||||
@@ -2570,6 +2606,27 @@ class rocm_aiter_ops:
|
||||
|
||||
return cls.is_linear_enabled() and on_gfx950()
|
||||
|
||||
@staticmethod
|
||||
@functools.cache
|
||||
def _kda_gluon_prefill_importable() -> bool:
|
||||
try:
|
||||
from aiter.ops.triton.attention.chunk_kda import chunk_kda # noqa: F401
|
||||
|
||||
return True
|
||||
except ImportError:
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
def is_kda_gluon_prefill_available(cls) -> bool:
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
|
||||
return (
|
||||
cls._AITER_ENABLED
|
||||
and get_cdna_version() >= 4
|
||||
and cls._kda_gluon_prefill_importable()
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_aiter_allreduce(cls):
|
||||
"""Return the TP device communicator's AITER custom-allreduce if it has
|
||||
@@ -3082,6 +3139,18 @@ class rocm_aiter_ops:
|
||||
fake_impl=_rocm_aiter_rmsnorm_fp8_group_quant_fake,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="rocm_aiter_fused_rms_fp8_per_tensor_static_quant",
|
||||
op_func=_rocm_aiter_fused_rms_fp8_per_tensor_static_quant_impl,
|
||||
fake_impl=_rocm_aiter_fused_rms_fp8_per_tensor_static_quant_fake,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="rocm_aiter_fused_add_rms_fp8_per_tensor_static_quant",
|
||||
op_func=_rocm_aiter_fused_add_rms_fp8_per_tensor_static_quant_impl,
|
||||
fake_impl=_rocm_aiter_fused_add_rms_fp8_per_tensor_static_quant_fake,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="rocm_aiter_fused_rms_gated_fp8_group_quant",
|
||||
op_func=_rocm_aiter_fused_rms_gated_fp8_group_quant_impl,
|
||||
@@ -3211,6 +3280,16 @@ class rocm_aiter_ops:
|
||||
def get_rmsnorm_group_fused_quant_op() -> OpOverload:
|
||||
return torch.ops.vllm.rocm_aiter_rmsnorm_fp8_group_quant.default
|
||||
|
||||
@staticmethod
|
||||
def get_fused_rms_fp8_per_tensor_static_quant_op() -> OpOverload:
|
||||
return torch.ops.vllm.rocm_aiter_fused_rms_fp8_per_tensor_static_quant.default
|
||||
|
||||
@staticmethod
|
||||
def get_fused_add_rms_fp8_per_tensor_static_quant_op() -> OpOverload:
|
||||
return (
|
||||
torch.ops.vllm.rocm_aiter_fused_add_rms_fp8_per_tensor_static_quant.default
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_fused_rms_gated_fp8_group_quant_op() -> OpOverload:
|
||||
"""Return the fused gated-RMSNorm + FP8 group quant custom op."""
|
||||
@@ -3224,6 +3303,10 @@ class rocm_aiter_ops:
|
||||
def get_per_token_quant_op() -> OpOverload:
|
||||
return torch.ops.vllm.rocm_aiter_per_token_quant.default
|
||||
|
||||
@staticmethod
|
||||
def get_per_tensor_quant_op() -> OpOverload:
|
||||
return torch.ops.vllm.rocm_aiter_per_tensor_quant.default
|
||||
|
||||
@staticmethod
|
||||
def get_group_quant_op() -> OpOverload:
|
||||
return torch.ops.vllm.rocm_aiter_group_fp8_quant.default
|
||||
@@ -5049,6 +5132,50 @@ class rocm_aiter_ops:
|
||||
layer_input.view(*outer_shape, hidden_size),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def kda_chunk_prefill(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
A_log: torch.Tensor,
|
||||
dt_bias: torch.Tensor | None,
|
||||
lower_bound: float | None,
|
||||
cu_seqlens: torch.Tensor | None,
|
||||
chunk_indices: torch.Tensor | None,
|
||||
chunk_offsets: torch.Tensor | None,
|
||||
scale: float,
|
||||
out: torch.Tensor | None,
|
||||
initial_state: torch.Tensor | None,
|
||||
output_final_state: bool,
|
||||
state_cache: torch.Tensor | None,
|
||||
state_indices: torch.Tensor | None,
|
||||
has_initial_state: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
from aiter.ops.triton.attention.chunk_kda import chunk_kda
|
||||
|
||||
return chunk_kda(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g,
|
||||
beta=beta,
|
||||
A_log=A_log,
|
||||
dt_bias=dt_bias,
|
||||
lower_bound=lower_bound,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
chunk_offsets=chunk_offsets,
|
||||
scale=scale,
|
||||
out=out,
|
||||
initial_state=initial_state,
|
||||
output_final_state=output_final_state,
|
||||
state_cache=state_cache,
|
||||
state_indices=state_indices,
|
||||
has_initial_state=has_initial_state,
|
||||
)
|
||||
|
||||
|
||||
rocm_aiter_ops.register_ops_once()
|
||||
_sync_aiter_situv2_moe_env()
|
||||
|
||||
@@ -2424,6 +2424,11 @@ def grouped_topk(
|
||||
raise NotImplementedError(
|
||||
"The fused grouped_topk kernel is only available on CUDA and XPU platforms"
|
||||
)
|
||||
if scores.shape[0] == 0:
|
||||
return (
|
||||
scores.new_empty((0, topk), dtype=torch.float32),
|
||||
scores.new_empty((0, topk), dtype=torch.int32),
|
||||
)
|
||||
return torch.ops._moe_C.grouped_topk(
|
||||
scores,
|
||||
num_expert_group,
|
||||
|
||||
@@ -120,6 +120,14 @@ def _make_engine_args(args: argparse.Namespace, cases: list[BenchCase]):
|
||||
|
||||
|
||||
def run_case(runner, case: BenchCase, args: argparse.Namespace) -> dict[str, Any]:
|
||||
required_drafts = max((r.num_draft_tokens for r in case.requests), default=0)
|
||||
if required_drafts > runner.num_speculative_steps:
|
||||
raise ValueError(
|
||||
f"Case {case.name!r} needs {required_drafts} draft tokens per request, "
|
||||
f"but the runner supports {runner.num_speculative_steps}. "
|
||||
"Set --speculative-config with a sufficient num_speculative_tokens."
|
||||
)
|
||||
|
||||
from vllm.benchmarks.forward.batch import build_batch
|
||||
from vllm.benchmarks.forward.capture import ForwardBench
|
||||
from vllm.benchmarks.forward.kv_init import touched_kv_views, zero_kv_blocks
|
||||
|
||||
@@ -394,10 +394,13 @@ class MatcherQuantFP8(MatcherCustomOp):
|
||||
self.is_tma_aligned = is_tma_aligned
|
||||
|
||||
if match_rocm_aiter:
|
||||
assert not quant_key.scale.group_shape.is_per_tensor(), (
|
||||
"ROCm aiter fusion pass does not support per tensor quantization"
|
||||
)
|
||||
if quant_key.scale.group_shape.is_per_token():
|
||||
if quant_key.scale.group_shape.is_per_tensor():
|
||||
assert quant_key.scale.static, (
|
||||
"ROCm aiter fusion pass only supports static per-tensor "
|
||||
"quantization"
|
||||
)
|
||||
self.QUANT_OP = rocm_aiter_ops.get_per_tensor_quant_op()
|
||||
elif quant_key.scale.group_shape.is_per_token():
|
||||
self.QUANT_OP = rocm_aiter_ops.get_per_token_quant_op()
|
||||
else:
|
||||
assert quant_key.scale.group_shape.col == 128, (
|
||||
@@ -432,14 +435,24 @@ class MatcherQuantFP8(MatcherCustomOp):
|
||||
scale: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
quant_key_group_shape = self.quant_key.scale.group_shape
|
||||
if quant_key_group_shape == GroupShape.PER_TENSOR:
|
||||
# rocm_aiter_per_tensor_quant mutates `out` and `scale` and returns
|
||||
# None, so it has to be matched through auto_functionalized.
|
||||
assert scale is not None
|
||||
out = torch.empty_like(input, dtype=self.quant_key.dtype)
|
||||
result = auto_functionalized(
|
||||
self.QUANT_OP, out=out, x=input, scale=scale, is_dynamic=False
|
||||
)
|
||||
# `scale` is mutable, so its functionalized output is a distinct
|
||||
# value; return it to keep the quant node inside the match.
|
||||
return result[1], result[2]
|
||||
if quant_key_group_shape == GroupShape.PER_TOKEN:
|
||||
return self.QUANT_OP( # type: ignore[no-any-return]
|
||||
x=input,
|
||||
quant_dtype=self.quant_key.dtype,
|
||||
scale=scale,
|
||||
)
|
||||
else:
|
||||
return self.QUANT_OP(input, quant_key_group_shape.col) # type: ignore[no-any-return]
|
||||
return self.QUANT_OP(input, quant_key_group_shape.col) # type: ignore[no-any-return]
|
||||
|
||||
def forward_custom(
|
||||
self,
|
||||
|
||||
@@ -19,6 +19,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
ScaleDesc,
|
||||
kFp8Dynamic128Sym,
|
||||
kStaticTensorScale,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -561,6 +562,140 @@ class AiterRMSNormGatedFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
)
|
||||
|
||||
|
||||
class AiterRMSNormStaticQuantPattern(AiterRMSNormQuantPattern):
|
||||
"""This pattern fuses rms_norm & aiter static per-tensor fp8 quant
|
||||
into an aiter fused_rms_fp8_per_tensor_static_quant op.
|
||||
"""
|
||||
|
||||
FUSED_OP = rocm_aiter_ops.get_fused_rms_fp8_per_tensor_static_quant_op()
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
epsilon: float,
|
||||
quant_dtype: torch.dtype,
|
||||
symmetric: bool = True,
|
||||
) -> None:
|
||||
key = FusedRMSQuantKey(
|
||||
fused_add=False,
|
||||
quant=QuantKey(
|
||||
dtype=quant_dtype, scale=kStaticTensorScale, symmetric=symmetric
|
||||
),
|
||||
)
|
||||
|
||||
super().__init__(epsilon, key)
|
||||
|
||||
def register(self, pm_pass: PatternMatcherPass) -> None:
|
||||
def pattern(
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
result_rms = torch.ops.vllm_ir.rms_norm(input, weight, self.epsilon)
|
||||
result, scale = self.quant_matcher(result_rms, scale)
|
||||
|
||||
return result, scale
|
||||
|
||||
def replacement(
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
result = self.FUSED_OP(
|
||||
x=input,
|
||||
weight=weight,
|
||||
epsilon=self.epsilon,
|
||||
scale=scale,
|
||||
quant_dtype=self.quant_dtype,
|
||||
)
|
||||
|
||||
# The static scale is an input, so it passes through unchanged.
|
||||
return result, scale
|
||||
|
||||
inputs = [
|
||||
self.empty(5, 16), # input
|
||||
self.empty(16), # weight
|
||||
self.empty_f32(1, 1), # scale
|
||||
]
|
||||
|
||||
pm.register_replacement(
|
||||
pattern,
|
||||
replacement,
|
||||
inputs,
|
||||
pm.fwd_only,
|
||||
pm_pass,
|
||||
)
|
||||
|
||||
|
||||
class AiterFusedAddRMSNormStaticQuantPattern(AiterRMSNormQuantPattern):
|
||||
"""This pattern fuses fused_add_rms_norm & aiter static per-tensor fp8
|
||||
quant into an aiter fused_add_rms_fp8_per_tensor_static_quant op.
|
||||
"""
|
||||
|
||||
FUSED_OP = rocm_aiter_ops.get_fused_add_rms_fp8_per_tensor_static_quant_op()
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
epsilon: float,
|
||||
quant_dtype: torch.dtype,
|
||||
symmetric: bool = True,
|
||||
) -> None:
|
||||
key = FusedRMSQuantKey(
|
||||
fused_add=True,
|
||||
quant=QuantKey(
|
||||
dtype=quant_dtype, scale=kStaticTensorScale, symmetric=symmetric
|
||||
),
|
||||
)
|
||||
|
||||
super().__init__(epsilon, key)
|
||||
|
||||
def register(self, pm_pass: PatternMatcherPass) -> None:
|
||||
def pattern(
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
result_rms, residual_out = torch.ops.vllm_ir.fused_add_rms_norm(
|
||||
input, residual, weight, self.epsilon
|
||||
)
|
||||
result, scale = self.quant_matcher(result_rms, scale)
|
||||
|
||||
return result, residual_out, scale
|
||||
|
||||
def replacement(
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
result = self.FUSED_OP(
|
||||
x=input,
|
||||
residual=residual,
|
||||
weight=weight,
|
||||
epsilon=self.epsilon,
|
||||
scale=scale,
|
||||
quant_dtype=self.quant_dtype,
|
||||
)
|
||||
|
||||
# The static scale is an input, so it passes through unchanged.
|
||||
return result[0], result[1], scale
|
||||
|
||||
inputs = [
|
||||
self.empty(5, 16), # input
|
||||
self.empty(16), # weight
|
||||
self.empty(5, 16), # residual
|
||||
self.empty_f32(1, 1), # scale
|
||||
]
|
||||
|
||||
pm.register_replacement(
|
||||
pattern,
|
||||
replacement,
|
||||
inputs,
|
||||
pm.fwd_only,
|
||||
pm_pass,
|
||||
)
|
||||
|
||||
|
||||
class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
|
||||
"""This pass fuses aiter rms_norm & vllm/aiter quant custom ops
|
||||
into a fused rms_norm_quant op.
|
||||
@@ -630,6 +765,14 @@ class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
|
||||
epsilon, FP8_DTYPE, GroupShape(1, 128), match_aiter_quant_op
|
||||
).register(self.patterns)
|
||||
|
||||
# Fuse aiter rms_norm + aiter static per-tensor fp8 quant
|
||||
AiterRMSNormStaticQuantPattern(epsilon, FP8_DTYPE).register(self.patterns)
|
||||
|
||||
# Fuse aiter fused_add_rms_norm + aiter static per-tensor fp8 quant
|
||||
AiterFusedAddRMSNormStaticQuantPattern(epsilon, FP8_DTYPE).register(
|
||||
self.patterns
|
||||
)
|
||||
|
||||
# When quant_fp8 custom ops are disabled, both AITER and native
|
||||
# quant matchers trace through QuantFP8's native implementation.
|
||||
# Registering both variants would create duplicate Inductor
|
||||
@@ -690,6 +833,8 @@ class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
|
||||
AiterFusedAddRMSNormDynamicQuantPattern,
|
||||
AiterRMSFp8GroupQuantPattern,
|
||||
AiterFusedAddRMSFp8GroupQuantPattern,
|
||||
AiterRMSNormStaticQuantPattern,
|
||||
AiterFusedAddRMSNormStaticQuantPattern,
|
||||
DoubleAiterRMSFp8GroupQuantPattern,
|
||||
DoubleAiterRMSFp8GroupQuantViewPattern,
|
||||
AiterRMSNormGatedFp8GroupQuantPattern,
|
||||
|
||||
@@ -85,6 +85,11 @@ class AttentionConfig:
|
||||
use_prefill_query_quantization: bool = False
|
||||
"""If set, quantize query for attention in prefill."""
|
||||
|
||||
trtllm_ragged_backend: Literal["trtllm-gen", "cute-dsl"] = "trtllm-gen"
|
||||
"""Kernel backend for cached-context TRTLLM_RAGGED MLA prefill.
|
||||
CuTe DSL requires FP8 queries for head dimension 192 and does not support
|
||||
head dimension 256. New-token prefill always uses TRTLLM."""
|
||||
|
||||
indexer_kv_dtype: IndexerKVDType = "auto"
|
||||
"""Data type for the sparse-attention indexer K cache. "auto" picks the
|
||||
model's default (bf16 for MiniMax M3, fp8 for the DeepSeek sparse
|
||||
|
||||
@@ -2034,6 +2034,10 @@ class ModelConfig:
|
||||
def has_noops(self) -> bool:
|
||||
return self._model_info.has_noops
|
||||
|
||||
@property
|
||||
def supports_kv_cache_grouping(self) -> bool:
|
||||
return self._model_info.supports_kv_cache_grouping
|
||||
|
||||
@property
|
||||
def has_inner_state(self):
|
||||
return self._model_info.has_inner_state
|
||||
|
||||
@@ -838,7 +838,7 @@ class ExternalElasticEPScaleCoordinator:
|
||||
if prepared.dp_rank == 0:
|
||||
self.client._coord_store = self.reconfig_store_ref
|
||||
self.client._ensure_stats_update_task()
|
||||
await self.client.first_req_send_socket.send(
|
||||
await self.client.scale_send_socket.send(
|
||||
msgspec.msgpack.encode(
|
||||
("SCALE_ELASTIC_EP", bootstrap.new_data_parallel_size)
|
||||
)
|
||||
|
||||
@@ -63,12 +63,13 @@ class WriteTask:
|
||||
request_id: ReqId
|
||||
transfer_id: TransferId
|
||||
dst_engine_id: str
|
||||
local_block_ids: list[int]
|
||||
remote_block_ids_hint: list[int] | None
|
||||
local_block_ids: BlockIds
|
||||
remote_block_ids_hint: BlockIds | None
|
||||
layer_name: str
|
||||
event: torch.cuda.Event
|
||||
remote_notify_port: int
|
||||
remote_ip: str
|
||||
remote_tp_size: int | None = None
|
||||
multi_pod_hosts: list[str] = field(default_factory=list)
|
||||
remote_dp_size_local: int = 0
|
||||
remote_dp_size: int = 1
|
||||
@@ -94,7 +95,7 @@ class LayerTransferPlan:
|
||||
class RemoteAllocInfo:
|
||||
"""Information about remote block allocation."""
|
||||
|
||||
block_ids: list[int]
|
||||
block_ids: BlockIds | None
|
||||
writes_done: int = 0
|
||||
writes_expected: int | None = None
|
||||
decode_dp_rank: int = 0
|
||||
|
||||
@@ -338,7 +338,10 @@ class MoRIIOConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
block_ids: BlockIds,
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
assert self.connector_scheduler is not None
|
||||
if self.connector_scheduler._has_mamba:
|
||||
if (
|
||||
self.connector_scheduler._has_mamba
|
||||
and self.connector_scheduler.mode == MoRIIOMode.READ
|
||||
):
|
||||
attn_block_ids, mamba_block_groups = (
|
||||
self.connector_scheduler.split_block_groups(block_ids)
|
||||
)
|
||||
@@ -420,6 +423,16 @@ class MoRIIOConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
)
|
||||
self.connector_worker.wait_for_save(self._connector_metadata)
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
return (
|
||||
self.mode == MoRIIOMode.WRITE
|
||||
and self.connector_scheduler is not None
|
||||
and bool(self.connector_scheduler._deferred_send_deadlines)
|
||||
)
|
||||
|
||||
def get_loaded_kv_cache_group_ids(self, request: "Request") -> tuple[int, ...]:
|
||||
return self._kv_cache_config.transfer_group_ids
|
||||
|
||||
def shutdown(self):
|
||||
if self.connector_worker is not None:
|
||||
self.connector_worker.shutdown()
|
||||
@@ -463,11 +476,10 @@ def _validate_hybrid_speculation(vllm_config: VllmConfig) -> None:
|
||||
speculative_config = vllm_config.speculative_config
|
||||
if speculative_config is None:
|
||||
return
|
||||
# DSpark is allowlisted because it is the only hybrid speculative method
|
||||
# validated end to end with MoRIIO READ.
|
||||
# Other hybrid speculative methods need their own transfer validation.
|
||||
if not speculative_config.use_dspark():
|
||||
raise MoRIIOError(
|
||||
"MoRIIO hybrid READ supports DSpark speculative decoding only, got "
|
||||
"MoRIIO hybrid transfer supports DSpark speculative decoding only, got "
|
||||
f"method={speculative_config.method!r}"
|
||||
)
|
||||
|
||||
@@ -480,14 +492,14 @@ def _validate_mamba_specs(specs: Collection[MambaSpec]) -> MambaSpec | None:
|
||||
for spec in specs:
|
||||
if len(spec.shapes) != 2 or len(spec.dtypes) != 2:
|
||||
raise MoRIIOError(
|
||||
"MoRIIO hybrid READ supports exactly two Mamba states "
|
||||
"MoRIIO hybrid transfer supports exactly two Mamba states "
|
||||
"(conv and SSM), got "
|
||||
f"{len(spec.shapes)} shapes and {len(spec.dtypes)} dtypes"
|
||||
)
|
||||
reference = specs[0]
|
||||
if any(spec != reference for spec in specs[1:]):
|
||||
raise MoRIIOError(
|
||||
"MoRIIO hybrid READ requires all transferable Mamba groups "
|
||||
"MoRIIO hybrid transfer requires all transferable Mamba groups "
|
||||
"to use the same cache spec"
|
||||
)
|
||||
return reference
|
||||
@@ -515,7 +527,9 @@ class MoRIIOConnectorScheduler:
|
||||
self._has_mamba = bool(self._mamba_group_ids)
|
||||
mamba_spec = None
|
||||
if self._has_mamba:
|
||||
if len(self._attn_group_ids) != 1:
|
||||
if not self._attn_group_ids:
|
||||
raise MoRIIOError("MoRIIO hybrid transfer requires an attention group")
|
||||
if self.mode == MoRIIOMode.READ and len(self._attn_group_ids) != 1:
|
||||
raise MoRIIOError(
|
||||
"MoRIIO hybrid READ requires exactly one transferable "
|
||||
"attention group, got "
|
||||
@@ -529,11 +543,6 @@ class MoRIIOConnectorScheduler:
|
||||
]
|
||||
mamba_spec = _validate_mamba_specs(mamba_specs)
|
||||
_validate_hybrid_speculation(vllm_config)
|
||||
if self.mode != MoRIIOMode.READ:
|
||||
raise MoRIIOError(
|
||||
"MoRIIO hybrid (mamba/KDA) transfer is implemented for READ "
|
||||
"mode only; set kv_connector_extra_config.read_mode=true"
|
||||
)
|
||||
self._num_ssm_scratch_blocks = (
|
||||
mamba_spec.num_speculative_blocks if mamba_spec is not None else 0
|
||||
)
|
||||
@@ -581,11 +590,13 @@ class MoRIIOConnectorScheduler:
|
||||
f"cache group spec(s): {sorted(set(unsupported))}. Pass "
|
||||
"--disable-hybrid-kv-cache-manager to run without HMA."
|
||||
)
|
||||
if self.mode == MoRIIOMode.WRITE:
|
||||
# TODO(simondanielsson): support HMA in WRITE mode
|
||||
if self.mode == MoRIIOMode.WRITE and any(
|
||||
isinstance(g.kv_cache_spec, SlidingWindowSpec)
|
||||
for g in kv_cache_config.transfer_groups
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"MoRIIO WRITE mode does not support hybrid KV cache groups "
|
||||
"(sliding-window attention). Use READ mode, or pass "
|
||||
"MoRIIO WRITE mode does not support sliding-window "
|
||||
"cache groups. Use READ mode, or pass "
|
||||
"--disable-hybrid-kv-cache-manager."
|
||||
)
|
||||
|
||||
@@ -784,10 +795,11 @@ class MoRIIOConnectorScheduler:
|
||||
if not params.get("do_remote_prefill"):
|
||||
return 0, False
|
||||
if self.mode == MoRIIOMode.WRITE:
|
||||
# MoriiO in write mode, no remote prefill. Hybrid models never get
|
||||
# here: register_kv_caches refuses WRITE mode for them, so there is
|
||||
# no recurrent-state accounting to do.
|
||||
return num_external_tokens, True
|
||||
if self._has_mamba:
|
||||
# P pushes h(N-1). Do not publish it under the N-token hash
|
||||
# when the scheduler caches a completed asynchronous load.
|
||||
num_external_tokens = max(num_external_tokens - 1, 0)
|
||||
return num_external_tokens, num_external_tokens > 0
|
||||
|
||||
# READ mode always recomputes the last token locally on the decoder.
|
||||
#
|
||||
@@ -1178,6 +1190,16 @@ class MoRIIOConnectorScheduler:
|
||||
f"remote_notify_port={remote_notify_port!r})"
|
||||
)
|
||||
|
||||
block_notify_list: list[int] | list[list[int]]
|
||||
if self._has_mamba:
|
||||
block_notify_list = list(
|
||||
self._clamp_to_prompt_blocks(
|
||||
request, blocks.get_block_ids()
|
||||
)
|
||||
)
|
||||
else:
|
||||
block_notify_list = blocks.get_block_ids()[0]
|
||||
|
||||
# Wide-EP multi-pod: a pod binds notify sockets only for
|
||||
# its LOCAL ranks, so the port offset must use the per-pod
|
||||
# local rank (% dp_local), not the global rank. Single-pod
|
||||
@@ -1204,7 +1226,7 @@ class MoRIIOConnectorScheduler:
|
||||
self.send_notify_block(
|
||||
req_id=request.request_id,
|
||||
transfer_id=request.kv_transfer_params["transfer_id"],
|
||||
block_notify_list=blocks.get_block_ids()[0],
|
||||
block_notify_list=block_notify_list,
|
||||
host=_notify_host,
|
||||
port=target_port,
|
||||
)
|
||||
@@ -1227,6 +1249,26 @@ class MoRIIOConnectorScheduler:
|
||||
sliding-window groups are left unchanged.
|
||||
|
||||
"""
|
||||
if self._has_mamba:
|
||||
# WRITE preserves transfer-group order, including a separate drafter
|
||||
# attention group. READ retains its attention-first wire format.
|
||||
transfer_blocks = self.kv_cache_config.select_transfer_block_ids(block_ids)
|
||||
dcp_size = self.vllm_config.parallel_config.decode_context_parallel_size
|
||||
clamped_hybrid: list[list[int]] = []
|
||||
for group, blocks in zip(
|
||||
self.kv_cache_config.transfer_groups, transfer_blocks, strict=True
|
||||
):
|
||||
spec = group.kv_cache_spec
|
||||
if isinstance(spec, MambaSpec):
|
||||
clamped_hybrid.append(
|
||||
clip_ssm_state_blocks(list(blocks), spec.num_speculative_blocks)
|
||||
)
|
||||
else:
|
||||
block_size = spec.block_size * (dcp_size if spec.dcp_sharded else 1)
|
||||
clamped_hybrid.append(
|
||||
list(blocks[: cdiv(req.num_prompt_tokens, block_size)])
|
||||
)
|
||||
return clamped_hybrid
|
||||
clamped = list(block_ids)
|
||||
changed = False
|
||||
for g, group_blocks in enumerate(block_ids):
|
||||
@@ -1252,6 +1294,19 @@ class MoRIIOConnectorScheduler:
|
||||
num_scheduled = scheduler_output.num_scheduled_tokens.get(req.request_id, 0)
|
||||
return req.num_computed_tokens + num_scheduled >= req.num_prompt_tokens
|
||||
|
||||
def _get_write_blocks(
|
||||
self, req: "Request", block_ids: BlockIds, scheduler_output: SchedulerOutput
|
||||
) -> BlockIds:
|
||||
if self._has_mamba:
|
||||
# Align-mode state slots can move without appending a new block.
|
||||
# Resolve the authoritative table at the final prefill step.
|
||||
block_state = scheduler_output.kv_connector_block_state
|
||||
assert block_state is not None
|
||||
current_blocks = block_state.get_block_ids(req.request_id)
|
||||
assert current_blocks is not None
|
||||
block_ids = current_blocks
|
||||
return self._clamp_to_prompt_blocks(req, block_ids)
|
||||
|
||||
def build_connector_meta(
|
||||
self,
|
||||
scheduler_output: SchedulerOutput,
|
||||
@@ -1274,10 +1329,10 @@ class MoRIIOConnectorScheduler:
|
||||
# reserved speculative-lookahead slack), so new_block_ids is None;
|
||||
# still fall through to the final-chunk check below.
|
||||
new_block_ids = scheduler_output.scheduled_cached_reqs.new_block_ids[i]
|
||||
if new_block_ids is not None:
|
||||
if new_block_ids is not None and not self._has_mamba:
|
||||
existing_blocks = [
|
||||
existing_blocks[g] + new_block_ids[g]
|
||||
for g in range(len(new_block_ids))
|
||||
old + new
|
||||
for old, new in zip(existing_blocks, new_block_ids, strict=True)
|
||||
]
|
||||
self._reqs_need_pending_save[req_id] = (req, existing_blocks)
|
||||
if not self._is_final_prefill_chunk(req, scheduler_output):
|
||||
@@ -1287,7 +1342,9 @@ class MoRIIOConnectorScheduler:
|
||||
kv_params = self._req_kv_params.pop(
|
||||
req_id, req.kv_transfer_params or {}
|
||||
)
|
||||
save_block_ids = self._clamp_to_prompt_blocks(req, existing_blocks)
|
||||
save_block_ids = self._get_write_blocks(
|
||||
req, existing_blocks, scheduler_output
|
||||
)
|
||||
meta.add_new_req(
|
||||
request_id=req_id,
|
||||
local_block_ids=save_block_ids,
|
||||
@@ -1328,7 +1385,7 @@ class MoRIIOConnectorScheduler:
|
||||
if not self._is_final_prefill_chunk(req, scheduler_output):
|
||||
self._reqs_need_pending_save[req_id] = (req, block_ids)
|
||||
continue
|
||||
save_block_ids = self._clamp_to_prompt_blocks(req, block_ids)
|
||||
save_block_ids = self._get_write_blocks(req, block_ids, scheduler_output)
|
||||
meta.add_new_req(
|
||||
request_id=req_id,
|
||||
local_block_ids=save_block_ids,
|
||||
@@ -1425,7 +1482,7 @@ class MoRIIOConnectorScheduler:
|
||||
"""Once a request is finished, determine whether request blocks
|
||||
should be freed now or will be sent asynchronously and freed later.
|
||||
|
||||
``block_ids`` normally contains all cache groups. Hybrid callers from
|
||||
``block_ids`` normally contains all cache groups. Hybrid READ callers from
|
||||
``request_finished_all_groups`` pass a flat attention list and the
|
||||
recurrent-state groups separately in ``mamba_block_groups``.
|
||||
"""
|
||||
@@ -1494,6 +1551,10 @@ class MoRIIOConnectorScheduler:
|
||||
list(cast(list[int], block_ids)),
|
||||
*[list(group) for group in mamba_block_groups],
|
||||
]
|
||||
elif self._has_mamba and self.mode == MoRIIOMode.WRITE:
|
||||
computed_block_ids = self._clamp_to_prompt_blocks(
|
||||
request, cast(BlockIds, block_ids)
|
||||
)
|
||||
elif self._has_mamba:
|
||||
raise MoRIIOError("Hybrid request completion requires all KV cache groups")
|
||||
else:
|
||||
@@ -1660,7 +1721,7 @@ class MoRIIOConnectorWorker:
|
||||
_, mamba_transfer_group_ids = _split_kv_cache_group_kinds(kv_cache_config)
|
||||
self._num_mamba_transfer_groups = len(mamba_transfer_group_ids)
|
||||
self._mamba_payload_index_by_layer = {
|
||||
layer_name: payload_index
|
||||
layer_name: group_id if self.mode == MoRIIOMode.WRITE else payload_index
|
||||
for payload_index, group_id in enumerate(mamba_transfer_group_ids, start=1)
|
||||
for layer_name in kv_cache_config.transfer_groups[group_id].layer_names
|
||||
}
|
||||
@@ -1884,8 +1945,8 @@ class MoRIIOConnectorWorker:
|
||||
request_id: ReqId,
|
||||
transfer_id: TransferId,
|
||||
dst_engine_id: str,
|
||||
local_block_ids: list[int],
|
||||
remote_block_ids: list[int] | None,
|
||||
local_block_ids: BlockIds,
|
||||
remote_block_ids: BlockIds | None,
|
||||
layer_name: str,
|
||||
kv_layer: torch.Tensor,
|
||||
remote_notify_port: int,
|
||||
@@ -1893,6 +1954,7 @@ class MoRIIOConnectorWorker:
|
||||
multi_pod_hosts: list[str],
|
||||
remote_dp_size_local: int,
|
||||
remote_dp_size: int,
|
||||
remote_tp_size: int | None = None,
|
||||
) -> None:
|
||||
"""Schedule a block write operation.
|
||||
|
||||
@@ -1909,6 +1971,7 @@ class MoRIIOConnectorWorker:
|
||||
multi_pod_hosts: List of pod IPs for multi-pod Wide-EP
|
||||
remote_dp_size_local: Per-pod DP size for multi-pod
|
||||
remote_dp_size: Global DP size of the decode instance
|
||||
remote_tp_size: Tensor parallel size advertised by the peer
|
||||
|
||||
"""
|
||||
# synchronization to prevent dirty reads between
|
||||
@@ -1933,6 +1996,7 @@ class MoRIIOConnectorWorker:
|
||||
multi_pod_hosts=multi_pod_hosts,
|
||||
remote_dp_size_local=remote_dp_size_local,
|
||||
remote_dp_size=remote_dp_size,
|
||||
remote_tp_size=remote_tp_size,
|
||||
)
|
||||
self._writer.schedule_write(task)
|
||||
|
||||
@@ -2359,15 +2423,6 @@ class MoRIIOConnectorWorker:
|
||||
self._has_mamba = any(
|
||||
self._is_mamba_layer(layer_name) for layer_name in kv_caches
|
||||
)
|
||||
if self._has_mamba and self.mode == MoRIIOMode.WRITE:
|
||||
# WRITE needs a distinct conv/ssm transfer plan. Reject before
|
||||
# allocating or registering any regions rather than silently
|
||||
# serving a zero recurrent state.
|
||||
raise MoRIIOError(
|
||||
"MoRIIO hybrid (mamba/KDA) transfer is implemented for READ "
|
||||
"mode only; set kv_connector_extra_config.read_mode=true "
|
||||
"for hybrid models"
|
||||
)
|
||||
if self._has_mamba:
|
||||
mamba_specs = [
|
||||
cast(MambaSpec, self.layer_to_spec[layer_name])
|
||||
@@ -3292,6 +3347,9 @@ class MoRIIOConnectorWorker:
|
||||
)
|
||||
|
||||
def _write_blocks_for_req(self, req_id: ReqId, meta: ReqMeta, layer_name, kv_layer):
|
||||
if self._has_mamba:
|
||||
# Reject before even an attention-layer write can be queued.
|
||||
self._mamba_tp_ratio(int(meta.tp_size))
|
||||
# Compute per-request values to pass through the task (no shared state).
|
||||
# This avoids race conditions when concurrent requests target different
|
||||
# decode pods - each WriteTask carries its own routing info.
|
||||
@@ -3303,16 +3361,12 @@ class MoRIIOConnectorWorker:
|
||||
if meta.remote_dp_size_local
|
||||
else int(meta.remote_dp_size)
|
||||
)
|
||||
# WRITE does not support HMA, so unwrap blocks into flat lists.
|
||||
# remote_block_ids can itself be empty so need to be careful when unwrapping.
|
||||
local_block_ids = meta.local_block_ids[0]
|
||||
remote_block_ids = meta.remote_block_ids[0] if meta.remote_block_ids else []
|
||||
self.schedule_write_blocks(
|
||||
request_id=req_id,
|
||||
transfer_id=meta.transfer_id,
|
||||
dst_engine_id=meta.remote_engine_id,
|
||||
local_block_ids=local_block_ids,
|
||||
remote_block_ids=remote_block_ids,
|
||||
local_block_ids=meta.local_block_ids,
|
||||
remote_block_ids=meta.remote_block_ids,
|
||||
layer_name=layer_name,
|
||||
kv_layer=kv_layer,
|
||||
remote_notify_port=meta.remote_notify_port,
|
||||
@@ -3320,6 +3374,7 @@ class MoRIIOConnectorWorker:
|
||||
multi_pod_hosts=hosts,
|
||||
remote_dp_size_local=dp_local,
|
||||
remote_dp_size=int(meta.remote_dp_size),
|
||||
remote_tp_size=int(meta.tp_size),
|
||||
)
|
||||
|
||||
def merge_contiguous_blocks(
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import threading
|
||||
import time
|
||||
from collections import OrderedDict, defaultdict
|
||||
from math import ceil
|
||||
from queue import Empty, Queue
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from weakref import ref as weakref_ref
|
||||
@@ -61,6 +62,11 @@ try:
|
||||
except ImportError:
|
||||
logger.error("MoRIIO is not available")
|
||||
|
||||
try:
|
||||
from mori.io import StatusCode
|
||||
except ImportError:
|
||||
StatusCode = None
|
||||
|
||||
"""Write task execution logic for MoRIIO connector."""
|
||||
|
||||
|
||||
@@ -372,11 +378,15 @@ class MoRIIOWriter:
|
||||
task.dst_engine_id
|
||||
)
|
||||
|
||||
# Prepare transfer plan
|
||||
plan = self._prepare_transfer_plan(task, request_info, remote_moriio_meta)
|
||||
|
||||
# Execute transfer
|
||||
transfer_statuses = self._do_layer_write(plan, sessions)
|
||||
if self.worker._is_mamba_layer(task.layer_name):
|
||||
plans = self._prepare_mamba_transfer_plans(task, request_info)
|
||||
else:
|
||||
plans = [
|
||||
self._prepare_transfer_plan(task, request_info, remote_moriio_meta)
|
||||
]
|
||||
transfer_statuses = []
|
||||
for plan in plans:
|
||||
transfer_statuses.extend(self._do_layer_write(plan, sessions))
|
||||
with self._write_state_lock:
|
||||
request_info.transfer_statuses.extend(transfer_statuses)
|
||||
|
||||
@@ -404,10 +414,17 @@ class MoRIIOWriter:
|
||||
key = (task.layer_name, *_get_write_geometry_key(layer_cache))
|
||||
offsets = request_info.transfer_offsets.get(key)
|
||||
if offsets is None:
|
||||
group = self.worker.layer_to_group[task.layer_name]
|
||||
assert request_info.block_ids is not None
|
||||
local_blocks = task.local_block_ids[group]
|
||||
remote_blocks = request_info.block_ids[group]
|
||||
if self.worker._has_mamba:
|
||||
# D can reserve a tail block for the token recomputed locally.
|
||||
remote_blocks = remote_blocks[: len(local_blocks)]
|
||||
offsets = self.worker._compute_block_transfer_offsets(
|
||||
task.layer_name,
|
||||
task.local_block_ids,
|
||||
request_info.block_ids,
|
||||
local_blocks,
|
||||
remote_blocks,
|
||||
remote_moriio_meta,
|
||||
remote_engine_id=task.dst_engine_id,
|
||||
)
|
||||
@@ -430,6 +447,44 @@ class MoRIIOWriter:
|
||||
use_batch=True,
|
||||
)
|
||||
|
||||
def _prepare_mamba_transfer_plans(
|
||||
self, task: WriteTask, request_info: RemoteAllocInfo
|
||||
) -> list[LayerTransferPlan]:
|
||||
"""Plan both registered regions as one scheduled layer write."""
|
||||
assert request_info.block_ids is not None
|
||||
local_slots = self.worker._mamba_blocks_for_layer(
|
||||
task.layer_name, task.local_block_ids
|
||||
)
|
||||
remote_slots = self.worker._mamba_blocks_for_layer(
|
||||
task.layer_name, request_info.block_ids
|
||||
)
|
||||
if len(local_slots) != 1 or len(remote_slots) != 1:
|
||||
raise MoRIIOError(
|
||||
"Hybrid WRITE requires one running-state slot per Mamba group"
|
||||
)
|
||||
local, remote, sizes, n_conv = self.worker._compute_mamba_transfer_offsets(
|
||||
task.layer_name,
|
||||
local_slots,
|
||||
remote_slots,
|
||||
remote_tp_size=task.remote_tp_size,
|
||||
)
|
||||
conv_session, ssm_session = self.worker._region_session_indices(task.layer_name)
|
||||
return [
|
||||
LayerTransferPlan(
|
||||
request_id=task.request_id,
|
||||
transfer_id=task.transfer_id,
|
||||
layer_name=task.layer_name,
|
||||
sess_idx=session,
|
||||
transfer_local_offsets=local[region],
|
||||
transfer_remote_offsets=remote[region],
|
||||
transfer_sizes=sizes[region],
|
||||
)
|
||||
for session, region in (
|
||||
(conv_session, slice(0, n_conv)),
|
||||
(ssm_session, slice(n_conv, None)),
|
||||
)
|
||||
]
|
||||
|
||||
def _do_layer_write(self, plan: LayerTransferPlan, sessions: list) -> list[Any]:
|
||||
"""Perform the actual layer write.
|
||||
|
||||
@@ -438,6 +493,8 @@ class MoRIIOWriter:
|
||||
sessions: List of transfer sessions
|
||||
|
||||
"""
|
||||
if not plan.transfer_sizes:
|
||||
return []
|
||||
if plan.use_batch:
|
||||
return [
|
||||
self.worker.moriio_wrapper.write_remote_data(
|
||||
@@ -721,7 +778,28 @@ class MoRIIOWrapper:
|
||||
if not transfers_to_wait:
|
||||
return
|
||||
|
||||
self._poll_transfers_until_done(transfers_to_wait)
|
||||
wait_all = getattr(self.moriio_engine, "wait_all", None)
|
||||
if wait_all is None or StatusCode is None:
|
||||
self._poll_transfers_until_done(transfers_to_wait)
|
||||
return
|
||||
|
||||
# Zero means a nonblocking progress call in Mori, not an immediate
|
||||
# deadline. Round up so even sub-millisecond waits remain blocking.
|
||||
timeout_ms = ceil(self._transfer_timeout * 1000)
|
||||
result = wait_all(transfers_to_wait, timeout_ms=timeout_ms)
|
||||
if result != StatusCode.SUCCESS:
|
||||
details = [
|
||||
f"{status.Message()} (code={status.Code()})"
|
||||
for status in transfers_to_wait
|
||||
if status.Failed()
|
||||
]
|
||||
if not details and result in (StatusCode.INIT, StatusCode.IN_PROGRESS):
|
||||
details.append(
|
||||
f"RDMA transfer timed out after {self._transfer_timeout:g}s"
|
||||
)
|
||||
raise TransferError(
|
||||
f"MoRIIO batch wait failed ({result}): " + "; ".join(details)
|
||||
)
|
||||
|
||||
def _poll_transfers_until_done(self, transfers_to_wait: list[Any]) -> None:
|
||||
"""Fallback for mori builds without the batched wait.
|
||||
@@ -858,7 +936,12 @@ class MoRIIOWrapper:
|
||||
)
|
||||
return
|
||||
self.done_remote_allocate_req_dict[transfer_id] = RemoteAllocInfo(
|
||||
block_ids=block_notify_list, decode_dp_rank=decode_dp_rank
|
||||
block_ids=(
|
||||
[block_notify_list]
|
||||
if isinstance(block_notify_list[0], int)
|
||||
else block_notify_list
|
||||
),
|
||||
decode_dp_rank=decode_dp_rank,
|
||||
)
|
||||
|
||||
def _handle_write_done_message(self, data: dict):
|
||||
|
||||
@@ -125,11 +125,9 @@ class MultiKVConnectorPromMetrics(KVConnectorPromMetrics):
|
||||
|
||||
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
|
||||
for connector_id, stats_data in transfer_stats_data.items():
|
||||
assert connector_id in self._prom_metrics, (
|
||||
f"{connector_id} is not contained in the list of registered connectors "
|
||||
f"with Prometheus metrics support: {self._prom_metrics.keys()}"
|
||||
)
|
||||
self._prom_metrics[connector_id].observe(stats_data, engine_idx)
|
||||
prom_metrics = self._prom_metrics.get(connector_id)
|
||||
if prom_metrics is not None:
|
||||
prom_metrics.observe(stats_data, engine_idx)
|
||||
|
||||
|
||||
class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
|
||||
@@ -101,6 +101,16 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
)
|
||||
cpu_capacity_per_rank = explicit
|
||||
|
||||
# When set, num_cpu_blocks overrides automatic pool sizing from byte budgets
|
||||
# (including cpu_bytes_to_use and cpu_bytes_to_use_per_rank), allowing ranks
|
||||
# with different KV layouts (e.g. PP stages) to use the same pool size.
|
||||
# Byte budgets still bound capacity.
|
||||
num_cpu_blocks = extra_config.get("num_cpu_blocks")
|
||||
if num_cpu_blocks is not None and (
|
||||
type(num_cpu_blocks) is not int or num_cpu_blocks <= 0
|
||||
):
|
||||
raise ValueError("num_cpu_blocks must be a positive integer")
|
||||
|
||||
lazy_offload = bool(extra_config.get("lazy_offload", False))
|
||||
|
||||
kv_offload_backend = str(extra_config.get("kv_offload_backend", "cpu"))
|
||||
@@ -160,6 +170,12 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
kv_offload_backend,
|
||||
disk_path or "none",
|
||||
)
|
||||
if num_cpu_blocks is not None:
|
||||
logger.info(
|
||||
"Using explicit num_cpu_blocks=%d for offload pool sizing "
|
||||
"instead of automatic sizing from byte budgets.",
|
||||
num_cpu_blocks,
|
||||
)
|
||||
|
||||
if role == KVConnectorRole.SCHEDULER:
|
||||
from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes
|
||||
@@ -177,6 +193,7 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
lazy_offload=lazy_offload,
|
||||
disk_capacity_bytes=disk_capacity_bytes if disk_mode else 0,
|
||||
use_page_cache=use_page_cache if disk_mode else False,
|
||||
num_cpu_blocks=num_cpu_blocks,
|
||||
)
|
||||
elif role == KVConnectorRole.WORKER:
|
||||
self.worker_handler = SimpleCPUOffloadWorker(
|
||||
@@ -188,6 +205,7 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
disk_capacity_bytes=disk_capacity_bytes,
|
||||
disk_buffer_slots=disk_buffer_slots,
|
||||
use_page_cache=use_page_cache,
|
||||
num_cpu_blocks=num_cpu_blocks,
|
||||
)
|
||||
|
||||
# --- Worker-side methods ---
|
||||
|
||||
@@ -9,16 +9,19 @@ runtime import, layout descriptors, and the conversion to `nccl.m2n` types —
|
||||
so the engine module stays about the transfer itself.
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.weight_transfer.base import ParamMeta
|
||||
from vllm.utils.nccl import unpinned_nccl_env
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
from vllm.distributed.utils import StatelessProcessGroup
|
||||
|
||||
# Placement code for a mesh axis that replicates. Any other (non-negative) code
|
||||
# is the tensor dim that axis shards. Ints rather than `nccl.m2n` objects so
|
||||
@@ -35,15 +38,15 @@ MESH_NDIMS = 2
|
||||
DESTINATION_REPLICA_AXIS = 0
|
||||
DESTINATION_SHARD_AXIS = 1
|
||||
|
||||
# m2n bounds how many source shards may feed one destination shard, and how many
|
||||
# destination shards one source shard may feed, with compile-time arrays in
|
||||
# `reshard_limits.h`. The bindings do not expose them, so they are mirrored
|
||||
# here; a build with larger arrays makes these conservative, never wrong.
|
||||
# Conservative capacities accepted by the supported nccl_m2n algorithms. They
|
||||
# are not exposed by its Python API yet; remove these mirrors once nccl_m2n
|
||||
# provides communication-free plan validation.
|
||||
MAX_SOURCE_SHARDS = 16
|
||||
MAX_DEST_SHARDS = 64
|
||||
|
||||
# Wire dtypes `ncclReshard` accepts. Notably excludes fp4 and any packed
|
||||
# sub-byte type, so quantized checkpoints are out of scope for now.
|
||||
# Increment when the JSON init payload changes incompatibly.
|
||||
M2N_WIRE_SCHEMA_VERSION = 2
|
||||
|
||||
SUPPORTED_DTYPES: frozenset[torch.dtype] = frozenset(
|
||||
dtype
|
||||
for dtype in (
|
||||
@@ -70,12 +73,62 @@ _IMPORT_HINT = (
|
||||
"The nccl_m2n weight transfer backend requires the `nccl-extensions` "
|
||||
"package (and its `nccl4py` dependency), which is not installed by vLLM. "
|
||||
"See https://github.com/NVIDIA/nccl-extensions for build "
|
||||
"and install instructions. It needs NCCL 2.30.5 or newer, and "
|
||||
"VLLM_NCCL_SO_PATH must point at the same libnccl.so that libnccl_m2n.so "
|
||||
"was linked against."
|
||||
"and install instructions. It needs NCCL 2.30.5 or newer."
|
||||
)
|
||||
|
||||
|
||||
def prepare_m2n_local_runtime(m2n: Any, max_cta: int | None) -> tuple[Any, int, Any]:
|
||||
"""Prepare rank-local resources before any rank enters NCCL init."""
|
||||
import nccl.core as nccl
|
||||
|
||||
device = torch.accelerator.current_device_index()
|
||||
if not isinstance(device, int) or isinstance(device, bool) or device < 0:
|
||||
raise RuntimeError(f"invalid current accelerator device index: {device!r}")
|
||||
handle = m2n.Handle.create(m2n.Config(max_cta=max_cta))
|
||||
return nccl, device, handle
|
||||
|
||||
|
||||
def create_m2n_communicator(
|
||||
nccl: Any,
|
||||
group: "StatelessProcessGroup",
|
||||
unique_id_bytes: bytes | None,
|
||||
) -> Any:
|
||||
"""Create the M2N data-plane communicator through NCCL4Py."""
|
||||
if unique_id_bytes is None:
|
||||
envelope: dict[str, bytes | str | None] | None = None
|
||||
if group.rank == 0:
|
||||
try:
|
||||
envelope = {
|
||||
"unique_id": bytes(nccl.get_unique_id().as_bytes),
|
||||
"error": None,
|
||||
}
|
||||
except Exception as exc:
|
||||
envelope = {
|
||||
"unique_id": None,
|
||||
"error": f"{type(exc).__name__}: {exc}",
|
||||
}
|
||||
envelope = group.broadcast_obj(envelope, src=0)
|
||||
if not isinstance(envelope, Mapping) or set(envelope) != {"unique_id", "error"}:
|
||||
raise RuntimeError("invalid NCCL4Py unique-id envelope")
|
||||
error = envelope["error"]
|
||||
received_unique_id = envelope["unique_id"]
|
||||
if error is not None:
|
||||
if not isinstance(error, str) or not error:
|
||||
raise RuntimeError("invalid NCCL4Py unique-id error")
|
||||
raise RuntimeError(f"NCCL4Py unique-id generation failed: {error}")
|
||||
if not isinstance(received_unique_id, bytes):
|
||||
raise RuntimeError("NCCL4Py unique-id broadcast did not return bytes")
|
||||
unique_id_bytes = received_unique_id
|
||||
|
||||
unique_id = nccl.UniqueId.from_bytes(unique_id_bytes)
|
||||
with unpinned_nccl_env():
|
||||
return nccl.Communicator.init(
|
||||
nranks=group.world_size,
|
||||
rank=group.rank,
|
||||
unique_id=unique_id,
|
||||
)
|
||||
|
||||
|
||||
def import_m2n() -> Any:
|
||||
"""Import `nccl.m2n` lazily, with an actionable error when it is missing.
|
||||
|
||||
@@ -97,17 +150,25 @@ class M2NMesh:
|
||||
`[start_rank, start_rank + dims[0] * dims[1])`. There is no 1-D mesh; a
|
||||
single-axis topology is spelled with a second axis of size 1.
|
||||
|
||||
One mesh describes every tensor on its side, so it is exchanged once at the
|
||||
init handshake rather than per parameter.
|
||||
A model may use a different factorization for each tensor. For example,
|
||||
dense weights can use a DP x TP mesh while expert weights use EDP x EP.
|
||||
"""
|
||||
|
||||
dims: tuple[int, int]
|
||||
start_rank: int
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if len(self.dims) != MESH_NDIMS or any(d <= 0 for d in self.dims):
|
||||
object.__setattr__(self, "dims", tuple(self.dims))
|
||||
if len(self.dims) != MESH_NDIMS or any(
|
||||
not isinstance(dim, int) or isinstance(dim, bool) or dim <= 0
|
||||
for dim in self.dims
|
||||
):
|
||||
raise ValueError(f"mesh dims must be {MESH_NDIMS} positive ints")
|
||||
if self.start_rank < 0:
|
||||
if (
|
||||
not isinstance(self.start_rank, int)
|
||||
or isinstance(self.start_rank, bool)
|
||||
or self.start_rank < 0
|
||||
):
|
||||
raise ValueError(
|
||||
f"mesh start_rank must be non-negative, got {self.start_rank}"
|
||||
)
|
||||
@@ -124,16 +185,26 @@ Placements = tuple[int, int]
|
||||
REPLICATED: Placements | None = None
|
||||
|
||||
|
||||
def check_placements(placements: Placements, context: str = "placements") -> None:
|
||||
"""Reject placement pairs m2n cannot express.
|
||||
@dataclass(frozen=True)
|
||||
class M2NLayout:
|
||||
"""A tensor's mesh and placement on one side of a transfer."""
|
||||
|
||||
The header requires exactly one SHARD axis and one REPLICATE axis.
|
||||
`{REPLICATE, REPLICATE}` hits a degenerate prepare branch, which is why
|
||||
full replication is carried as `REPLICATED` and resolved separately;
|
||||
sharding both axes is not expressible at all.
|
||||
"""
|
||||
mesh: M2NMesh
|
||||
placements: Placements | None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not isinstance(self.mesh, M2NMesh):
|
||||
raise TypeError("M2N layout mesh must be an M2NMesh")
|
||||
if self.placements is not None:
|
||||
object.__setattr__(self, "placements", tuple(self.placements))
|
||||
|
||||
|
||||
def check_placements(placements: Placements, context: str = "placements") -> None:
|
||||
"""Reject placement pairs nccl_m2n cannot express."""
|
||||
if len(placements) != MESH_NDIMS:
|
||||
raise ValueError(f"{context} must have {MESH_NDIMS} entries")
|
||||
if any(not isinstance(code, int) or isinstance(code, bool) for code in placements):
|
||||
raise TypeError(f"{context} must contain integer placement codes")
|
||||
invalid = [code for code in placements if code < REPLICATE]
|
||||
if invalid:
|
||||
raise ValueError(
|
||||
@@ -144,8 +215,7 @@ def check_placements(placements: Placements, context: str = "placements") -> Non
|
||||
if num_sharded == 0:
|
||||
raise ValueError(
|
||||
f"{context}: a fully replicated tensor is carried as REPLICATED, "
|
||||
"not as two "
|
||||
"REPLICATE axes"
|
||||
"not as two REPLICATE axes"
|
||||
)
|
||||
if num_sharded == MESH_NDIMS:
|
||||
raise ValueError(
|
||||
@@ -175,87 +245,46 @@ def resolve_layout(
|
||||
return mesh, placements
|
||||
|
||||
|
||||
def shard_count(mesh: M2NMesh, placements: Placements) -> int:
|
||||
"""How many pieces this layout splits the tensor into."""
|
||||
return next(
|
||||
(mesh.dims[axis] for axis, code in enumerate(placements) if code != REPLICATE),
|
||||
1,
|
||||
def validate_layout(
|
||||
mesh: M2NMesh,
|
||||
placements: Placements,
|
||||
shape: Sequence[int],
|
||||
context: str,
|
||||
) -> tuple[int, ...]:
|
||||
"""Validate one resolved layout and return its required local shape."""
|
||||
local_shape = list(shape)
|
||||
for axis, tensor_dim in enumerate(placements):
|
||||
if tensor_dim == REPLICATE:
|
||||
continue
|
||||
if tensor_dim >= len(shape):
|
||||
raise ValueError(
|
||||
f"{context} shards tensor dim {tensor_dim}, but parameter has "
|
||||
f"rank {len(shape)}"
|
||||
)
|
||||
factor = mesh.dims[axis]
|
||||
if shape[tensor_dim] % factor:
|
||||
raise ValueError(
|
||||
f"{context} shards dim {tensor_dim} (size {shape[tensor_dim]}) "
|
||||
f"over {factor} ranks, which does not divide evenly"
|
||||
)
|
||||
local_shape[tensor_dim] //= factor
|
||||
return tuple(local_shape)
|
||||
|
||||
|
||||
def expected_local_shape(meta: "M2NParamMeta") -> tuple[int, ...]:
|
||||
"""Validate a parameter's source layout and return its local shape."""
|
||||
mesh, placements = resolve_layout(
|
||||
meta.source_layout.mesh,
|
||||
meta.source_layout.placements,
|
||||
f"parameter '{meta.name}' source placements",
|
||||
)
|
||||
return validate_layout(
|
||||
mesh, placements, meta.shape, f"parameter '{meta.name}' source layout"
|
||||
)
|
||||
|
||||
|
||||
def check_plan_limits(
|
||||
src: tuple[M2NMesh, Placements],
|
||||
dst: tuple[M2NMesh, Placements],
|
||||
name: str,
|
||||
) -> None:
|
||||
"""Reject plans that exceed m2n's static per-shard fan-in / fan-out arrays.
|
||||
|
||||
Only the unambiguous cases are checked: when one side is a single shard it
|
||||
is fed by (or feeds) every shard on the other side, so the count is exactly
|
||||
the other side's shard count. In the general sharded-to-sharded case the
|
||||
overlap depends on the library's chunking, and reproducing that arithmetic
|
||||
here would duplicate internals that can change under us.
|
||||
|
||||
m2n re-checks authoritatively and, because the plan is derived from the
|
||||
shared descriptors, fails identically on every rank -- so this is about
|
||||
reporting at init with a message that names the parameter, not about
|
||||
avoiding a hang.
|
||||
"""
|
||||
src_shards = shard_count(*src)
|
||||
dst_shards = shard_count(*dst)
|
||||
if dst_shards == 1 and src_shards > MAX_SOURCE_SHARDS:
|
||||
raise ValueError(
|
||||
f"parameter '{name}' would feed {src_shards} source shards into one "
|
||||
f"replicated destination, over m2n's MAX_SOURCES={MAX_SOURCE_SHARDS}. "
|
||||
"Shard the destination, replicate the source, or rebuild m2n with "
|
||||
"larger arrays."
|
||||
)
|
||||
if src_shards == 1 and dst_shards > MAX_DEST_SHARDS:
|
||||
raise ValueError(
|
||||
f"parameter '{name}' would fan one source shard out to {dst_shards} "
|
||||
f"destination shards, over m2n's MAX_TARGETS={MAX_DEST_SHARDS}. "
|
||||
"Rebuild m2n with larger arrays to raise the bound."
|
||||
)
|
||||
|
||||
|
||||
def validate_layout(
|
||||
mesh: M2NMesh, placements: Placements, shape: Sequence[int], side: str
|
||||
) -> None:
|
||||
"""Check a resolved layout can describe `shape` before any collective runs.
|
||||
|
||||
Called on both sides at init so a bad plan surfaces as an error from the
|
||||
init RPC rather than as a hang inside the first reshard.
|
||||
"""
|
||||
for axis, code in enumerate(placements):
|
||||
if code == REPLICATE:
|
||||
continue
|
||||
if code >= len(shape):
|
||||
raise ValueError(
|
||||
f"{side} layout shards tensor dim {code}, but the tensor "
|
||||
f"has rank {len(shape)}"
|
||||
)
|
||||
factor = mesh.dims[axis]
|
||||
if shape[code] % factor:
|
||||
raise ValueError(
|
||||
f"{side} layout shards dim {code} (size {shape[code]}) over "
|
||||
f"{factor} ranks, which does not divide evenly"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class M2NParamMeta(ParamMeta):
|
||||
"""`ParamMeta` extended with how the trainer places this tensor.
|
||||
|
||||
The base class carries only name / dtype / full shape, which is not enough
|
||||
to plan a reshard. `placements` is relative to its side's `M2NMesh`, or
|
||||
`REPLICATED` when every rank holds the whole tensor.
|
||||
"""
|
||||
|
||||
placements: Placements | None
|
||||
|
||||
|
||||
def check_transferable(name: str, dtype: torch.dtype, shape: Sequence[int]) -> None:
|
||||
"""Reject tensors m2n cannot move at all, with the parameter named."""
|
||||
"""Reject parameters nccl_m2n cannot transfer."""
|
||||
if dtype not in SUPPORTED_DTYPES:
|
||||
raise ValueError(
|
||||
f"parameter '{name}' has dtype {dtype}, which nccl_m2n does not "
|
||||
@@ -268,6 +297,514 @@ def check_transferable(name: str, dtype: torch.dtype, shape: Sequence[int]) -> N
|
||||
)
|
||||
|
||||
|
||||
def _shard_spec(mesh: M2NMesh, placements: Placements) -> tuple[int, int]:
|
||||
return next(
|
||||
(code, mesh.dims[axis])
|
||||
for axis, code in enumerate(placements)
|
||||
if code != REPLICATE
|
||||
)
|
||||
|
||||
|
||||
def _max_partition_overlaps(
|
||||
extent: int,
|
||||
partitions: int,
|
||||
other_partitions: int,
|
||||
) -> int:
|
||||
"""Return the maximum other partitions intersected by any one partition."""
|
||||
if extent == 0:
|
||||
return 0
|
||||
width = extent // partitions
|
||||
other_width = extent // other_partitions
|
||||
max_overlaps = 0
|
||||
for index in range(partitions):
|
||||
start = index * width
|
||||
end = start + width
|
||||
first_other = start // other_width
|
||||
last_other = (end - 1) // other_width
|
||||
max_overlaps = max(max_overlaps, last_other - first_other + 1)
|
||||
return max_overlaps
|
||||
|
||||
|
||||
def check_plan_limits(
|
||||
src: tuple[M2NMesh, Placements],
|
||||
dst: tuple[M2NMesh, Placements],
|
||||
name: str,
|
||||
shape: Sequence[int],
|
||||
) -> None:
|
||||
"""Reject plans whose per-shard fan-in or fan-out exceeds M2N capacity."""
|
||||
src_dim, src_shards = _shard_spec(*src)
|
||||
dst_dim, dst_shards = _shard_spec(*dst)
|
||||
if any(extent == 0 for extent in shape):
|
||||
max_sources = max_targets = 0
|
||||
elif src_dim != dst_dim:
|
||||
max_sources = src_shards
|
||||
max_targets = dst_shards
|
||||
else:
|
||||
extent = shape[src_dim]
|
||||
max_sources = _max_partition_overlaps(extent, dst_shards, src_shards)
|
||||
max_targets = _max_partition_overlaps(extent, src_shards, dst_shards)
|
||||
|
||||
if max_sources > MAX_SOURCE_SHARDS:
|
||||
raise ValueError(
|
||||
f"parameter '{name}' would feed up to {max_sources} source shards "
|
||||
f"into one destination shard, over nccl_m2n MAX_SOURCES="
|
||||
f"{MAX_SOURCE_SHARDS}"
|
||||
)
|
||||
if max_targets > MAX_DEST_SHARDS:
|
||||
raise ValueError(
|
||||
f"parameter '{name}' would fan one source shard out to up to "
|
||||
f"{max_targets} destination shards, over nccl_m2n "
|
||||
f"MAX_TARGETS={MAX_DEST_SHARDS}"
|
||||
)
|
||||
|
||||
|
||||
def validate_transfer_plan(
|
||||
metas: Sequence["M2NParamMeta"],
|
||||
dst_mesh: M2NMesh,
|
||||
dst_placements: Sequence[Placements | None],
|
||||
) -> None:
|
||||
"""Validate the complete ordered source-to-destination transfer plan."""
|
||||
if len(dst_placements) != len(metas):
|
||||
raise ValueError(
|
||||
f"destination plan has {len(dst_placements)} parameters, expected "
|
||||
f"{len(metas)}"
|
||||
)
|
||||
for meta, placement in zip(metas, dst_placements):
|
||||
check_transferable(meta.name, meta.dtype, meta.shape)
|
||||
src = resolve_layout(
|
||||
meta.source_layout.mesh,
|
||||
meta.source_layout.placements,
|
||||
f"parameter '{meta.name}' source placements",
|
||||
)
|
||||
validate_layout(*src, meta.shape, f"parameter '{meta.name}' source layout")
|
||||
dst = resolve_layout(
|
||||
dst_mesh,
|
||||
placement,
|
||||
f"parameter '{meta.name}' destination placements",
|
||||
)
|
||||
validate_layout(*dst, meta.shape, f"parameter '{meta.name}' destination layout")
|
||||
check_plan_limits(src, dst, meta.name, meta.shape)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class M2NParamMeta(ParamMeta):
|
||||
"""`ParamMeta` extended with how the trainer places this tensor.
|
||||
|
||||
The base class carries only name / dtype / full shape, which is not enough
|
||||
to plan a reshard. `source_layout` carries both the per-parameter mesh and
|
||||
its placement on that mesh.
|
||||
"""
|
||||
|
||||
source_layout: M2NLayout
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not isinstance(self.source_layout, M2NLayout):
|
||||
raise TypeError("source_layout must be an M2NLayout")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class M2NWireParam:
|
||||
"""JSON-safe source metadata for one parameter in the init handshake."""
|
||||
|
||||
name: str
|
||||
dtype_name: str
|
||||
shape: tuple[int, ...]
|
||||
src_mesh_dims: tuple[int, int]
|
||||
src_placements: Placements | None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
shape = tuple(self.shape)
|
||||
dims = tuple(self.src_mesh_dims)
|
||||
placements = None if self.src_placements is None else tuple(self.src_placements)
|
||||
object.__setattr__(self, "shape", shape)
|
||||
object.__setattr__(self, "src_mesh_dims", dims)
|
||||
object.__setattr__(self, "src_placements", placements)
|
||||
if not isinstance(self.name, str) or not self.name:
|
||||
raise ValueError("M2N wire parameter name must not be empty")
|
||||
if not isinstance(self.dtype_name, str) or not self.dtype_name:
|
||||
raise ValueError(f"parameter '{self.name}' has an empty wire dtype name")
|
||||
if not shape or any(
|
||||
not isinstance(dim, int) or isinstance(dim, bool) or dim <= 0
|
||||
for dim in shape
|
||||
):
|
||||
raise ValueError(
|
||||
f"parameter '{self.name}' shape must contain positive integers"
|
||||
)
|
||||
M2NMesh(cast(tuple[int, int], dims), 0)
|
||||
if placements is not None:
|
||||
check_placements(
|
||||
cast(Placements, placements),
|
||||
f"parameter '{self.name}' source placements",
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, value: Mapping[str, Any]) -> "M2NWireParam":
|
||||
"""Parse one strict wire record from its JSON-decoded mapping."""
|
||||
fields = {
|
||||
"name",
|
||||
"dtype_name",
|
||||
"shape",
|
||||
"src_mesh_dims",
|
||||
"src_placements",
|
||||
}
|
||||
missing = fields - value.keys()
|
||||
extra = value.keys() - fields
|
||||
if missing or extra:
|
||||
details = []
|
||||
if missing:
|
||||
details.append(f"missing {sorted(missing)}")
|
||||
if extra:
|
||||
details.append(f"unexpected {sorted(extra)}")
|
||||
raise ValueError(f"invalid M2N wire parameter: {', '.join(details)}")
|
||||
return cls(**value)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""Serialize this record using only JSON-compatible values."""
|
||||
return {
|
||||
"name": self.name,
|
||||
"dtype_name": self.dtype_name,
|
||||
"shape": list(self.shape),
|
||||
"src_mesh_dims": list(self.src_mesh_dims),
|
||||
"src_placements": (
|
||||
None if self.src_placements is None else list(self.src_placements)
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def source_plan_digest(params: Sequence[M2NWireParam]) -> str:
|
||||
"""Return a rank-independent digest of the ordered source wire plan."""
|
||||
payload = json.dumps(
|
||||
{
|
||||
"schema_version": M2N_WIRE_SCHEMA_VERSION,
|
||||
"params": [param.to_dict() for param in params],
|
||||
},
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode()
|
||||
return hashlib.sha256(payload).hexdigest()
|
||||
|
||||
|
||||
def complete_plan_digest(
|
||||
source_digest: str,
|
||||
dst_mesh: M2NMesh,
|
||||
dst_placements: Sequence[Placements | None],
|
||||
) -> str:
|
||||
"""Return a digest of the agreed source and destination plan."""
|
||||
if not isinstance(source_digest, str) or not source_digest:
|
||||
raise ValueError("source digest must be a non-empty string")
|
||||
payload = json.dumps(
|
||||
{
|
||||
"schema_version": M2N_WIRE_SCHEMA_VERSION,
|
||||
"source_digest": source_digest,
|
||||
"destination_mesh": {
|
||||
"dims": list(dst_mesh.dims),
|
||||
"start_rank": dst_mesh.start_rank,
|
||||
},
|
||||
"destination_placements": [
|
||||
None if placement is None else list(placement)
|
||||
for placement in dst_placements
|
||||
],
|
||||
},
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode()
|
||||
return hashlib.sha256(payload).hexdigest()
|
||||
|
||||
|
||||
def check_source_plan_agreement(
|
||||
group: "StatelessProcessGroup",
|
||||
params: Sequence[M2NWireParam],
|
||||
expected_digest: str | None = None,
|
||||
local_error: str | None = None,
|
||||
) -> str:
|
||||
"""Verify every participant entered with the same source plan."""
|
||||
prepared_error = local_error
|
||||
digest: str | None = None
|
||||
declared: str | None = expected_digest
|
||||
if prepared_error is None:
|
||||
try:
|
||||
digest = source_plan_digest(params)
|
||||
declared = digest if expected_digest is None else expected_digest
|
||||
if not isinstance(declared, str) or not declared:
|
||||
raise ValueError("declared source digest must be a non-empty string")
|
||||
except Exception as exc:
|
||||
prepared_error = f"{type(exc).__name__}: {exc}"
|
||||
elif not isinstance(prepared_error, str) or not prepared_error:
|
||||
prepared_error = "TypeError: local source error must be a non-empty string"
|
||||
|
||||
gathered = group.all_gather_obj(
|
||||
{
|
||||
"phase": "source",
|
||||
"digest": digest,
|
||||
"declared_digest": declared,
|
||||
"error": prepared_error,
|
||||
}
|
||||
)
|
||||
errors: list[str] = []
|
||||
expected_fields = {"phase", "digest", "declared_digest", "error"}
|
||||
if len(gathered) != group.world_size:
|
||||
errors.append(
|
||||
"control-plane all-gather returned "
|
||||
f"{len(gathered)} records for world size {group.world_size}"
|
||||
)
|
||||
agreed: list[tuple[int, str, str]] = []
|
||||
for rank, item in enumerate(gathered):
|
||||
if not isinstance(item, Mapping) or set(item) != expected_fields:
|
||||
errors.append(f"rank {rank}: invalid source preflight envelope")
|
||||
continue
|
||||
if item["phase"] != "source":
|
||||
errors.append(f"rank {rank}: invalid source preflight phase")
|
||||
continue
|
||||
error = item["error"]
|
||||
if error is not None:
|
||||
if not isinstance(error, str) or not error:
|
||||
errors.append(f"rank {rank}: invalid source preflight error")
|
||||
else:
|
||||
errors.append(f"rank {rank}: {error}")
|
||||
continue
|
||||
computed = item["digest"]
|
||||
claimed = item["declared_digest"]
|
||||
if (
|
||||
not isinstance(computed, str)
|
||||
or not computed
|
||||
or not isinstance(claimed, str)
|
||||
or not claimed
|
||||
):
|
||||
errors.append(f"rank {rank}: invalid source digest")
|
||||
continue
|
||||
agreed.append((rank, computed, claimed))
|
||||
if errors:
|
||||
raise RuntimeError(
|
||||
"nccl_m2n source preflight rejected before NCCL init: " + "; ".join(errors)
|
||||
)
|
||||
|
||||
canonical = agreed[0][1]
|
||||
for rank, computed, claimed in agreed:
|
||||
if computed != claimed:
|
||||
errors.append(
|
||||
f"rank {rank}: computed digest {computed} does not match "
|
||||
f"declared digest {claimed}"
|
||||
)
|
||||
if computed != canonical:
|
||||
errors.append(f"rank {rank}: source digest disagrees with rank 0")
|
||||
if errors:
|
||||
raise RuntimeError(
|
||||
"nccl_m2n source preflight rejected before NCCL init: " + "; ".join(errors)
|
||||
)
|
||||
return canonical
|
||||
|
||||
|
||||
def check_complete_plan_agreement(
|
||||
group: "StatelessProcessGroup",
|
||||
source_digest: str,
|
||||
dst_mesh: M2NMesh,
|
||||
dst_placements: Sequence[Placements | None],
|
||||
local_error: str | None = None,
|
||||
) -> str:
|
||||
"""Require every rank to accept the same complete transfer plan."""
|
||||
prepared_error = local_error
|
||||
digest: str | None = None
|
||||
if prepared_error is None:
|
||||
try:
|
||||
digest = complete_plan_digest(source_digest, dst_mesh, dst_placements)
|
||||
except Exception as exc:
|
||||
prepared_error = f"{type(exc).__name__}: {exc}"
|
||||
elif not isinstance(prepared_error, str) or not prepared_error:
|
||||
prepared_error = "TypeError: local plan error must be a non-empty string"
|
||||
|
||||
gathered = group.all_gather_obj(
|
||||
{"phase": "complete_plan", "digest": digest, "error": prepared_error}
|
||||
)
|
||||
errors: list[str] = []
|
||||
valid: list[tuple[int, str]] = []
|
||||
expected_fields = {"phase", "digest", "error"}
|
||||
if len(gathered) != group.world_size:
|
||||
errors.append(
|
||||
"control-plane all-gather returned "
|
||||
f"{len(gathered)} records for world size {group.world_size}"
|
||||
)
|
||||
for rank, item in enumerate(gathered):
|
||||
if (
|
||||
not isinstance(item, Mapping)
|
||||
or set(item) != expected_fields
|
||||
or item.get("phase") != "complete_plan"
|
||||
):
|
||||
errors.append(f"rank {rank}: invalid complete-plan envelope")
|
||||
continue
|
||||
error = item["error"]
|
||||
if error is not None:
|
||||
if not isinstance(error, str) or not error:
|
||||
errors.append(f"rank {rank}: invalid complete-plan error")
|
||||
else:
|
||||
errors.append(f"rank {rank}: {error}")
|
||||
continue
|
||||
rank_digest = item["digest"]
|
||||
if not isinstance(rank_digest, str) or not rank_digest:
|
||||
errors.append(f"rank {rank}: invalid complete-plan digest")
|
||||
continue
|
||||
valid.append((rank, rank_digest))
|
||||
if not errors:
|
||||
canonical = valid[0][1]
|
||||
for rank, rank_digest in valid[1:]:
|
||||
if rank_digest != canonical:
|
||||
errors.append(
|
||||
f"rank {rank}: complete-plan digest disagrees with rank 0"
|
||||
)
|
||||
if errors:
|
||||
raise RuntimeError(
|
||||
"nccl_m2n complete-plan preflight rejected before first reshard: "
|
||||
+ "; ".join(errors)
|
||||
)
|
||||
return valid[0][1]
|
||||
|
||||
|
||||
def check_runtime_ready_agreement(
|
||||
group: "StatelessProcessGroup", local_error: str | None
|
||||
) -> None:
|
||||
"""Require every rank to prepare local resources before NCCL init."""
|
||||
prepared_error = local_error
|
||||
if prepared_error is not None and (
|
||||
not isinstance(prepared_error, str) or not prepared_error
|
||||
):
|
||||
prepared_error = "TypeError: local runtime error must be a non-empty string"
|
||||
gathered = group.all_gather_obj({"phase": "runtime_ready", "error": prepared_error})
|
||||
errors: list[str] = []
|
||||
if len(gathered) != group.world_size:
|
||||
errors.append(
|
||||
"control-plane all-gather returned "
|
||||
f"{len(gathered)} records for world size {group.world_size}"
|
||||
)
|
||||
for rank, item in enumerate(gathered):
|
||||
if (
|
||||
not isinstance(item, Mapping)
|
||||
or set(item) != {"phase", "error"}
|
||||
or item.get("phase") != "runtime_ready"
|
||||
):
|
||||
errors.append(f"rank {rank}: invalid runtime readiness envelope")
|
||||
continue
|
||||
error = item.get("error")
|
||||
if error is not None:
|
||||
if not isinstance(error, str) or not error:
|
||||
errors.append(f"rank {rank}: invalid runtime readiness error")
|
||||
else:
|
||||
errors.append(f"rank {rank}: {error}")
|
||||
if errors:
|
||||
raise RuntimeError(
|
||||
"nccl_m2n local runtime preflight failed before NCCL init: "
|
||||
+ "; ".join(errors)
|
||||
)
|
||||
|
||||
|
||||
def check_data_plane_agreement(
|
||||
group: "StatelessProcessGroup",
|
||||
unique_id_bytes: bytes | None,
|
||||
local_error: str | None = None,
|
||||
) -> None:
|
||||
"""Agree on the NCCL rendezvous identity before NCCL init."""
|
||||
prepared_error = local_error
|
||||
uid_digest: str | None = None
|
||||
if prepared_error is None:
|
||||
try:
|
||||
if unique_id_bytes is not None:
|
||||
if not isinstance(unique_id_bytes, bytes):
|
||||
raise TypeError("NCCL unique id must decode to bytes")
|
||||
uid_digest = hashlib.sha256(unique_id_bytes).hexdigest()
|
||||
except Exception as exc:
|
||||
prepared_error = f"{type(exc).__name__}: {exc}"
|
||||
elif not isinstance(prepared_error, str) or not prepared_error:
|
||||
prepared_error = "TypeError: local data-plane error must be non-empty"
|
||||
|
||||
gathered = group.all_gather_obj(
|
||||
{
|
||||
"phase": "data_plane",
|
||||
"mode": "uid" if unique_id_bytes is not None else "tcp",
|
||||
"uid_digest": uid_digest,
|
||||
"error": prepared_error,
|
||||
}
|
||||
)
|
||||
expected_fields = {
|
||||
"phase",
|
||||
"mode",
|
||||
"uid_digest",
|
||||
"error",
|
||||
}
|
||||
errors: list[str] = []
|
||||
valid: list[tuple[int, str, str | None]] = []
|
||||
if len(gathered) != group.world_size:
|
||||
errors.append(
|
||||
"control-plane all-gather returned "
|
||||
f"{len(gathered)} records for world size {group.world_size}"
|
||||
)
|
||||
for rank, item in enumerate(gathered):
|
||||
if (
|
||||
not isinstance(item, Mapping)
|
||||
or set(item) != expected_fields
|
||||
or item.get("phase") != "data_plane"
|
||||
):
|
||||
errors.append(f"rank {rank}: invalid data-plane preflight envelope")
|
||||
continue
|
||||
error = item["error"]
|
||||
if error is not None:
|
||||
if not isinstance(error, str) or not error:
|
||||
errors.append(f"rank {rank}: invalid data-plane preflight error")
|
||||
else:
|
||||
errors.append(f"rank {rank}: {error}")
|
||||
continue
|
||||
mode = item["mode"]
|
||||
uid = item["uid_digest"]
|
||||
if mode not in {"tcp", "uid"} or (mode == "uid") != isinstance(uid, str):
|
||||
errors.append(f"rank {rank}: invalid data-plane identity")
|
||||
continue
|
||||
valid.append((rank, mode, uid))
|
||||
if not errors:
|
||||
_, mode, uid = valid[0]
|
||||
for rank, other_mode, other_uid in valid[1:]:
|
||||
if (other_mode, other_uid) != (mode, uid):
|
||||
errors.append(f"rank {rank}: NCCL data-plane identity disagrees")
|
||||
if errors:
|
||||
raise RuntimeError(
|
||||
"nccl_m2n data-plane preflight rejected before NCCL init: "
|
||||
+ "; ".join(errors)
|
||||
)
|
||||
|
||||
|
||||
def validate_local_buffer(m2n: Any, meta: M2NParamMeta, buffer: Any) -> None:
|
||||
"""Check source dtype/shape while delegating buffer validity to M2N."""
|
||||
# A raw pointer carries no inspectable metadata. The source owns its
|
||||
# correctness; _send supplies the declared dtype and local shape to M2N.
|
||||
if isinstance(buffer, int):
|
||||
return
|
||||
|
||||
mesh, placements = resolve_layout(
|
||||
meta.source_layout.mesh, meta.source_layout.placements
|
||||
)
|
||||
actual = m2n.DistTensor(
|
||||
buffer,
|
||||
mesh=to_mesh(m2n, mesh),
|
||||
placements=to_placements(m2n, placements),
|
||||
)
|
||||
declared = m2n.DistTensor(
|
||||
None,
|
||||
local_shape=actual.local_shape,
|
||||
dtype=meta.dtype,
|
||||
mesh=to_mesh(m2n, mesh),
|
||||
placements=to_placements(m2n, placements),
|
||||
)
|
||||
if actual.dtype != declared.dtype:
|
||||
raise ValueError(
|
||||
f"parameter '{meta.name}' source returned dtype {actual.dtype}, "
|
||||
f"but its metadata declared {declared.dtype}"
|
||||
)
|
||||
|
||||
expected = expected_local_shape(meta)
|
||||
if actual.local_shape != expected:
|
||||
raise ValueError(
|
||||
f"parameter '{meta.name}' source returned local shape "
|
||||
f"{actual.local_shape}, but layout {meta.source_layout} implies "
|
||||
f"{expected} from global shape {meta.shape}"
|
||||
)
|
||||
|
||||
|
||||
def to_mesh(m2n: Any, mesh: M2NMesh) -> Any:
|
||||
return m2n.Mesh(mesh.dims, start_rank=mesh.start_rank)
|
||||
|
||||
@@ -279,7 +816,9 @@ def to_placements(m2n: Any, placements: Placements) -> list[Any]:
|
||||
|
||||
|
||||
def publish_destination_placements(
|
||||
comm: "PyNcclCommunicator",
|
||||
comm: Any,
|
||||
device: torch.device,
|
||||
stream: torch.cuda.Stream,
|
||||
first_worker_rank: int,
|
||||
placements: "Sequence[Placements | None] | None",
|
||||
num_parameters: int,
|
||||
@@ -294,15 +833,15 @@ def publish_destination_placements(
|
||||
rather than deadlocking later.
|
||||
|
||||
Only placements travel: the destination *mesh* is still derived from the
|
||||
rank split, identically on both sides. Use the NCCL communicator directly
|
||||
because unique-id rendezvous deliberately has no bootstrap process group.
|
||||
rank split, identically on both sides. The plan is published on the data
|
||||
plane before any M2N reshard collective.
|
||||
"""
|
||||
replicated_sentinel = REPLICATE - 1
|
||||
encoded = torch.full(
|
||||
(num_parameters, MESH_NDIMS),
|
||||
replicated_sentinel,
|
||||
dtype=torch.int8,
|
||||
device=comm.device,
|
||||
device=device,
|
||||
)
|
||||
if comm.rank == first_worker_rank:
|
||||
if placements is None or len(placements) != num_parameters:
|
||||
@@ -314,25 +853,12 @@ def publish_destination_placements(
|
||||
for placement in placements
|
||||
]
|
||||
if rows:
|
||||
encoded.copy_(torch.tensor(rows, dtype=torch.int8, device=comm.device))
|
||||
encoded.copy_(torch.tensor(rows, dtype=torch.int8, device=device))
|
||||
|
||||
comm.broadcast(encoded, src=first_worker_rank)
|
||||
comm.broadcast(encoded, encoded, root=first_worker_rank, stream=stream)
|
||||
return [
|
||||
REPLICATED
|
||||
if all(code == replicated_sentinel for code in row)
|
||||
else cast(Placements, tuple(row))
|
||||
for row in encoded.tolist()
|
||||
]
|
||||
|
||||
|
||||
def comm_ptr(comm: "PyNcclCommunicator") -> int:
|
||||
"""Raw `ncclComm_t` behind vLLM's `PyNcclCommunicator`.
|
||||
|
||||
m2n links its own NCCL, so the handle only means anything if vLLM loaded
|
||||
the same `libnccl.so` — set `VLLM_NCCL_SO_PATH` accordingly.
|
||||
"""
|
||||
handle = comm.comm
|
||||
ptr = getattr(handle, "value", handle)
|
||||
if not ptr:
|
||||
raise RuntimeError("PyNcclCommunicator has no live NCCL communicator")
|
||||
return int(ptr)
|
||||
|
||||
@@ -20,7 +20,7 @@ inside `receive_weights` while the trainer is sending. Driving that from the
|
||||
trainer is the trainer engine's job.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterator
|
||||
from collections.abc import Iterator, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
@@ -34,20 +34,28 @@ from vllm.distributed.weight_transfer.base import (
|
||||
)
|
||||
from vllm.distributed.weight_transfer.m2n_common import (
|
||||
DESTINATION_SHARD_AXIS,
|
||||
M2N_WIRE_SCHEMA_VERSION,
|
||||
MESH_NDIMS,
|
||||
REPLICATED,
|
||||
M2NLayout,
|
||||
M2NMesh,
|
||||
M2NParamMeta,
|
||||
M2NWireParam,
|
||||
Placements,
|
||||
check_plan_limits,
|
||||
check_complete_plan_agreement,
|
||||
check_data_plane_agreement,
|
||||
check_runtime_ready_agreement,
|
||||
check_source_plan_agreement,
|
||||
check_transferable,
|
||||
comm_ptr,
|
||||
create_m2n_communicator,
|
||||
expected_local_shape,
|
||||
import_m2n,
|
||||
prepare_m2n_local_runtime,
|
||||
publish_destination_placements,
|
||||
resolve_layout,
|
||||
source_plan_digest,
|
||||
to_mesh,
|
||||
to_placements,
|
||||
validate_layout,
|
||||
validate_transfer_plan,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.m2n_layout import (
|
||||
M2NDestination,
|
||||
@@ -56,12 +64,14 @@ from vllm.distributed.weight_transfer.m2n_layout import (
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
NCCLWeightTransferInitInfo,
|
||||
decode_nccl_unique_id,
|
||||
worker_init_process_group,
|
||||
worker_init_metadata_group,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
__all__ = [
|
||||
"M2NWeightTransferInitInfo",
|
||||
@@ -84,59 +94,99 @@ class M2NWeightTransferInitInfo(WeightTransferInitInfo):
|
||||
plain JSON so the HTTP control plane carries it unchanged.
|
||||
"""
|
||||
|
||||
master_address: str | None = None
|
||||
master_port: int | None = None
|
||||
schema_version: int
|
||||
master_address: str
|
||||
master_port: int
|
||||
rank_offset: int
|
||||
"""First worker rank, i.e. the number of trainer ranks."""
|
||||
world_size: int
|
||||
"""Trainer ranks + all inference workers."""
|
||||
src_mesh_dims: list[int]
|
||||
"""The trainer's mesh, shared by every parameter (`start_rank` is 0: the
|
||||
trainer occupies the front of the communicator)."""
|
||||
dst_mesh_dims: list[int]
|
||||
"""The inference mesh, starting at `rank_offset`. Declared rather than
|
||||
derived so both sides describe the destination identically."""
|
||||
names: list[str]
|
||||
dtype_names: list[str]
|
||||
shapes: list[list[int]]
|
||||
src_placements: list[list[int] | None]
|
||||
"""Per parameter, relative to `src_mesh_dims`; `None` means replicated."""
|
||||
source_digest: str
|
||||
"""SHA-256 of the ordered, versioned source parameter plan."""
|
||||
params: list[dict[str, Any]]
|
||||
"""Ordered per-parameter source metadata and layouts."""
|
||||
nccl_unique_id_b64: str | None = field(default=None, repr=False)
|
||||
max_cta: int | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_ = self.nccl_unique_id_bytes
|
||||
num_params = len(self.names)
|
||||
for label, values in (
|
||||
("dtype_names", self.dtype_names),
|
||||
("shapes", self.shapes),
|
||||
("src_placements", self.src_placements),
|
||||
if not isinstance(self.master_address, str) or not self.master_address:
|
||||
raise ValueError("`master_address` must name the metadata rendezvous")
|
||||
if (
|
||||
not isinstance(self.master_port, int)
|
||||
or isinstance(self.master_port, bool)
|
||||
or not 0 < self.master_port < 65536
|
||||
):
|
||||
raise ValueError("`master_port` must be a valid metadata port")
|
||||
if (
|
||||
not isinstance(self.rank_offset, int)
|
||||
or isinstance(self.rank_offset, bool)
|
||||
or not isinstance(self.world_size, int)
|
||||
or isinstance(self.world_size, bool)
|
||||
or self.rank_offset < 1
|
||||
or self.rank_offset >= self.world_size
|
||||
):
|
||||
if len(values) != num_params:
|
||||
raise ValueError(
|
||||
f"`{label}` should be of the same size as `names`: "
|
||||
f"got {len(values)} and {num_params}"
|
||||
)
|
||||
if self.rank_offset < 1 or self.rank_offset >= self.world_size:
|
||||
raise ValueError(
|
||||
f"`rank_offset` ({self.rank_offset}) must leave at least one "
|
||||
f"trainer rank and one worker in world_size {self.world_size}"
|
||||
)
|
||||
num_workers = self.world_size - self.rank_offset
|
||||
dst = tuple(self.dst_mesh_dims)
|
||||
if len(dst) != MESH_NDIMS or dst[0] * dst[1] != num_workers:
|
||||
try:
|
||||
dst_mesh = M2NMesh(cast(tuple[int, int], dst), self.rank_offset)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError(f"invalid `dst_mesh_dims` {dst}: {exc}") from exc
|
||||
if dst_mesh.size != num_workers:
|
||||
raise ValueError(
|
||||
f"`dst_mesh_dims` {dst} must be {MESH_NDIMS} dims covering the "
|
||||
f"{num_workers} inference workers"
|
||||
)
|
||||
|
||||
def parse_wire_params(self) -> tuple[M2NWireParam, ...]:
|
||||
"""Parse and authenticate the ordered source plan."""
|
||||
if self.schema_version != M2N_WIRE_SCHEMA_VERSION:
|
||||
raise ValueError(
|
||||
"unsupported nccl_m2n init schema version "
|
||||
f"{self.schema_version}; expected {M2N_WIRE_SCHEMA_VERSION}"
|
||||
)
|
||||
if not isinstance(self.params, list):
|
||||
raise TypeError("`params` must be a list of wire parameter mappings")
|
||||
if not isinstance(self.source_digest, str) or not self.source_digest:
|
||||
raise ValueError("`source_digest` must be a non-empty string")
|
||||
wire_params: list[M2NWireParam] = []
|
||||
names: set[str] = set()
|
||||
for index, value in enumerate(self.params):
|
||||
if not isinstance(value, Mapping):
|
||||
raise TypeError(
|
||||
f"`params[{index}]` must be a mapping, got {type(value).__name__}"
|
||||
)
|
||||
param = M2NWireParam.from_dict(value)
|
||||
if param.name in names:
|
||||
raise ValueError(
|
||||
f"nccl_m2n init params contain duplicate parameter '{param.name}'"
|
||||
)
|
||||
names.add(param.name)
|
||||
wire_params.append(param)
|
||||
parsed = tuple(wire_params)
|
||||
computed_digest = source_plan_digest(parsed)
|
||||
if self.source_digest != computed_digest:
|
||||
raise ValueError(
|
||||
"nccl_m2n init source digest does not match params: "
|
||||
f"computed {computed_digest}, declared {self.source_digest}"
|
||||
)
|
||||
return parsed
|
||||
|
||||
@property
|
||||
def nccl_unique_id_bytes(self) -> bytes | None:
|
||||
if self.nccl_unique_id_b64 is None:
|
||||
return None
|
||||
return decode_nccl_unique_id(
|
||||
master_address=self.master_address,
|
||||
master_port=self.master_port,
|
||||
master_address=None,
|
||||
master_port=None,
|
||||
nccl_unique_id_b64=self.nccl_unique_id_b64,
|
||||
ctx="M2NWeightTransferInitInfo",
|
||||
ctx="M2NWeightTransferInitInfo data plane",
|
||||
)
|
||||
|
||||
|
||||
@@ -182,33 +232,172 @@ class M2NWeightTransferEngine(
|
||||
model: torch.nn.Module,
|
||||
) -> None:
|
||||
super().__init__(config, vllm_config, device, model)
|
||||
self.model_update_group: PyNcclCommunicator | None = None
|
||||
self.model_update_group: Any = None
|
||||
self._m2n: Any = None
|
||||
self._handle: Any = None
|
||||
self._metas: list[M2NParamMeta] = []
|
||||
self._src_mesh: M2NMesh | None = None
|
||||
self._dst_mesh: M2NMesh | None = None
|
||||
self._parameter_destinations: list[M2NDestination] = []
|
||||
self._index: dict[str, int] = {}
|
||||
self._uses_load_weights = False
|
||||
|
||||
def init_transfer_engine(self, init_info: M2NWeightTransferInitInfo) -> None:
|
||||
"""Join the trainer's communicator and rebuild the transfer plan.
|
||||
|
||||
Every precondition (dtype, tensor rank, divisibility) is checked here so
|
||||
a bad plan fails the init RPC instead of hanging the first collective.
|
||||
"""
|
||||
self._m2n = import_m2n()
|
||||
self._metas = []
|
||||
|
||||
self._src_mesh = M2NMesh(
|
||||
cast(tuple[int, int], tuple(init_info.src_mesh_dims)), 0
|
||||
)
|
||||
if self._src_mesh.size != init_info.rank_offset:
|
||||
raise ValueError(
|
||||
f"source mesh covers {self._src_mesh.size} ranks, but there are "
|
||||
f"{init_info.rank_offset} trainer ranks"
|
||||
"""Agree on source and local readiness before constructing NCCL."""
|
||||
if self._handle is not None or self.model_update_group is not None:
|
||||
raise RuntimeError(
|
||||
"nccl_m2n weight transfer is already initialized; call "
|
||||
"shutdown_weight_transfer_engine() before initializing again"
|
||||
)
|
||||
rendezvous_info = NCCLWeightTransferInitInfo(
|
||||
master_address=init_info.master_address,
|
||||
master_port=init_info.master_port,
|
||||
rank_offset=init_info.rank_offset,
|
||||
world_size=init_info.world_size,
|
||||
)
|
||||
metadata_group = worker_init_metadata_group(
|
||||
rendezvous_info, self.parallel_config
|
||||
)
|
||||
|
||||
wire_params: tuple[M2NWireParam, ...] = ()
|
||||
source_error: str | None = None
|
||||
try:
|
||||
self._m2n = import_m2n()
|
||||
wire_params = init_info.parse_wire_params()
|
||||
self._prepare_source_plan(wire_params, init_info.rank_offset)
|
||||
except Exception as exc:
|
||||
source_error = f"{type(exc).__name__}: {exc}"
|
||||
self._metas = []
|
||||
check_source_plan_agreement(
|
||||
metadata_group,
|
||||
wire_params,
|
||||
expected_digest=init_info.source_digest,
|
||||
local_error=source_error,
|
||||
)
|
||||
|
||||
destination_error: str | None = None
|
||||
try:
|
||||
self._prepare_destination_plan(init_info)
|
||||
except Exception as exc:
|
||||
destination_error = f"{type(exc).__name__}: {exc}"
|
||||
self._parameter_destinations = []
|
||||
|
||||
unique_id_bytes: bytes | None = None
|
||||
data_plane_error: str | None = None
|
||||
try:
|
||||
unique_id_bytes = init_info.nccl_unique_id_bytes
|
||||
except Exception as exc:
|
||||
data_plane_error = f"{type(exc).__name__}: {exc}"
|
||||
check_data_plane_agreement(
|
||||
metadata_group,
|
||||
unique_id_bytes,
|
||||
data_plane_error,
|
||||
)
|
||||
|
||||
handle = None
|
||||
group = None
|
||||
try:
|
||||
runtime_error = destination_error
|
||||
nccl = None
|
||||
device = None
|
||||
if runtime_error is None:
|
||||
try:
|
||||
nccl, device, handle = prepare_m2n_local_runtime(
|
||||
self._m2n, init_info.max_cta
|
||||
)
|
||||
except Exception as exc:
|
||||
runtime_error = f"{type(exc).__name__}: {exc}"
|
||||
check_runtime_ready_agreement(metadata_group, runtime_error)
|
||||
assert nccl is not None and device is not None and handle is not None
|
||||
group = create_m2n_communicator(nccl, metadata_group, unique_id_bytes)
|
||||
|
||||
# Keep the merged destination-placement publication unchanged: source
|
||||
# agreement is independent of worker-side destination selection.
|
||||
mine = [
|
||||
destination.placements for destination in self._parameter_destinations
|
||||
]
|
||||
agreed = publish_destination_placements(
|
||||
group,
|
||||
self.device,
|
||||
torch.cuda.current_stream(),
|
||||
init_info.rank_offset,
|
||||
mine,
|
||||
len(self._metas),
|
||||
)
|
||||
complete_plan_error: str | None = None
|
||||
try:
|
||||
if agreed != mine:
|
||||
mismatched = next(
|
||||
meta.name
|
||||
for meta, ours, theirs in zip(self._metas, mine, agreed)
|
||||
if ours != theirs
|
||||
)
|
||||
raise RuntimeError(
|
||||
"inference workers resolved different nccl_m2n "
|
||||
f"destinations (first mismatch: '{mismatched}'); all "
|
||||
"workers must run the same model and parallel config"
|
||||
)
|
||||
assert self._dst_mesh is not None
|
||||
validate_transfer_plan(self._metas, self._dst_mesh, agreed)
|
||||
except Exception as exc:
|
||||
complete_plan_error = f"{type(exc).__name__}: {exc}"
|
||||
assert self._dst_mesh is not None
|
||||
check_complete_plan_agreement(
|
||||
metadata_group,
|
||||
init_info.source_digest,
|
||||
self._dst_mesh,
|
||||
agreed,
|
||||
complete_plan_error,
|
||||
)
|
||||
except BaseException:
|
||||
if group is not None:
|
||||
try:
|
||||
# Transport cleanup only; reload validity belongs to the
|
||||
# model-update lifecycle, not this backend.
|
||||
group.abort()
|
||||
except BaseException:
|
||||
logger.exception(
|
||||
"failed to abort partially initialized nccl_m2n communicator"
|
||||
)
|
||||
if handle is not None:
|
||||
try:
|
||||
handle.destroy()
|
||||
except BaseException:
|
||||
logger.exception(
|
||||
"failed to destroy partially initialized nccl_m2n handle"
|
||||
)
|
||||
raise
|
||||
|
||||
self._handle = handle
|
||||
self.model_update_group = group
|
||||
|
||||
def _prepare_source_plan(
|
||||
self,
|
||||
wire_params: tuple[M2NWireParam, ...],
|
||||
num_trainer_ranks: int,
|
||||
) -> None:
|
||||
"""Validate worker-visible source metadata without constructing NCCL."""
|
||||
self._metas = []
|
||||
for param in wire_params:
|
||||
dtype = getattr(torch, param.dtype_name, None)
|
||||
if not isinstance(dtype, torch.dtype):
|
||||
raise ValueError(
|
||||
f"parameter '{param.name}' has invalid dtype name "
|
||||
f"'{param.dtype_name}'; expected the name of a torch.dtype"
|
||||
)
|
||||
check_transferable(param.name, dtype, param.shape)
|
||||
layout = M2NLayout(M2NMesh(param.src_mesh_dims, 0), param.src_placements)
|
||||
if layout.mesh.size != num_trainer_ranks:
|
||||
raise ValueError(
|
||||
f"parameter '{param.name}' source mesh covers "
|
||||
f"{layout.mesh.size} ranks, but there are "
|
||||
f"{num_trainer_ranks} trainer ranks"
|
||||
)
|
||||
meta = M2NParamMeta(param.name, dtype, param.shape, layout)
|
||||
expected_local_shape(meta)
|
||||
self._metas.append(meta)
|
||||
|
||||
def _prepare_destination_plan(self, init_info: M2NWeightTransferInitInfo) -> None:
|
||||
"""Resolve the merged-#51520 in-place/full-fallback destination plan."""
|
||||
# The inference topology, as declared by the trainer. This is the mesh
|
||||
# a sharded destination is placed over; a replicated one is described
|
||||
# by the alternate descriptor `resolve_layout` derives from it, which
|
||||
@@ -217,33 +406,6 @@ class M2NWeightTransferEngine(
|
||||
cast(tuple[int, int], tuple(init_info.dst_mesh_dims)),
|
||||
init_info.rank_offset,
|
||||
)
|
||||
|
||||
for name, name_dtype, shape, placements in zip(
|
||||
init_info.names,
|
||||
init_info.dtype_names,
|
||||
init_info.shapes,
|
||||
init_info.src_placements,
|
||||
):
|
||||
dtype = getattr(torch, name_dtype, None)
|
||||
if not isinstance(dtype, torch.dtype):
|
||||
raise ValueError(
|
||||
f"parameter '{name}' has invalid dtype name "
|
||||
f"'{name_dtype}'; expected the name of a torch.dtype"
|
||||
)
|
||||
check_transferable(name, dtype, shape)
|
||||
codes = (
|
||||
REPLICATED
|
||||
if placements is None
|
||||
else cast(Placements, tuple(placements))
|
||||
)
|
||||
src_mesh, src_placements = resolve_layout(
|
||||
self._src_mesh, codes, f"parameter '{name}' source placements"
|
||||
)
|
||||
validate_layout(src_mesh, src_placements, shape, "source")
|
||||
# Destination checks are deferred until loader-aware resolution;
|
||||
# assuming replication here can reject a valid sharded plan.
|
||||
self._metas.append(M2NParamMeta(name, dtype, tuple(shape), codes))
|
||||
|
||||
parallel_config = self.parallel_config
|
||||
shard_axis_size = self._dst_mesh.dims[DESTINATION_SHARD_AXIS]
|
||||
quantization = getattr(self.model_config, "quantization", None)
|
||||
@@ -266,27 +428,11 @@ class M2NWeightTransferEngine(
|
||||
shard_axis_size=shard_axis_size,
|
||||
allow_direct=allow_direct,
|
||||
)
|
||||
# Check the resolved layout rather than a provisional replicated one.
|
||||
# For example, 32 source shards feeding 4 destination shards may need
|
||||
# only 8 sources per destination, while a replicated plan needs all 32.
|
||||
for meta, destination in zip(self._metas, self._parameter_destinations):
|
||||
src_mesh, src_placements = resolve_layout(
|
||||
self._src_mesh,
|
||||
meta.placements,
|
||||
f"parameter '{meta.name}' source placements",
|
||||
)
|
||||
dst_mesh, dst_placements = resolve_layout(
|
||||
self._dst_mesh,
|
||||
destination.placements,
|
||||
f"parameter '{meta.name}' destination placements",
|
||||
)
|
||||
validate_layout(dst_mesh, dst_placements, meta.shape, "destination")
|
||||
check_plan_limits(
|
||||
(src_mesh, src_placements),
|
||||
(dst_mesh, dst_placements),
|
||||
meta.name,
|
||||
)
|
||||
|
||||
validate_transfer_plan(
|
||||
self._metas,
|
||||
self._dst_mesh,
|
||||
[destination.placements for destination in self._parameter_destinations],
|
||||
)
|
||||
# Update requests carry names only, so cache their plan indices. Track
|
||||
# whether any fallback entry requires the `load_weights` lifecycle.
|
||||
self._index = {meta.name: i for i, meta in enumerate(self._metas)}
|
||||
@@ -294,38 +440,6 @@ class M2NWeightTransferEngine(
|
||||
not destination.direct for destination in self._parameter_destinations
|
||||
)
|
||||
|
||||
self.model_update_group = worker_init_process_group(
|
||||
NCCLWeightTransferInitInfo(
|
||||
master_address=init_info.master_address,
|
||||
master_port=init_info.master_port,
|
||||
nccl_unique_id_b64=init_info.nccl_unique_id_b64,
|
||||
rank_offset=init_info.rank_offset,
|
||||
world_size=init_info.world_size,
|
||||
),
|
||||
parallel_config,
|
||||
)
|
||||
# Destinations are per-parameter and depend on the inference model, so
|
||||
# the trainer cannot derive them. The first worker publishes the plan
|
||||
# and every other rank checks its own against it — a silent
|
||||
# disagreement between workers would mean mismatched collectives.
|
||||
mine = [destination.placements for destination in self._parameter_destinations]
|
||||
agreed = publish_destination_placements(
|
||||
self.model_update_group, init_info.rank_offset, mine, len(self._metas)
|
||||
)
|
||||
if agreed != mine:
|
||||
mismatched = next(
|
||||
meta.name for meta, a, b in zip(self._metas, mine, agreed) if a != b
|
||||
)
|
||||
raise RuntimeError(
|
||||
"inference workers resolved different nccl_m2n destinations "
|
||||
f"(first mismatch: '{mismatched}'); all workers must run the "
|
||||
"same model and parallel config"
|
||||
)
|
||||
|
||||
self._handle = self._m2n.Handle.create(
|
||||
self._m2n.Config(max_cta=init_info.max_cta)
|
||||
)
|
||||
|
||||
def start_weight_update(self) -> None:
|
||||
"""Set up layerwise reloading, but only if some parameter needs it.
|
||||
|
||||
@@ -335,7 +449,9 @@ class M2NWeightTransferEngine(
|
||||
"""
|
||||
if not self._uses_load_weights:
|
||||
return
|
||||
from vllm.model_executor.model_loader.reload import initialize_layerwise_reload
|
||||
from vllm.model_executor.model_loader.reload import (
|
||||
initialize_layerwise_reload,
|
||||
)
|
||||
|
||||
initialize_layerwise_reload(self.model)
|
||||
|
||||
@@ -354,6 +470,10 @@ class M2NWeightTransferEngine(
|
||||
"nccl_m2n weight transfer not initialized. "
|
||||
"Call init_transfer_engine() first."
|
||||
)
|
||||
if not isinstance(update_info.names, list) or any(
|
||||
not isinstance(name, str) for name in update_info.names
|
||||
):
|
||||
raise TypeError("nccl_m2n update names must be a list of strings")
|
||||
|
||||
requested = []
|
||||
for name in update_info.names:
|
||||
@@ -369,7 +489,6 @@ class M2NWeightTransferEngine(
|
||||
disable_mtp_completeness_check,
|
||||
)
|
||||
|
||||
comm = comm_ptr(self.model_update_group)
|
||||
stream = torch.cuda.current_stream()
|
||||
|
||||
# Reshard lazily so one `load_weights` invocation sees the complete
|
||||
@@ -385,7 +504,13 @@ class M2NWeightTransferEngine(
|
||||
meta.shape, dtype=meta.dtype, device=self.device
|
||||
)
|
||||
|
||||
self._reshard(comm, stream, meta, destination.placements, buffer)
|
||||
self._reshard(
|
||||
self.model_update_group,
|
||||
stream,
|
||||
meta,
|
||||
destination.placements,
|
||||
buffer,
|
||||
)
|
||||
|
||||
if destination.tensor is None:
|
||||
# `load_weights` reads on the host stream, so the transfer
|
||||
@@ -410,7 +535,7 @@ class M2NWeightTransferEngine(
|
||||
|
||||
def _reshard(
|
||||
self,
|
||||
comm: int,
|
||||
comm: Any,
|
||||
stream: torch.cuda.Stream,
|
||||
src_meta: M2NParamMeta,
|
||||
dst_placements: Placements | None,
|
||||
@@ -421,11 +546,10 @@ class M2NWeightTransferEngine(
|
||||
`dst_placements` describes the buffer's logical placement over the
|
||||
worker mesh; `dst_buffer` is this rank's physical storage.
|
||||
"""
|
||||
assert self._src_mesh is not None
|
||||
assert self._dst_mesh is not None
|
||||
m2n = self._m2n
|
||||
src_mesh, resolved_src_placements = resolve_layout(
|
||||
self._src_mesh, src_meta.placements
|
||||
src_meta.source_layout.mesh, src_meta.source_layout.placements
|
||||
)
|
||||
dst_mesh, resolved_dst_placements = resolve_layout(
|
||||
self._dst_mesh, dst_placements
|
||||
@@ -444,12 +568,32 @@ class M2NWeightTransferEngine(
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Finish pending GPU work and release M2N communication state."""
|
||||
if self._handle is not None:
|
||||
# M2N does not synchronize caller streams on finalize.
|
||||
torch.accelerator.synchronize()
|
||||
self._handle.destroy()
|
||||
self._handle = None
|
||||
if self.model_update_group is not None:
|
||||
self.model_update_group.destroy()
|
||||
self.model_update_group = None
|
||||
handle, self._handle = self._handle, None
|
||||
group, self.model_update_group = self.model_update_group, None
|
||||
self._m2n = None
|
||||
self._metas = []
|
||||
self._dst_mesh = None
|
||||
self._parameter_destinations = []
|
||||
self._index = {}
|
||||
self._uses_load_weights = False
|
||||
|
||||
first_error: BaseException | None = None
|
||||
|
||||
def attempt(action: Any, message: str) -> None:
|
||||
nonlocal first_error
|
||||
try:
|
||||
action()
|
||||
except BaseException as exc:
|
||||
if first_error is None:
|
||||
first_error = exc
|
||||
logger.exception(message)
|
||||
|
||||
if handle is not None or group is not None:
|
||||
# M2N does not synchronize caller streams on finalize.
|
||||
attempt(torch.accelerator.synchronize, "failed to synchronize nccl_m2n")
|
||||
if handle is not None:
|
||||
attempt(handle.destroy, "failed to destroy nccl_m2n handle")
|
||||
if group is not None:
|
||||
attempt(group.abort, "failed to abort nccl_m2n communicator")
|
||||
if first_error is not None:
|
||||
raise first_error
|
||||
|
||||
@@ -5,21 +5,22 @@
|
||||
m2n plans a transfer from both sides' layouts, so the trainer has to say how it
|
||||
holds each parameter — which the base `WeightSource` / `ParamMeta` pair does not
|
||||
express. This module supplies the m2n flavor of both, and the DTensor-backed
|
||||
implementation that covers the common trainer.
|
||||
|
||||
A source declares its mesh once (`mesh()`) and a placement per parameter, since
|
||||
one topology describes every tensor on the side.
|
||||
implementation that covers the common trainer. Each parameter owns its source
|
||||
mesh and placements because dense and expert weights can use different
|
||||
factorizations over the same trainer ranks.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterator
|
||||
from collections.abc import Callable, Iterator, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.weight_transfer.base import ParamMeta, WeightSource
|
||||
from vllm.distributed.weight_transfer.base import WeightSource
|
||||
from vllm.distributed.weight_transfer.m2n_common import (
|
||||
REPLICATE,
|
||||
REPLICATED,
|
||||
M2NLayout,
|
||||
M2NMesh,
|
||||
M2NParamMeta,
|
||||
Placements,
|
||||
@@ -27,6 +28,8 @@ from vllm.distributed.weight_transfer.m2n_common import (
|
||||
|
||||
__all__ = [
|
||||
"M2NWeightSource",
|
||||
"M2NManifestEntry",
|
||||
"ManifestM2NWeightSource",
|
||||
"DTensorModuleSource",
|
||||
"mesh_from_tensor",
|
||||
"placements_from_tensor",
|
||||
@@ -40,17 +43,76 @@ class M2NWeightSource(WeightSource):
|
||||
materialized full tensor: gathering is exactly the cost m2n removes.
|
||||
"""
|
||||
|
||||
def mesh(self) -> M2NMesh:
|
||||
"""The trainer's rank topology, shared by every parameter."""
|
||||
def metadata(self) -> list[M2NParamMeta]: # type: ignore[override]
|
||||
"""Name, dtype, full shape, and source layout for each parameter."""
|
||||
raise NotImplementedError
|
||||
|
||||
def metadata(self) -> list[ParamMeta]:
|
||||
"""Name, dtype, full shape, and trainer placement for each parameter."""
|
||||
def __iter__(self) -> Iterator[tuple[str, Any]]: # type: ignore[override]
|
||||
"""Yield `(name, local buffer)` pairs in metadata order.
|
||||
|
||||
Providers must enqueue materialization on the caller's current CUDA
|
||||
stream, or return a buffer whose writes are already visible to it.
|
||||
Buffers may be torch tensors, CUDA Array Interface objects, or raw
|
||||
device pointers accepted by nccl4py. Providers own correct
|
||||
communicator-device placement. Raw-pointer providers additionally own
|
||||
allocation lifetime, dtype, and shape correctness.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def __iter__(self) -> Iterator[tuple[str, torch.Tensor]]:
|
||||
"""Yield `(name, local shard)` pairs in the same order as `metadata()`."""
|
||||
raise NotImplementedError
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class M2NManifestEntry:
|
||||
"""One immutable source descriptor with a lazy rank-local tensor view."""
|
||||
|
||||
key: str
|
||||
dtype: torch.dtype
|
||||
global_shape: tuple[int, ...]
|
||||
source_layout: M2NLayout
|
||||
local_tensor: Callable[[], Any]
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
object.__setattr__(self, "global_shape", tuple(self.global_shape))
|
||||
if not self.key:
|
||||
raise ValueError("M2N manifest key must not be empty")
|
||||
if not isinstance(self.dtype, torch.dtype):
|
||||
raise TypeError(
|
||||
f"manifest entry '{self.key}' has invalid dtype {self.dtype!r}"
|
||||
)
|
||||
if not isinstance(self.source_layout, M2NLayout):
|
||||
raise TypeError(
|
||||
f"manifest entry '{self.key}' source_layout must be an M2NLayout"
|
||||
)
|
||||
if not callable(self.local_tensor):
|
||||
raise TypeError(
|
||||
f"manifest entry '{self.key}' local_tensor must be callable"
|
||||
)
|
||||
|
||||
@property
|
||||
def metadata(self) -> M2NParamMeta:
|
||||
"""Return the serializable portion consumed during initialization."""
|
||||
return M2NParamMeta(
|
||||
self.key,
|
||||
self.dtype,
|
||||
self.global_shape,
|
||||
self.source_layout,
|
||||
)
|
||||
|
||||
|
||||
class ManifestM2NWeightSource(M2NWeightSource):
|
||||
"""An ordered M2N source backed by lazy, rank-local manifest entries."""
|
||||
|
||||
def __init__(self, entries: Sequence[M2NManifestEntry]) -> None:
|
||||
self._entries = tuple(entries)
|
||||
self._metadata = tuple(entry.metadata for entry in self._entries)
|
||||
|
||||
def metadata(self) -> list[M2NParamMeta]: # type: ignore[override]
|
||||
"""Return a fresh list while preserving the immutable manifest order."""
|
||||
return list(self._metadata)
|
||||
|
||||
def __iter__(self) -> Iterator[tuple[str, Any]]: # type: ignore[override]
|
||||
"""Materialize one rank-local buffer at a time."""
|
||||
for entry in self._entries:
|
||||
yield entry.key, entry.local_tensor()
|
||||
|
||||
|
||||
def _placement_code(placement: Any) -> int:
|
||||
@@ -69,7 +131,8 @@ def _placement_code(placement: Any) -> int:
|
||||
def mesh_from_tensor(tensor: torch.Tensor, num_trainer_ranks: int) -> M2NMesh:
|
||||
"""The mesh a parameter lives on, as an `M2NMesh`.
|
||||
|
||||
A plain tensor is identical on every trainer rank, so it spans all of them.
|
||||
Tensors without DeviceMesh metadata are treated as replicated. Implicitly
|
||||
sharded tensors require a custom source with an explicit layout.
|
||||
"""
|
||||
device_mesh = getattr(tensor, "device_mesh", None)
|
||||
if device_mesh is None:
|
||||
@@ -122,29 +185,18 @@ class DTensorModuleSource(M2NWeightSource):
|
||||
self._module = module
|
||||
self._num_trainer_ranks = num_trainer_ranks
|
||||
|
||||
def mesh(self) -> M2NMesh:
|
||||
"""The mesh every sharded parameter agrees on.
|
||||
|
||||
Replicated parameters span the trainer without a device mesh of their
|
||||
own, so they never decide this; a model whose *sharded* parameters
|
||||
disagree cannot be described by one side mesh and is rejected.
|
||||
"""
|
||||
meshes = {
|
||||
mesh_from_tensor(p, self._num_trainer_ranks)
|
||||
for _, p in self._module.named_parameters()
|
||||
if placements_from_tensor(p) is not REPLICATED
|
||||
}
|
||||
if len(meshes) > 1:
|
||||
raise ValueError(
|
||||
"nccl_m2n needs one mesh per side, but this module's sharded "
|
||||
f"parameters span several: {sorted(m.dims for m in meshes)}"
|
||||
)
|
||||
return meshes.pop() if meshes else M2NMesh((self._num_trainer_ranks, 1), 0)
|
||||
|
||||
def metadata(self) -> list[ParamMeta]:
|
||||
"""Read global shape/dtype and placements without gathering shards."""
|
||||
def metadata(self) -> list[M2NParamMeta]: # type: ignore[override]
|
||||
"""Read each global shape and source layout without gathering shards."""
|
||||
return [
|
||||
M2NParamMeta(name, p.dtype, tuple(p.shape), placements_from_tensor(p))
|
||||
M2NParamMeta(
|
||||
name,
|
||||
p.dtype,
|
||||
tuple(p.shape),
|
||||
M2NLayout(
|
||||
mesh_from_tensor(p, self._num_trainer_ranks),
|
||||
placements_from_tensor(p),
|
||||
),
|
||||
)
|
||||
for name, p in self._module.named_parameters()
|
||||
]
|
||||
|
||||
|
||||
@@ -13,6 +13,8 @@ engine -- where the `TrainerWeightTransferEngine` contract puts it -- rather tha
|
||||
in every caller.
|
||||
"""
|
||||
|
||||
import socket
|
||||
from collections.abc import Mapping
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, Self, cast
|
||||
@@ -26,39 +28,43 @@ from vllm.distributed.weight_transfer.base import (
|
||||
WeightSource,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.m2n_common import (
|
||||
M2N_WIRE_SCHEMA_VERSION,
|
||||
M2NMesh,
|
||||
M2NParamMeta,
|
||||
M2NWireParam,
|
||||
Placements,
|
||||
check_complete_plan_agreement,
|
||||
check_data_plane_agreement,
|
||||
check_runtime_ready_agreement,
|
||||
check_source_plan_agreement,
|
||||
check_transferable,
|
||||
comm_ptr,
|
||||
create_m2n_communicator,
|
||||
expected_local_shape,
|
||||
import_m2n,
|
||||
prepare_m2n_local_runtime,
|
||||
publish_destination_placements,
|
||||
resolve_layout,
|
||||
source_plan_digest,
|
||||
to_mesh,
|
||||
to_placements,
|
||||
validate_layout,
|
||||
validate_local_buffer,
|
||||
validate_transfer_plan,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.m2n_source import M2NWeightSource
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
decode_nccl_unique_id,
|
||||
trainer_init,
|
||||
uid_init_process_group,
|
||||
stateless_init_metadata_group,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
from vllm.distributed.utils import StatelessProcessGroup
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
__all__ = ["M2NTrainerInitInfo", "M2NTrainerWeightTransferEngine"]
|
||||
|
||||
|
||||
def _dtype_name(dtype: torch.dtype) -> str:
|
||||
"""Short name for a dtype, e.g. `bfloat16`, for the worker-side init dict."""
|
||||
return str(dtype).split(".")[-1]
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class M2NTrainerInitInfo(TrainerInitInfo):
|
||||
"""Trainer-side init info for nccl_m2n.
|
||||
@@ -70,8 +76,8 @@ class M2NTrainerInitInfo(TrainerInitInfo):
|
||||
|
||||
backend: ClassVar[str] = "nccl_m2n"
|
||||
|
||||
master_address: str | None = None
|
||||
master_port: int | None = None
|
||||
master_address: str
|
||||
master_port: int
|
||||
world_size: int
|
||||
"""Trainer ranks + all inference workers."""
|
||||
num_trainer_ranks: int = 1
|
||||
@@ -79,20 +85,34 @@ class M2NTrainerInitInfo(TrainerInitInfo):
|
||||
"""How the inference ranks are laid out: axis 0 replicates and axis 1
|
||||
shards. The trainer declares it so both sides describe the destination
|
||||
identically; defaults to a flat `(num_workers, 1)`, which is all a
|
||||
replicated destination needs."""
|
||||
max_cta: int | None = None
|
||||
replicated destination needs.
|
||||
|
||||
Destination meshes remain common to every parameter; per-parameter
|
||||
destination meshes are intentionally future work.
|
||||
"""
|
||||
nccl_unique_id_b64: str | None = field(default=None, repr=False)
|
||||
"""Optional UID for the NCCL data plane; TCP remains the metadata plane."""
|
||||
max_cta: int | None = None
|
||||
listen_socket: socket.socket | None = field(default=None, repr=False, compare=False)
|
||||
"""Optional rank-zero TCPStore reservation; never sent to workers."""
|
||||
|
||||
@property
|
||||
def destination_mesh_dims(self) -> tuple[int, int]:
|
||||
"""`dst_mesh_dims`, or a flat mesh over every inference worker."""
|
||||
if self.dst_mesh_dims is not None:
|
||||
return self.dst_mesh_dims
|
||||
return cast(tuple[int, int], tuple(self.dst_mesh_dims))
|
||||
return (self.world_size - self.num_trainer_ranks, 1)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
"""Reject rank counts or destination meshes that cannot form a group."""
|
||||
_ = self.nccl_unique_id_bytes
|
||||
if not isinstance(self.master_address, str) or not self.master_address:
|
||||
raise ValueError("`master_address` must name the metadata rendezvous")
|
||||
if (
|
||||
not isinstance(self.master_port, int)
|
||||
or isinstance(self.master_port, bool)
|
||||
or not 0 < self.master_port < 65536
|
||||
):
|
||||
raise ValueError("`master_port` must be a valid metadata port")
|
||||
if not 0 < self.num_trainer_ranks < self.world_size:
|
||||
raise ValueError(
|
||||
f"`num_trainer_ranks` ({self.num_trainer_ranks}) must leave at "
|
||||
@@ -101,7 +121,11 @@ class M2NTrainerInitInfo(TrainerInitInfo):
|
||||
num_workers = self.world_size - self.num_trainer_ranks
|
||||
if self.dst_mesh_dims is not None:
|
||||
dims = tuple(self.dst_mesh_dims)
|
||||
if len(dims) != 2 or dims[0] * dims[1] != num_workers:
|
||||
try:
|
||||
mesh = M2NMesh(cast(tuple[int, int], dims), self.num_trainer_ranks)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError(f"invalid `dst_mesh_dims` {dims}: {exc}") from exc
|
||||
if mesh.size != num_workers:
|
||||
raise ValueError(
|
||||
f"`dst_mesh_dims` {dims} must be two dims covering the "
|
||||
f"{num_workers} inference workers"
|
||||
@@ -111,14 +135,23 @@ class M2NTrainerInitInfo(TrainerInitInfo):
|
||||
f"trainer `rank` ({self.rank}) must be below "
|
||||
f"`num_trainer_ranks` ({self.num_trainer_ranks})"
|
||||
)
|
||||
if self.listen_socket is not None:
|
||||
if self.rank != 0:
|
||||
raise ValueError("only trainer rank zero may reserve the TCPStore")
|
||||
if not isinstance(self.listen_socket, socket.socket):
|
||||
raise TypeError("listen_socket must be a socket.socket")
|
||||
if self.listen_socket.fileno() < 0:
|
||||
raise ValueError("listen_socket is already closed")
|
||||
|
||||
@property
|
||||
def nccl_unique_id_bytes(self) -> bytes | None:
|
||||
if self.nccl_unique_id_b64 is None:
|
||||
return None
|
||||
return decode_nccl_unique_id(
|
||||
master_address=self.master_address,
|
||||
master_port=self.master_port,
|
||||
master_address=None,
|
||||
master_port=None,
|
||||
nccl_unique_id_b64=self.nccl_unique_id_b64,
|
||||
ctx="M2NTrainerInitInfo",
|
||||
ctx="M2NTrainerInitInfo data plane",
|
||||
)
|
||||
|
||||
|
||||
@@ -136,22 +169,20 @@ class M2NTrainerWeightTransferEngine(TrainerWeightTransferEngine[M2NTrainerInitI
|
||||
*,
|
||||
client: VLLMWeightSyncClient,
|
||||
source: M2NWeightSource,
|
||||
is_controller: bool,
|
||||
num_trainer_ranks: int,
|
||||
is_sender: bool,
|
||||
) -> None:
|
||||
"""Hold the client and source; `trainer_init` fills in the group."""
|
||||
super().__init__(client=client, source=source, is_sender=is_controller)
|
||||
self.source: M2NWeightSource = source
|
||||
self.is_controller = is_controller
|
||||
self.num_trainer_ranks = num_trainer_ranks
|
||||
self.group: PyNcclCommunicator | None = None
|
||||
super().__init__(client=client, source=source, is_sender=is_sender)
|
||||
self.group: Any = None
|
||||
self._m2n: Any = None
|
||||
self._handle: Any = None
|
||||
self._metas: list[M2NParamMeta] = []
|
||||
self._src_mesh: M2NMesh | None = None
|
||||
self._wire_params: list[M2NWireParam] = []
|
||||
self._source_digest = ""
|
||||
self._dst_mesh: M2NMesh | None = None
|
||||
self._dst_placements: list[Placements | None] = []
|
||||
self._executor: ThreadPoolExecutor | None = None
|
||||
self._metadata_group: StatelessProcessGroup | None = None
|
||||
self._num_trainer_ranks = 0
|
||||
|
||||
@classmethod
|
||||
def trainer_init(
|
||||
@@ -168,204 +199,351 @@ class M2NTrainerWeightTransferEngine(TrainerWeightTransferEngine[M2NTrainerInitI
|
||||
build local state and join the communicator. Every rank participates
|
||||
in every reshard and sends its local shard.
|
||||
|
||||
The ordering here is not arbitrary -- see the comments inline. In
|
||||
short: validate before anything blocks; start the init RPC without
|
||||
waiting (workers join the communicator inside that RPC, so it cannot
|
||||
return until this side has joined too); then join the NCCL
|
||||
communicator; then create the M2N handle; and only then wait for the RPC
|
||||
to finish.
|
||||
Build a local validation envelope, start the worker RPC without
|
||||
waiting, join the metadata rendezvous, and require unanimous source,
|
||||
data-plane, and runtime readiness before constructing NCCL.
|
||||
"""
|
||||
# The base WeightSource carries name/dtype/shape but not layout, which
|
||||
# is not enough to plan a reshard. Fail here rather than at the first
|
||||
# missing attribute deep inside the send loop.
|
||||
if not isinstance(source, M2NWeightSource):
|
||||
raise TypeError(
|
||||
"nccl_m2n needs per-parameter layouts, so its source must be an "
|
||||
f"M2NWeightSource (e.g. DTensorModuleSource); got "
|
||||
f"{type(source).__name__}"
|
||||
)
|
||||
|
||||
# Trainer-side weight transfer engine for this rank. `__init__` only
|
||||
# holds the client and source; the communicator, meshes, and M2N handle
|
||||
# are filled in below.
|
||||
engine = cls(
|
||||
client=client,
|
||||
source=source,
|
||||
is_controller=init_info.rank == 0,
|
||||
num_trainer_ranks=init_info.num_trainer_ranks,
|
||||
source=cast(M2NWeightSource, source),
|
||||
is_sender=init_info.is_sender,
|
||||
)
|
||||
engine._m2n = import_m2n()
|
||||
|
||||
engine._src_mesh = source.mesh()
|
||||
if engine._src_mesh.size != init_info.num_trainer_ranks:
|
||||
raise ValueError(
|
||||
f"source mesh covers {engine._src_mesh.size} ranks, but there "
|
||||
f"are {init_info.num_trainer_ranks} trainer ranks"
|
||||
)
|
||||
|
||||
# Validate the whole plan before any rendezvous. A layout m2n cannot
|
||||
# express is a mismatched collective at send time, which hangs rather
|
||||
# than raises -- so every precondition is checked while an exception can
|
||||
# still propagate to the caller.
|
||||
engine._metas = cast(list[M2NParamMeta], source.metadata())
|
||||
for meta in engine._metas:
|
||||
check_transferable(meta.name, meta.dtype, meta.shape)
|
||||
mesh, placements = resolve_layout(engine._src_mesh, meta.placements)
|
||||
validate_layout(mesh, placements, meta.shape, "source")
|
||||
|
||||
rendezvous: Future | None = None
|
||||
if engine.is_controller:
|
||||
engine._executor = ThreadPoolExecutor(
|
||||
max_workers=1, thread_name_prefix="m2n-weight-sync"
|
||||
)
|
||||
# Only trainer rank 0 starts the control-plane RPC. Workers join
|
||||
# the communicator inside it, so submit it first, join below, and
|
||||
# wait for it at the end.
|
||||
rendezvous = engine._executor.submit(
|
||||
client.init_weight_transfer_engine,
|
||||
engine._worker_init_info(init_info),
|
||||
)
|
||||
|
||||
# Every trainer rank joins -- reshard is collective over the whole
|
||||
# communicator. Trainer ranks take
|
||||
# [0, num_trainer_ranks) and the workers follow, which is the
|
||||
# contiguous-interval layout m2n requires of both meshes.
|
||||
unique_id_bytes = init_info.nccl_unique_id_bytes
|
||||
if unique_id_bytes is not None:
|
||||
engine.group = uid_init_process_group(
|
||||
unique_id_bytes,
|
||||
rank=init_info.rank,
|
||||
world_size=init_info.world_size,
|
||||
device=torch.accelerator.current_device_index(),
|
||||
)
|
||||
else:
|
||||
assert init_info.master_address is not None
|
||||
assert init_info.master_port is not None
|
||||
engine.group = trainer_init(
|
||||
{
|
||||
"master_address": init_info.master_address,
|
||||
"master_port": init_info.master_port,
|
||||
"world_size": init_info.world_size,
|
||||
},
|
||||
rank=init_info.rank,
|
||||
)
|
||||
|
||||
# The trainer declares the inference topology; the worker uses exactly
|
||||
# this, so neither side has to infer the other's factorization.
|
||||
engine._dst_mesh = M2NMesh(
|
||||
init_info.destination_mesh_dims, init_info.num_trainer_ranks
|
||||
)
|
||||
engine._dst_placements = engine._receive_destination_plan(
|
||||
init_info.num_trainer_ranks
|
||||
)
|
||||
logger.info(
|
||||
"nccl_m2n trainer ready: %d parameters, trainer ranks [0, %d), "
|
||||
"inference ranks [%d, %d)",
|
||||
len(engine._metas),
|
||||
init_info.num_trainer_ranks,
|
||||
init_info.num_trainer_ranks,
|
||||
init_info.world_size,
|
||||
)
|
||||
|
||||
# Created after the group so a failed rendezvous does not leave a live
|
||||
# handle behind; shutdown() releases it once caller streams are synced.
|
||||
engine._handle = engine._m2n.Handle.create(
|
||||
engine._m2n.Config(max_cta=init_info.max_cta)
|
||||
)
|
||||
|
||||
# Now safe to collect: both sides have joined, so the RPC has returned
|
||||
# or raised. Doing this last means a worker-side init failure surfaces
|
||||
# here, as an exception from trainer_init, rather than at the first send.
|
||||
if rendezvous is not None:
|
||||
rendezvous.result()
|
||||
engine._initialize(init_info, source)
|
||||
return engine
|
||||
|
||||
def _receive_destination_plan(
|
||||
self, first_worker_rank: int
|
||||
) -> list[Placements | None]:
|
||||
"""Receive and validate the inference-side per-parameter placements."""
|
||||
if self.group is None or self._dst_mesh is None:
|
||||
raise RuntimeError(
|
||||
"destination plan requires the communicator and destination mesh"
|
||||
def _initialize(
|
||||
self,
|
||||
init_info: M2NTrainerInitInfo,
|
||||
source: WeightSource | None,
|
||||
) -> None:
|
||||
"""Run source validation, metadata agreement, and NCCL rendezvous."""
|
||||
self._num_trainer_ranks = init_info.num_trainer_ranks
|
||||
local_source_error: str | None = None
|
||||
try:
|
||||
if not isinstance(source, M2NWeightSource):
|
||||
raise TypeError(
|
||||
"nccl_m2n needs per-parameter layouts, so its source must "
|
||||
f"be an M2NWeightSource; got {type(source).__name__}"
|
||||
)
|
||||
self._m2n = import_m2n()
|
||||
self._prepare_source_plan(source, init_info.num_trainer_ranks)
|
||||
except Exception as exc:
|
||||
local_source_error = f"{type(exc).__name__}: {exc}"
|
||||
self._metas = []
|
||||
self._wire_params = []
|
||||
self._source_digest = source_plan_digest([])
|
||||
|
||||
executor: ThreadPoolExecutor | None = None
|
||||
init_future: Future[None] | None = None
|
||||
if self.is_sender:
|
||||
executor = ThreadPoolExecutor(
|
||||
max_workers=1, thread_name_prefix="m2n-weight-sync"
|
||||
)
|
||||
init_future = executor.submit(
|
||||
self.client.init_weight_transfer_engine,
|
||||
self._worker_init_info(init_info),
|
||||
)
|
||||
handle = None
|
||||
group = None
|
||||
try:
|
||||
reservation = init_info.listen_socket
|
||||
try:
|
||||
metadata_group = stateless_init_metadata_group(
|
||||
init_info.master_address,
|
||||
init_info.master_port,
|
||||
init_info.rank,
|
||||
init_info.world_size,
|
||||
listen_socket=reservation,
|
||||
)
|
||||
finally:
|
||||
init_info.listen_socket = None
|
||||
if reservation is not None:
|
||||
reservation.close()
|
||||
plan_digest = check_source_plan_agreement(
|
||||
metadata_group,
|
||||
self._wire_params,
|
||||
expected_digest=self._source_digest,
|
||||
local_error=local_source_error,
|
||||
)
|
||||
|
||||
# Retain the common destination mesh and the merged NCCL publication
|
||||
# of per-parameter placements.
|
||||
self._dst_mesh = M2NMesh(
|
||||
init_info.destination_mesh_dims, init_info.num_trainer_ranks
|
||||
)
|
||||
unique_id_bytes: bytes | None = None
|
||||
data_plane_error: str | None = None
|
||||
try:
|
||||
unique_id_bytes = init_info.nccl_unique_id_bytes
|
||||
except Exception as exc:
|
||||
data_plane_error = f"{type(exc).__name__}: {exc}"
|
||||
check_data_plane_agreement(
|
||||
metadata_group,
|
||||
unique_id_bytes,
|
||||
data_plane_error,
|
||||
)
|
||||
|
||||
runtime_error: str | None = None
|
||||
nccl = None
|
||||
device = None
|
||||
try:
|
||||
nccl, device, handle = prepare_m2n_local_runtime(
|
||||
self._m2n, init_info.max_cta
|
||||
)
|
||||
except Exception as exc:
|
||||
runtime_error = f"{type(exc).__name__}: {exc}"
|
||||
check_runtime_ready_agreement(metadata_group, runtime_error)
|
||||
assert nccl is not None and device is not None and handle is not None
|
||||
group = create_m2n_communicator(nccl, metadata_group, unique_id_bytes)
|
||||
destination_placements = self._receive_destination_plan(
|
||||
group,
|
||||
init_info.num_trainer_ranks,
|
||||
torch.device("cuda", device),
|
||||
)
|
||||
|
||||
complete_plan_error: str | None = None
|
||||
try:
|
||||
assert self._dst_mesh is not None
|
||||
validate_transfer_plan(
|
||||
self._metas, self._dst_mesh, destination_placements
|
||||
)
|
||||
except Exception as exc:
|
||||
complete_plan_error = f"{type(exc).__name__}: {exc}"
|
||||
assert self._dst_mesh is not None
|
||||
check_complete_plan_agreement(
|
||||
metadata_group,
|
||||
plan_digest,
|
||||
self._dst_mesh,
|
||||
destination_placements,
|
||||
complete_plan_error,
|
||||
)
|
||||
|
||||
if init_future is not None:
|
||||
init_future.result()
|
||||
|
||||
logger.info(
|
||||
"nccl_m2n trainer ready: %d parameters, trainer ranks [0, %d), "
|
||||
"inference ranks [%d, %d)",
|
||||
len(self._metas),
|
||||
init_info.num_trainer_ranks,
|
||||
init_info.num_trainer_ranks,
|
||||
init_info.world_size,
|
||||
)
|
||||
logger.info("nccl_m2n source plan digest: %s", plan_digest)
|
||||
except BaseException:
|
||||
if group is not None:
|
||||
try:
|
||||
# Transport cleanup only; reload validity belongs to the
|
||||
# model-update lifecycle, not this backend.
|
||||
group.abort()
|
||||
except BaseException:
|
||||
logger.exception(
|
||||
"failed to abort partially initialized nccl_m2n communicator"
|
||||
)
|
||||
if handle is not None:
|
||||
try:
|
||||
handle.destroy()
|
||||
except BaseException:
|
||||
logger.exception(
|
||||
"failed to destroy partially initialized nccl_m2n handle"
|
||||
)
|
||||
raise
|
||||
finally:
|
||||
if executor is not None:
|
||||
executor.shutdown(wait=False, cancel_futures=True)
|
||||
|
||||
self._metadata_group = metadata_group
|
||||
self._handle = handle
|
||||
self.group = group
|
||||
self._dst_placements = destination_placements
|
||||
|
||||
def _receive_destination_plan(
|
||||
self, group: Any, first_worker_rank: int, device: torch.device
|
||||
) -> list[Placements | None]:
|
||||
"""Receive the inference-side per-parameter placements."""
|
||||
# The first inference worker publishes one placement per parameter.
|
||||
# Every trainer rank receives the complete plan in this single,
|
||||
# initialization-time broadcast.
|
||||
placements = publish_destination_placements(
|
||||
self.group, first_worker_rank, None, len(self._metas)
|
||||
group,
|
||||
device,
|
||||
torch.cuda.current_stream(),
|
||||
first_worker_rank,
|
||||
None,
|
||||
len(self._metas),
|
||||
)
|
||||
# The send loop matches placements to metadata by position, so reject
|
||||
# an incomplete or oversized plan before entering any M2N collective.
|
||||
if len(placements) != len(self._metas):
|
||||
raise RuntimeError(
|
||||
f"inference side planned {len(placements)} parameters, but "
|
||||
f"this trainer announced {len(self._metas)}"
|
||||
)
|
||||
# Check every worker-selected destination against the corresponding
|
||||
# global parameter shape before the first transfer.
|
||||
for meta, placement in zip(self._metas, placements):
|
||||
mesh, resolved = resolve_layout(self._dst_mesh, placement)
|
||||
validate_layout(mesh, resolved, meta.shape, "destination")
|
||||
return placements
|
||||
|
||||
def _worker_init_info(self, init_info: M2NTrainerInitInfo) -> dict[str, Any]:
|
||||
"""Handshake payload: rendezvous, both meshes, and the transfer plan."""
|
||||
assert self._src_mesh is not None
|
||||
payload: dict[str, Any] = {
|
||||
"schema_version": M2N_WIRE_SCHEMA_VERSION,
|
||||
"master_address": init_info.master_address,
|
||||
"master_port": init_info.master_port,
|
||||
"nccl_unique_id_b64": init_info.nccl_unique_id_b64,
|
||||
"rank_offset": init_info.num_trainer_ranks,
|
||||
"world_size": init_info.world_size,
|
||||
"src_mesh_dims": list(self._src_mesh.dims),
|
||||
"dst_mesh_dims": list(init_info.destination_mesh_dims),
|
||||
"names": [m.name for m in self._metas],
|
||||
"dtype_names": [_dtype_name(m.dtype) for m in self._metas],
|
||||
"shapes": [list(m.shape) for m in self._metas],
|
||||
"src_placements": [
|
||||
None if m.placements is None else list(m.placements)
|
||||
for m in self._metas
|
||||
],
|
||||
"source_digest": self._source_digest,
|
||||
"params": [param.to_dict() for param in self._wire_params],
|
||||
"max_cta": init_info.max_cta,
|
||||
}
|
||||
return {key: value for key, value in payload.items() if value is not None}
|
||||
|
||||
def _prepare_source_plan(
|
||||
self, source: M2NWeightSource, num_trainer_ranks: int
|
||||
) -> None:
|
||||
"""Build and validate this rank's immutable source plan."""
|
||||
self._metas = source.metadata()
|
||||
names: set[str] = set()
|
||||
for meta in self._metas:
|
||||
if not isinstance(meta, M2NParamMeta):
|
||||
raise TypeError(
|
||||
"nccl_m2n source metadata must contain M2NParamMeta "
|
||||
f"entries, got {type(meta).__name__}"
|
||||
)
|
||||
if meta.name in names:
|
||||
raise ValueError(
|
||||
"nccl_m2n source metadata contains duplicate parameter "
|
||||
f"'{meta.name}'"
|
||||
)
|
||||
names.add(meta.name)
|
||||
check_transferable(meta.name, meta.dtype, meta.shape)
|
||||
layout = meta.source_layout
|
||||
if layout.mesh.start_rank != 0:
|
||||
raise ValueError(
|
||||
f"parameter '{meta.name}' source mesh must start at rank 0, "
|
||||
f"got {layout.mesh.start_rank}"
|
||||
)
|
||||
if layout.mesh.size != num_trainer_ranks:
|
||||
raise ValueError(
|
||||
f"parameter '{meta.name}' source mesh covers "
|
||||
f"{layout.mesh.size} ranks, but there are "
|
||||
f"{num_trainer_ranks} trainer ranks"
|
||||
)
|
||||
expected_local_shape(meta)
|
||||
self._wire_params = [
|
||||
M2NWireParam(
|
||||
name=meta.name,
|
||||
dtype_name=str(meta.dtype).split(".")[-1],
|
||||
shape=meta.shape,
|
||||
src_mesh_dims=meta.source_layout.mesh.dims,
|
||||
src_placements=meta.source_layout.placements,
|
||||
)
|
||||
for meta in self._metas
|
||||
]
|
||||
self._source_digest = source_plan_digest(self._wire_params)
|
||||
|
||||
def send_weights(self) -> None:
|
||||
"""Drive one update round: start, reshard concurrently, then finish."""
|
||||
if self._handle is None or self.group is None:
|
||||
raise RuntimeError("nccl_m2n trainer engine is not initialized")
|
||||
|
||||
update: Future | None = None
|
||||
if self.is_controller:
|
||||
self.client.start_weight_update()
|
||||
# The workers reshard from inside `update_weights`, concurrently
|
||||
# with the sends below.
|
||||
assert self._executor is not None
|
||||
update = self._executor.submit(
|
||||
self.client.update_weights,
|
||||
{"names": [m.name for m in self._metas]},
|
||||
)
|
||||
if not self.is_sender:
|
||||
try:
|
||||
self._send()
|
||||
except BaseException:
|
||||
try:
|
||||
self._receive_round_outcome()
|
||||
except BaseException:
|
||||
logger.exception(
|
||||
"failed to consume nccl_m2n round outcome after local failure"
|
||||
)
|
||||
raise
|
||||
self._receive_round_outcome()
|
||||
return
|
||||
|
||||
try:
|
||||
self._send()
|
||||
finally:
|
||||
if update is not None:
|
||||
self.client.start_weight_update()
|
||||
executor = ThreadPoolExecutor(
|
||||
max_workers=1, thread_name_prefix="m2n-weight-sync"
|
||||
)
|
||||
try:
|
||||
update = executor.submit(
|
||||
self.client.update_weights,
|
||||
{"names": [m.name for m in self._metas]},
|
||||
)
|
||||
# Surface an immediate worker-side rejection before trainers
|
||||
# enter the collective sequence.
|
||||
if update.done():
|
||||
update.result()
|
||||
self._send()
|
||||
update.result()
|
||||
|
||||
if self.is_controller:
|
||||
finally:
|
||||
executor.shutdown(wait=False, cancel_futures=True)
|
||||
self.client.finish_weight_update()
|
||||
except BaseException as exc:
|
||||
try:
|
||||
self._publish_round_outcome(exc)
|
||||
except BaseException:
|
||||
logger.exception("failed to publish nccl_m2n round failure")
|
||||
raise
|
||||
|
||||
self._publish_round_outcome(None)
|
||||
|
||||
def _publish_round_outcome(self, failure: BaseException | None) -> None:
|
||||
"""Tell non-sender trainer ranks whether worker finalization passed."""
|
||||
if self._num_trainer_ranks <= 1:
|
||||
return
|
||||
if self._metadata_group is None:
|
||||
raise RuntimeError("nccl_m2n metadata group is unavailable")
|
||||
envelope = {
|
||||
"ok": failure is None,
|
||||
"error": (
|
||||
None if failure is None else f"{type(failure).__name__}: {failure}"
|
||||
),
|
||||
}
|
||||
for rank in range(1, self._num_trainer_ranks):
|
||||
self._metadata_group.send_obj(envelope, rank)
|
||||
# Keep rank zero's TCPStore alive until every trainer consumed its
|
||||
# outcome. This also keeps the per-rank message counters aligned.
|
||||
expected_ack = {"phase": "round_outcome_consumed"}
|
||||
for rank in range(1, self._num_trainer_ranks):
|
||||
ack = self._metadata_group.recv_obj(rank)
|
||||
if ack != expected_ack:
|
||||
raise RuntimeError(
|
||||
f"trainer rank {rank} sent an invalid round-outcome acknowledgement"
|
||||
)
|
||||
|
||||
def _receive_round_outcome(self) -> None:
|
||||
"""Wait until trainer rank zero observed every worker's finish RPC."""
|
||||
if self._metadata_group is None:
|
||||
raise RuntimeError("nccl_m2n metadata group is unavailable")
|
||||
outcome = self._metadata_group.recv_obj(0)
|
||||
self._metadata_group.send_obj({"phase": "round_outcome_consumed"}, 0)
|
||||
if not isinstance(outcome, Mapping) or set(outcome) != {"ok", "error"}:
|
||||
raise RuntimeError("trainer rank zero sent an invalid round outcome")
|
||||
ok = outcome["ok"]
|
||||
error = outcome["error"]
|
||||
if not isinstance(ok, bool) or ok != (error is None):
|
||||
raise RuntimeError("trainer rank zero sent an invalid round outcome")
|
||||
if not ok:
|
||||
if not isinstance(error, str) or not error:
|
||||
raise RuntimeError("trainer rank zero sent an invalid round error")
|
||||
raise RuntimeError(
|
||||
"nccl_m2n worker update failed on trainer rank zero: " + error
|
||||
)
|
||||
|
||||
def _send(self) -> None:
|
||||
"""Reshard each local shard into its worker-planned destination."""
|
||||
assert self.group is not None
|
||||
assert self._src_mesh is not None
|
||||
assert self._dst_mesh is not None
|
||||
assert self.source is not None
|
||||
m2n = self._m2n
|
||||
comm = comm_ptr(self.group)
|
||||
stream = torch.cuda.current_stream()
|
||||
|
||||
for index, (name, tensor) in enumerate(self.source):
|
||||
device_index = torch.accelerator.current_device_index()
|
||||
# Optimizer work may have used another CUDA stream. Make prior writes
|
||||
# visible before a provider materializes current-stream local shards.
|
||||
torch.accelerator.synchronize(device_index)
|
||||
inflight_buffers: list[Any] = []
|
||||
sent = 0
|
||||
for index, (name, buffer) in enumerate(self.source):
|
||||
if index >= len(self._metas):
|
||||
raise RuntimeError(
|
||||
"weight source yielded more parameters than metadata() "
|
||||
f"declared (first extra parameter: '{name}')"
|
||||
)
|
||||
meta = self._metas[index]
|
||||
if name != meta.name:
|
||||
raise RuntimeError(
|
||||
@@ -373,28 +551,47 @@ class M2NTrainerWeightTransferEngine(TrainerWeightTransferEngine[M2NTrainerInitI
|
||||
f"its metadata declared '{meta.name}'; the source must "
|
||||
"iterate in a stable order"
|
||||
)
|
||||
shard = tensor.detach().contiguous()
|
||||
src_mesh, src_placements = resolve_layout(self._src_mesh, meta.placements)
|
||||
validate_local_buffer(m2n, meta, buffer)
|
||||
src_mesh, src_placements = resolve_layout(
|
||||
meta.source_layout.mesh, meta.source_layout.placements
|
||||
)
|
||||
dst_mesh, dst_placements = resolve_layout(
|
||||
self._dst_mesh, self._dst_placements[index]
|
||||
)
|
||||
raw_pointer_args = {}
|
||||
if isinstance(buffer, int):
|
||||
raw_pointer_args = {
|
||||
"src_local_shape": expected_local_shape(meta),
|
||||
"src_dtype": meta.dtype,
|
||||
}
|
||||
m2n.reshard(
|
||||
shard,
|
||||
buffer,
|
||||
None,
|
||||
comm,
|
||||
self.group,
|
||||
stream,
|
||||
src_mesh=to_mesh(m2n, src_mesh),
|
||||
src_placements=to_placements(m2n, src_placements),
|
||||
dst_mesh=to_mesh(m2n, dst_mesh),
|
||||
dst_placements=to_placements(m2n, dst_placements),
|
||||
handle=self._handle,
|
||||
**raw_pointer_args,
|
||||
)
|
||||
# Let allocator-aware buffers recycle storage in stream order.
|
||||
# Other object owners must stay alive until the final synchronize;
|
||||
# raw-pointer lifetime remains the source's responsibility.
|
||||
record_stream = getattr(buffer, "record_stream", None)
|
||||
if callable(record_stream):
|
||||
record_stream(stream)
|
||||
elif not isinstance(buffer, int):
|
||||
inflight_buffers.append(buffer)
|
||||
sent += 1
|
||||
|
||||
if sent != len(self._metas):
|
||||
missing = self._metas[sent].name
|
||||
raise RuntimeError(
|
||||
f"weight source yielded {sent} parameters, but metadata() "
|
||||
f"declared {len(self._metas)} (first missing: '{missing}')"
|
||||
)
|
||||
# A custom source may yield temporary storage allocated on another
|
||||
# stream. M2N retains only its device pointer for asynchronous work,
|
||||
# so tell the allocator not to reuse that storage until this stream
|
||||
# has finished with it. For storage allocated on this stream,
|
||||
# record_stream is effectively a no-op.
|
||||
shard.record_stream(stream)
|
||||
|
||||
# As in the other weight-transfer backends, wait before returning
|
||||
# rather than relying on same-stream ordering. The caller may then
|
||||
@@ -402,14 +599,30 @@ class M2NTrainerWeightTransferEngine(TrainerWeightTransferEngine[M2NTrainerInitI
|
||||
stream.synchronize()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Destroy the m2n handle, join the helper thread, and drop the group."""
|
||||
if self._handle is not None:
|
||||
torch.accelerator.synchronize()
|
||||
self._handle.destroy()
|
||||
self._handle = None
|
||||
if self._executor is not None:
|
||||
self._executor.shutdown()
|
||||
self._executor = None
|
||||
if self.group is not None:
|
||||
self.group.destroy()
|
||||
self.group = None
|
||||
"""Finish pending GPU work and release M2N communication state."""
|
||||
handle, self._handle = self._handle, None
|
||||
group, self.group = self.group, None
|
||||
self._metadata_group = None
|
||||
self._dst_placements = []
|
||||
self._dst_mesh = None
|
||||
self._m2n = None
|
||||
|
||||
first_error: BaseException | None = None
|
||||
|
||||
def attempt(action: Any, message: str) -> None:
|
||||
nonlocal first_error
|
||||
try:
|
||||
action()
|
||||
except BaseException as exc:
|
||||
if first_error is None:
|
||||
first_error = exc
|
||||
logger.exception(message)
|
||||
|
||||
if handle is not None or group is not None:
|
||||
attempt(torch.accelerator.synchronize, "failed to synchronize nccl_m2n")
|
||||
if handle is not None:
|
||||
attempt(handle.destroy, "failed to destroy nccl_m2n handle")
|
||||
if group is not None:
|
||||
attempt(group.abort, "failed to abort nccl_m2n communicator")
|
||||
if first_error is not None:
|
||||
raise first_error
|
||||
|
||||
@@ -8,6 +8,7 @@ The dense (`NCCLWeightTransferEngine`) and sparse
|
||||
sparse engine does not have to subclass the dense one.
|
||||
"""
|
||||
|
||||
import socket
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from typing import TYPE_CHECKING, Protocol
|
||||
|
||||
@@ -17,6 +18,7 @@ import torch
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
from vllm.distributed.utils import StatelessProcessGroup
|
||||
|
||||
from vllm.distributed.device_communicators.pynccl_wrapper import (
|
||||
NCCL_UNIQUE_ID_BYTES,
|
||||
@@ -136,6 +138,26 @@ class NCCLRendezvous(Protocol):
|
||||
world_size: int
|
||||
|
||||
|
||||
def stateless_init_metadata_group(
|
||||
master_address: str,
|
||||
master_port: int,
|
||||
rank: int,
|
||||
world_size: int,
|
||||
*,
|
||||
listen_socket: socket.socket | None = None,
|
||||
) -> "StatelessProcessGroup":
|
||||
"""Create only the TCPStore group used for metadata agreement."""
|
||||
from vllm.distributed.utils import StatelessProcessGroup
|
||||
|
||||
return StatelessProcessGroup.create(
|
||||
host=master_address,
|
||||
port=master_port,
|
||||
rank=rank,
|
||||
world_size=world_size,
|
||||
listen_socket=listen_socket,
|
||||
)
|
||||
|
||||
|
||||
def stateless_init_process_group(
|
||||
master_address: str,
|
||||
master_port: int,
|
||||
@@ -181,6 +203,29 @@ def uid_init_process_group(
|
||||
)
|
||||
|
||||
|
||||
def _worker_group_rank(rank_offset: int, parallel_config: "ParallelConfig") -> int:
|
||||
"""Return this worker's rank in a trainer-to-worker process group."""
|
||||
dp_rank = parallel_config.data_parallel_index
|
||||
world_size_per_dp = parallel_config.world_size # TP * PP
|
||||
rank_within_dp = parallel_config.rank
|
||||
return rank_offset + dp_rank * world_size_per_dp + rank_within_dp
|
||||
|
||||
|
||||
def worker_init_metadata_group(
|
||||
init_info: NCCLWeightTransferInitInfo,
|
||||
parallel_config: "ParallelConfig",
|
||||
) -> "StatelessProcessGroup":
|
||||
"""Join the trainer-to-worker TCP metadata group without constructing NCCL."""
|
||||
assert init_info.master_address is not None
|
||||
assert init_info.master_port is not None
|
||||
return stateless_init_metadata_group(
|
||||
init_info.master_address,
|
||||
init_info.master_port,
|
||||
_worker_group_rank(init_info.rank_offset, parallel_config),
|
||||
init_info.world_size,
|
||||
)
|
||||
|
||||
|
||||
def worker_init_process_group(
|
||||
init_info: NCCLWeightTransferInitInfo,
|
||||
parallel_config: "ParallelConfig",
|
||||
@@ -190,15 +235,7 @@ def worker_init_process_group(
|
||||
Computes a unique rank for this worker across all data-parallel groups and
|
||||
joins the trainer via whichever rendezvous mode `init_info` carries.
|
||||
"""
|
||||
# Calculate the global rank in the trainer-worker process group.
|
||||
# Must account for data parallel to get unique ranks across all workers.
|
||||
dp_rank = parallel_config.data_parallel_index
|
||||
world_size_per_dp = parallel_config.world_size # TP * PP
|
||||
rank_within_dp = parallel_config.rank
|
||||
|
||||
# Unique rank across all DP groups
|
||||
worker_rank = dp_rank * world_size_per_dp + rank_within_dp
|
||||
rank = worker_rank + init_info.rank_offset
|
||||
rank = _worker_group_rank(init_info.rank_offset, parallel_config)
|
||||
|
||||
device = torch.accelerator.current_device_index()
|
||||
unique_id_bytes = init_info.nccl_unique_id_bytes
|
||||
|
||||
@@ -820,7 +820,14 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
elif isinstance(generator, ChatCompletionResponse):
|
||||
return self.messages_full_converter(generator)
|
||||
|
||||
return self.message_stream_converter(generator)
|
||||
return generator
|
||||
|
||||
def chat_completion_stream_generator(
|
||||
self, *args: Any, **kwargs: Any
|
||||
) -> AsyncGenerator[str, None]:
|
||||
return self.message_stream_converter(
|
||||
self.chat_completion_stream_generator_raw(*args, **kwargs)
|
||||
)
|
||||
|
||||
def messages_full_converter(
|
||||
self,
|
||||
@@ -881,7 +888,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
|
||||
async def message_stream_converter(
|
||||
self,
|
||||
generator: AsyncGenerator[str, None],
|
||||
generator: AsyncGenerator[ChatCompletionStreamResponse | ErrorResponse, None],
|
||||
) -> AsyncGenerator[str, None]:
|
||||
try:
|
||||
|
||||
@@ -991,238 +998,208 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
events.extend(stop_active_block())
|
||||
return events
|
||||
|
||||
async for item in generator:
|
||||
if item.startswith("data:"):
|
||||
data_str = item[5:].strip().rstrip("\n")
|
||||
if data_str == "[DONE]":
|
||||
for event in stop_and_flush():
|
||||
yield event
|
||||
stop_message = AnthropicStreamEvent(
|
||||
type="message_stop",
|
||||
)
|
||||
data = stop_message.model_dump_json(
|
||||
exclude_unset=True, exclude_none=True
|
||||
)
|
||||
yield wrap_data_with_event(data, "message_stop")
|
||||
else:
|
||||
origin_chunk = ChatCompletionStreamResponse.model_validate_json(
|
||||
data_str
|
||||
)
|
||||
|
||||
if first_item:
|
||||
chunk = AnthropicStreamEvent(
|
||||
type="message_start",
|
||||
message=AnthropicMessagesResponse(
|
||||
id=origin_chunk.id,
|
||||
# Set explicitly: this event is serialized
|
||||
# with exclude_unset=True, which drops
|
||||
# default-valued fields, while strict
|
||||
# Anthropic SDK clients require
|
||||
# message.type/role (issue #45367).
|
||||
type="message",
|
||||
role="assistant",
|
||||
content=[],
|
||||
model=origin_chunk.model,
|
||||
stop_reason=None,
|
||||
stop_sequence=None,
|
||||
usage=_build_anthropic_usage(origin_chunk.usage),
|
||||
),
|
||||
)
|
||||
first_item = False
|
||||
data = chunk.model_dump_json(exclude_unset=True)
|
||||
yield wrap_data_with_event(data, "message_start")
|
||||
continue
|
||||
|
||||
# last chunk including usage info
|
||||
if len(origin_chunk.choices) == 0:
|
||||
for event in stop_and_flush():
|
||||
yield event
|
||||
anthropic_stop_reason, anthropic_stop_sequence = (
|
||||
_to_anthropic_stop_reason(
|
||||
finish_reason or "stop",
|
||||
stop_sequence,
|
||||
tool_used=tool_used,
|
||||
tool_complete=(
|
||||
tool_used and _is_json("".join(last_tool_args))
|
||||
),
|
||||
)
|
||||
)
|
||||
stop_delta = AnthropicDelta(
|
||||
stop_reason=anthropic_stop_reason,
|
||||
# Set explicitly so exclude_unset=True keeps it.
|
||||
stop_sequence=anthropic_stop_sequence,
|
||||
)
|
||||
chunk = AnthropicStreamEvent(
|
||||
type="message_delta",
|
||||
delta=stop_delta,
|
||||
usage=_build_anthropic_usage(origin_chunk.usage),
|
||||
)
|
||||
data = chunk.model_dump_json(exclude_unset=True)
|
||||
yield wrap_data_with_event(data, "message_delta")
|
||||
continue
|
||||
|
||||
if origin_chunk.choices[0].finish_reason is not None:
|
||||
finish_reason = origin_chunk.choices[0].finish_reason
|
||||
stop_sequence = origin_chunk.choices[0].stop_reason
|
||||
# continue
|
||||
|
||||
# thinking / text content
|
||||
reasoning_delta = origin_chunk.choices[0].delta.reasoning
|
||||
if reasoning_delta is not None:
|
||||
if reasoning_delta == "":
|
||||
pass
|
||||
else:
|
||||
if state.block_type != "thinking":
|
||||
for event in stop_and_flush():
|
||||
yield event
|
||||
start_event = start_block(
|
||||
AnthropicContentBlock(
|
||||
type="thinking", thinking=""
|
||||
)
|
||||
)
|
||||
yield start_event
|
||||
chunk = AnthropicStreamEvent(
|
||||
index=(
|
||||
state.block_index
|
||||
if state.block_index is not None
|
||||
else state.content_block_index
|
||||
),
|
||||
type="content_block_delta",
|
||||
delta=AnthropicDelta(
|
||||
type="thinking_delta",
|
||||
thinking=reasoning_delta,
|
||||
),
|
||||
)
|
||||
data = chunk.model_dump_json(exclude_unset=True)
|
||||
yield wrap_data_with_event(data, "content_block_delta")
|
||||
|
||||
if origin_chunk.choices[0].delta.content is not None:
|
||||
if origin_chunk.choices[0].delta.content == "":
|
||||
pass
|
||||
elif state.block_type == "tool_use":
|
||||
state.pending_content.append(
|
||||
origin_chunk.choices[0].delta.content
|
||||
)
|
||||
else:
|
||||
if state.block_type != "text":
|
||||
for event in stop_and_flush():
|
||||
yield event
|
||||
start_event = start_block(
|
||||
AnthropicContentBlock(type="text", text="")
|
||||
)
|
||||
yield start_event
|
||||
chunk = AnthropicStreamEvent(
|
||||
index=(
|
||||
state.block_index
|
||||
if state.block_index is not None
|
||||
else state.content_block_index
|
||||
),
|
||||
type="content_block_delta",
|
||||
delta=AnthropicDelta(
|
||||
type="text_delta",
|
||||
text=origin_chunk.choices[0].delta.content,
|
||||
),
|
||||
)
|
||||
data = chunk.model_dump_json(exclude_unset=True)
|
||||
yield wrap_data_with_event(data, "content_block_delta")
|
||||
|
||||
# tool calls - process all tool calls in the delta
|
||||
if len(origin_chunk.choices[0].delta.tool_calls) > 0:
|
||||
for tool_call in origin_chunk.choices[0].delta.tool_calls:
|
||||
if tool_call.id is not None:
|
||||
# Update mapping for incremental updates
|
||||
tool_index_to_id[tool_call.index] = tool_call.id
|
||||
# Only create new block if different tool call
|
||||
# AND has a name
|
||||
tool_name = (
|
||||
tool_call.function.name
|
||||
if tool_call.function
|
||||
else None
|
||||
)
|
||||
if (
|
||||
state.tool_use_id != tool_call.id
|
||||
and tool_name is not None
|
||||
):
|
||||
for event in stop_and_flush():
|
||||
yield event
|
||||
tool_used = True
|
||||
last_tool_args = []
|
||||
start_event = start_block(
|
||||
AnthropicContentBlock(
|
||||
type="tool_use",
|
||||
id=tool_call.id,
|
||||
name=tool_name,
|
||||
input={},
|
||||
)
|
||||
)
|
||||
yield start_event
|
||||
# Handle initial arguments if present
|
||||
if (
|
||||
tool_call.function
|
||||
and tool_call.function.arguments
|
||||
and state.tool_use_id == tool_call.id
|
||||
):
|
||||
last_tool_args.append(
|
||||
tool_call.function.arguments
|
||||
)
|
||||
chunk = AnthropicStreamEvent(
|
||||
index=(
|
||||
state.block_index
|
||||
if state.block_index is not None
|
||||
else state.content_block_index
|
||||
),
|
||||
type="content_block_delta",
|
||||
delta=AnthropicDelta(
|
||||
type="input_json_delta",
|
||||
partial_json=tool_call.function.arguments,
|
||||
),
|
||||
)
|
||||
data = chunk.model_dump_json(exclude_unset=True)
|
||||
yield wrap_data_with_event(
|
||||
data, "content_block_delta"
|
||||
)
|
||||
else:
|
||||
# Incremental update - use index to find tool_use_id
|
||||
tool_use_id = tool_index_to_id.get(tool_call.index)
|
||||
if (
|
||||
tool_use_id is not None
|
||||
and tool_call.function
|
||||
and tool_call.function.arguments
|
||||
and state.tool_use_id == tool_use_id
|
||||
):
|
||||
last_tool_args.append(
|
||||
tool_call.function.arguments
|
||||
)
|
||||
chunk = AnthropicStreamEvent(
|
||||
index=(
|
||||
state.block_index
|
||||
if state.block_index is not None
|
||||
else state.content_block_index
|
||||
),
|
||||
type="content_block_delta",
|
||||
delta=AnthropicDelta(
|
||||
type="input_json_delta",
|
||||
partial_json=tool_call.function.arguments,
|
||||
),
|
||||
)
|
||||
data = chunk.model_dump_json(exclude_unset=True)
|
||||
yield wrap_data_with_event(
|
||||
data, "content_block_delta"
|
||||
)
|
||||
continue
|
||||
else:
|
||||
error_response = AnthropicStreamEvent(
|
||||
type="error",
|
||||
error=AnthropicError(
|
||||
type="internal_error",
|
||||
message="Invalid data format received",
|
||||
async for origin_chunk in generator:
|
||||
if isinstance(origin_chunk, ErrorResponse):
|
||||
raise RuntimeError(origin_chunk.error.message)
|
||||
if first_item:
|
||||
chunk = AnthropicStreamEvent(
|
||||
type="message_start",
|
||||
message=AnthropicMessagesResponse(
|
||||
id=origin_chunk.id,
|
||||
# Set explicitly: this event is serialized
|
||||
# with exclude_unset=True, which drops
|
||||
# default-valued fields, while strict
|
||||
# Anthropic SDK clients require
|
||||
# message.type/role (issue #45367).
|
||||
type="message",
|
||||
role="assistant",
|
||||
content=[],
|
||||
model=origin_chunk.model,
|
||||
stop_reason=None,
|
||||
stop_sequence=None,
|
||||
usage=_build_anthropic_usage(origin_chunk.usage),
|
||||
),
|
||||
)
|
||||
data = error_response.model_dump_json(exclude_unset=True)
|
||||
yield wrap_data_with_event(data, "error")
|
||||
first_item = False
|
||||
data = chunk.model_dump_json(exclude_unset=True)
|
||||
yield wrap_data_with_event(data, "message_start")
|
||||
continue
|
||||
|
||||
# last chunk including usage info
|
||||
if len(origin_chunk.choices) == 0:
|
||||
for event in stop_and_flush():
|
||||
yield event
|
||||
anthropic_stop_reason, anthropic_stop_sequence = (
|
||||
_to_anthropic_stop_reason(
|
||||
finish_reason or "stop",
|
||||
stop_sequence,
|
||||
tool_used=tool_used,
|
||||
tool_complete=(
|
||||
tool_used and _is_json("".join(last_tool_args))
|
||||
),
|
||||
)
|
||||
)
|
||||
stop_delta = AnthropicDelta(
|
||||
stop_reason=anthropic_stop_reason,
|
||||
# Set explicitly so exclude_unset=True keeps it.
|
||||
stop_sequence=anthropic_stop_sequence,
|
||||
)
|
||||
chunk = AnthropicStreamEvent(
|
||||
type="message_delta",
|
||||
delta=stop_delta,
|
||||
usage=_build_anthropic_usage(origin_chunk.usage),
|
||||
)
|
||||
data = chunk.model_dump_json(exclude_unset=True)
|
||||
yield wrap_data_with_event(data, "message_delta")
|
||||
continue
|
||||
|
||||
if origin_chunk.choices[0].finish_reason is not None:
|
||||
finish_reason = origin_chunk.choices[0].finish_reason
|
||||
stop_sequence = origin_chunk.choices[0].stop_reason
|
||||
# continue
|
||||
|
||||
# thinking / text content
|
||||
reasoning_delta = origin_chunk.choices[0].delta.reasoning
|
||||
if reasoning_delta is not None:
|
||||
if reasoning_delta == "":
|
||||
pass
|
||||
else:
|
||||
if state.block_type != "thinking":
|
||||
for event in stop_and_flush():
|
||||
yield event
|
||||
start_event = start_block(
|
||||
AnthropicContentBlock(type="thinking", thinking="")
|
||||
)
|
||||
yield start_event
|
||||
chunk = AnthropicStreamEvent(
|
||||
index=(
|
||||
state.block_index
|
||||
if state.block_index is not None
|
||||
else state.content_block_index
|
||||
),
|
||||
type="content_block_delta",
|
||||
delta=AnthropicDelta(
|
||||
type="thinking_delta",
|
||||
thinking=reasoning_delta,
|
||||
),
|
||||
)
|
||||
data = chunk.model_dump_json(exclude_unset=True)
|
||||
yield wrap_data_with_event(data, "content_block_delta")
|
||||
|
||||
if origin_chunk.choices[0].delta.content is not None:
|
||||
if origin_chunk.choices[0].delta.content == "":
|
||||
pass
|
||||
elif state.block_type == "tool_use":
|
||||
state.pending_content.append(
|
||||
origin_chunk.choices[0].delta.content
|
||||
)
|
||||
else:
|
||||
if state.block_type != "text":
|
||||
for event in stop_and_flush():
|
||||
yield event
|
||||
start_event = start_block(
|
||||
AnthropicContentBlock(type="text", text="")
|
||||
)
|
||||
yield start_event
|
||||
chunk = AnthropicStreamEvent(
|
||||
index=(
|
||||
state.block_index
|
||||
if state.block_index is not None
|
||||
else state.content_block_index
|
||||
),
|
||||
type="content_block_delta",
|
||||
delta=AnthropicDelta(
|
||||
type="text_delta",
|
||||
text=origin_chunk.choices[0].delta.content,
|
||||
),
|
||||
)
|
||||
data = chunk.model_dump_json(exclude_unset=True)
|
||||
yield wrap_data_with_event(data, "content_block_delta")
|
||||
|
||||
# tool calls - process all tool calls in the delta
|
||||
if len(origin_chunk.choices[0].delta.tool_calls) > 0:
|
||||
for tool_call in origin_chunk.choices[0].delta.tool_calls:
|
||||
if tool_call.id is not None:
|
||||
# Update mapping for incremental updates
|
||||
tool_index_to_id[tool_call.index] = tool_call.id
|
||||
# Only create new block if different tool call
|
||||
# AND has a name
|
||||
tool_name = (
|
||||
tool_call.function.name if tool_call.function else None
|
||||
)
|
||||
if (
|
||||
state.tool_use_id != tool_call.id
|
||||
and tool_name is not None
|
||||
):
|
||||
for event in stop_and_flush():
|
||||
yield event
|
||||
tool_used = True
|
||||
last_tool_args = []
|
||||
start_event = start_block(
|
||||
AnthropicContentBlock(
|
||||
type="tool_use",
|
||||
id=tool_call.id,
|
||||
name=tool_name,
|
||||
input={},
|
||||
)
|
||||
)
|
||||
yield start_event
|
||||
# Handle initial arguments if present
|
||||
if (
|
||||
tool_call.function
|
||||
and tool_call.function.arguments
|
||||
and state.tool_use_id == tool_call.id
|
||||
):
|
||||
last_tool_args.append(tool_call.function.arguments)
|
||||
chunk = AnthropicStreamEvent(
|
||||
index=(
|
||||
state.block_index
|
||||
if state.block_index is not None
|
||||
else state.content_block_index
|
||||
),
|
||||
type="content_block_delta",
|
||||
delta=AnthropicDelta(
|
||||
type="input_json_delta",
|
||||
partial_json=tool_call.function.arguments,
|
||||
),
|
||||
)
|
||||
data = chunk.model_dump_json(exclude_unset=True)
|
||||
yield wrap_data_with_event(data, "content_block_delta")
|
||||
else:
|
||||
# Incremental update - use index to find tool_use_id
|
||||
tool_use_id = tool_index_to_id.get(tool_call.index)
|
||||
if (
|
||||
tool_use_id is not None
|
||||
and tool_call.function
|
||||
and tool_call.function.arguments
|
||||
and state.tool_use_id == tool_use_id
|
||||
):
|
||||
last_tool_args.append(tool_call.function.arguments)
|
||||
chunk = AnthropicStreamEvent(
|
||||
index=(
|
||||
state.block_index
|
||||
if state.block_index is not None
|
||||
else state.content_block_index
|
||||
),
|
||||
type="content_block_delta",
|
||||
delta=AnthropicDelta(
|
||||
type="input_json_delta",
|
||||
partial_json=tool_call.function.arguments,
|
||||
),
|
||||
)
|
||||
data = chunk.model_dump_json(exclude_unset=True)
|
||||
yield wrap_data_with_event(data, "content_block_delta")
|
||||
continue
|
||||
for event in stop_and_flush():
|
||||
yield event
|
||||
stop_message = AnthropicStreamEvent(type="message_stop")
|
||||
data = stop_message.model_dump_json(exclude_unset=True, exclude_none=True)
|
||||
yield wrap_data_with_event(data, "message_stop")
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("Error in message stream converter.")
|
||||
for event in stop_active_block():
|
||||
yield event
|
||||
error_response = AnthropicStreamEvent(
|
||||
type="error",
|
||||
error=AnthropicError(
|
||||
|
||||
@@ -321,7 +321,14 @@ class CohereServingChatV2(OpenAIServingChat):
|
||||
case ChatCompletionResponse():
|
||||
return self._chat_completion_to_v2(generator, request)
|
||||
case _:
|
||||
return self._chat_completion_stream_to_v2(generator, request)
|
||||
return generator
|
||||
|
||||
def chat_completion_stream_generator(
|
||||
self, *args: Any, **kwargs: Any
|
||||
) -> AsyncGenerator[str, None]:
|
||||
return self._chat_completion_stream_to_v2(
|
||||
self.chat_completion_stream_generator_raw(*args, **kwargs)
|
||||
)
|
||||
|
||||
def to_chat_completion_request(
|
||||
self, request: CohereChatV2Request
|
||||
@@ -1428,10 +1435,9 @@ class CohereServingChatV2(OpenAIServingChat):
|
||||
|
||||
async def _chat_completion_stream_to_v2(
|
||||
self,
|
||||
generator: AsyncGenerator[str, None],
|
||||
request: CohereChatV2Request,
|
||||
generator: AsyncGenerator[ChatCompletionStreamResponse | ErrorResponse, None],
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""Translate an OpenAI-style chat completion SSE stream into Cohere's
|
||||
"""Translate raw OpenAI chat completion chunks into Cohere's
|
||||
v2 stream-event format.
|
||||
|
||||
Cohere's v2 stream lifecycle is:
|
||||
@@ -1445,19 +1451,9 @@ class CohereServingChatV2(OpenAIServingChat):
|
||||
state = _StreamState()
|
||||
|
||||
try:
|
||||
async for item in generator:
|
||||
if not item.startswith("data:"):
|
||||
continue
|
||||
data_str = item[len("data:") :].strip().rstrip("\n")
|
||||
if not data_str:
|
||||
continue
|
||||
if data_str == "[DONE]":
|
||||
# OpenAI's stream terminator. Fall through to the
|
||||
# post-loop cleanup so we always emit ``message-end``
|
||||
# even if the usage-only chunk was skipped.
|
||||
break
|
||||
|
||||
chunk = ChatCompletionStreamResponse.model_validate_json(data_str)
|
||||
async for chunk in generator:
|
||||
if isinstance(chunk, ErrorResponse):
|
||||
raise RuntimeError(chunk.error.message)
|
||||
state.last_chunk_id = chunk.id
|
||||
|
||||
if not state.started:
|
||||
@@ -1514,7 +1510,9 @@ class CohereServingChatV2(OpenAIServingChat):
|
||||
|
||||
except Exception as exc:
|
||||
logger.exception("Error converting chat completion stream to v2")
|
||||
if state.started and not state.ended:
|
||||
if not state.ended:
|
||||
for ev in self._close_open_blocks(state):
|
||||
yield ev
|
||||
yield _sse(
|
||||
json.dumps(
|
||||
{
|
||||
@@ -1530,11 +1528,7 @@ class CohereServingChatV2(OpenAIServingChat):
|
||||
yield _DONE_FRAME
|
||||
return
|
||||
|
||||
# Normal completion or ``[DONE]``: ensure ``message-end`` is always
|
||||
# emitted. Upstream may close the stream without sending the final
|
||||
# usage-only chunk (e.g. on shutdown, or when ``[DONE]`` is the only
|
||||
# terminator); without this fallback Cohere clients would hang
|
||||
# waiting for the closing event.
|
||||
# Ensure message-end is emitted even without a final usage chunk.
|
||||
if state.started and not state.ended:
|
||||
for ev in self._close_open_blocks(state):
|
||||
yield ev
|
||||
|
||||
@@ -40,6 +40,7 @@ from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.outputs import WeightVersionSpan
|
||||
from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
|
||||
from vllm.renderers.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
@@ -129,6 +130,8 @@ class ChatCompletionResponseChoice(OpenAIBaseModel):
|
||||
# ``None`` if (a) the request was aborted before any forward pass,
|
||||
# or (b) ``enable_return_routed_experts`` is off server-side.
|
||||
routed_experts: str | None = None
|
||||
# Contiguous weight version spans over output-token indexes.
|
||||
weight_versions: list[WeightVersionSpan] | None = None
|
||||
|
||||
|
||||
class ChatCompletionResponse(OpenAIBaseModel):
|
||||
@@ -168,6 +171,8 @@ class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
|
||||
stop_reason: int | str | None = None
|
||||
# not part of the OpenAI spec but for tracing the tokens
|
||||
token_ids: list[int] | None = None
|
||||
# Set on the final chunk only; covers the whole output.
|
||||
weight_versions: list[WeightVersionSpan] | None = None
|
||||
|
||||
|
||||
class ChatCompletionStreamResponse(OpenAIBaseModel):
|
||||
@@ -430,6 +435,15 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
"need to map generated text back to input tokens."
|
||||
),
|
||||
)
|
||||
return_weight_versions: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"If true, each choice includes weight_versions: the weight-version "
|
||||
"label that sampled each half-open range of its output tokens. "
|
||||
"When streaming, only the last chunk of each choice includes it. "
|
||||
"Beam search does not return it."
|
||||
),
|
||||
)
|
||||
routed_experts_prompt_start: int = Field(
|
||||
default=0,
|
||||
ge=0,
|
||||
|
||||
@@ -478,6 +478,44 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
chat_template_kwargs: dict[str, Any] | None = None,
|
||||
mm_token_counts: dict[str, int] | None = None,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
try:
|
||||
async for chunk in self.chat_completion_stream_generator_raw(
|
||||
request,
|
||||
result_generator,
|
||||
request_id,
|
||||
model_name,
|
||||
conversation,
|
||||
tokenizer,
|
||||
request_metadata,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
mm_token_counts=mm_token_counts,
|
||||
):
|
||||
if isinstance(chunk, ErrorResponse):
|
||||
data = chunk.model_dump_json()
|
||||
else:
|
||||
data = chunk.model_dump_json(
|
||||
exclude_unset=True, exclude_none=not chunk.choices
|
||||
)
|
||||
yield f"data: {data}\n\n"
|
||||
except Exception as e:
|
||||
logger.exception("Error serializing chat completion stream.")
|
||||
data = self.create_streaming_error_response(e)
|
||||
yield f"data: {data}\n\n"
|
||||
yield "data: [DONE]\n\n"
|
||||
|
||||
async def chat_completion_stream_generator_raw(
|
||||
self,
|
||||
request: ChatCompletionRequest,
|
||||
result_generator: AsyncIterator[RequestOutput],
|
||||
request_id: str,
|
||||
model_name: str,
|
||||
conversation: list[ConversationMessage],
|
||||
tokenizer: TokenizerLike,
|
||||
request_metadata: RequestResponseMetadata,
|
||||
chat_template_kwargs: dict[str, Any] | None = None,
|
||||
mm_token_counts: dict[str, int] | None = None,
|
||||
) -> AsyncGenerator[ChatCompletionStreamResponse | ErrorResponse, None]:
|
||||
"""Yield chat chunks or errors without serializing them into SSE frames."""
|
||||
created_time = int(time.time())
|
||||
chunk_object_type: Final = "chat.completion.chunk"
|
||||
first_iteration = True
|
||||
@@ -510,9 +548,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
parsers = [None] * num_choices
|
||||
except Exception as e:
|
||||
logger.exception("Error in parser creation.")
|
||||
data = self.create_streaming_error_response(e)
|
||||
yield f"data: {data}\n\n"
|
||||
yield "data: [DONE]\n\n"
|
||||
yield self.create_error_response(e)
|
||||
return
|
||||
|
||||
stream_options = request.stream_options
|
||||
@@ -583,8 +619,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
),
|
||||
)
|
||||
|
||||
data = chunk.model_dump_json(exclude_unset=True)
|
||||
yield f"data: {data}\n\n"
|
||||
yield chunk
|
||||
|
||||
# Send response to echo the input portion of the
|
||||
# last message
|
||||
@@ -624,8 +659,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
),
|
||||
)
|
||||
|
||||
data = chunk.model_dump_json(exclude_unset=True)
|
||||
yield f"data: {data}\n\n"
|
||||
yield chunk
|
||||
first_iteration = False
|
||||
|
||||
for output in res.outputs:
|
||||
@@ -651,17 +685,21 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
|
||||
delta_text = output.text
|
||||
|
||||
if (
|
||||
nothing_generated = (
|
||||
not delta_text
|
||||
and not output.token_ids
|
||||
and not previous_num_tokens[i]
|
||||
):
|
||||
)
|
||||
if nothing_generated and output.finish_reason is None:
|
||||
# Chunked prefill case, don't return empty chunks
|
||||
continue
|
||||
|
||||
delta_message: DeltaMessage | None
|
||||
|
||||
if parser is not None:
|
||||
if nothing_generated:
|
||||
# Finished without output: send the final chunk unparsed.
|
||||
delta_message = DeltaMessage()
|
||||
elif parser is not None:
|
||||
delta_message = parser.parse_delta(
|
||||
delta_text=delta_text,
|
||||
delta_token_ids=as_list(output.token_ids),
|
||||
@@ -773,6 +811,11 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
token_ids=(
|
||||
as_list(output.token_ids) if include_token_ids else None
|
||||
),
|
||||
weight_versions=(
|
||||
output.weight_versions
|
||||
if request.return_weight_versions
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
finish_reason_sent[i] = True
|
||||
@@ -812,8 +855,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
),
|
||||
)
|
||||
|
||||
data = chunk.model_dump_json(exclude_unset=True)
|
||||
yield f"data: {data}\n\n"
|
||||
yield chunk
|
||||
|
||||
if self._include_reasoning_tokens_details and not include_continuous_usage:
|
||||
for i, parser in enumerate(parsers):
|
||||
@@ -873,10 +915,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
system_fingerprint=self.system_fingerprint,
|
||||
metrics=stream_per_request_metrics,
|
||||
)
|
||||
final_usage_data = final_usage_chunk.model_dump_json(
|
||||
exclude_unset=True, exclude_none=True
|
||||
)
|
||||
yield f"data: {final_usage_data}\n\n"
|
||||
yield final_usage_chunk
|
||||
|
||||
# report to FastAPI middleware aggregate usage across all choices
|
||||
num_completion_tokens = sum(previous_num_tokens)
|
||||
@@ -910,13 +949,12 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
)
|
||||
|
||||
except GenerationError as e:
|
||||
yield f"data: {self._convert_generation_error_to_streaming_response(e)}\n\n"
|
||||
yield self.create_error_response(
|
||||
str(e), err_type="InternalServerError", status_code=e.status_code
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("Error in chat completion stream generator.")
|
||||
data = self.create_streaming_error_response(e)
|
||||
yield f"data: {data}\n\n"
|
||||
# Send the final done message after all response.n are finished
|
||||
yield "data: [DONE]\n\n"
|
||||
yield self.create_error_response(e)
|
||||
|
||||
async def chat_completion_full_generator(
|
||||
self,
|
||||
@@ -1097,6 +1135,9 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
else None
|
||||
),
|
||||
routed_experts=routed_experts_b64,
|
||||
weight_versions=(
|
||||
output.weight_versions if request.return_weight_versions else None
|
||||
),
|
||||
)
|
||||
choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.outputs import WeightVersionSpan
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.sampling_params import (
|
||||
BeamSearchParams,
|
||||
@@ -178,6 +179,15 @@ class CompletionRequest(OpenAIBaseModel):
|
||||
"need to map generated text back to input tokens."
|
||||
),
|
||||
)
|
||||
return_weight_versions: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"If true, each choice includes weight_versions: the weight-version "
|
||||
"label that sampled each half-open range of its output tokens. "
|
||||
"When streaming, only the last chunk of each choice includes it. "
|
||||
"Beam search does not return it."
|
||||
),
|
||||
)
|
||||
routed_experts_prompt_start: int = Field(
|
||||
default=0,
|
||||
ge=0,
|
||||
@@ -677,6 +687,8 @@ class CompletionResponseChoice(OpenAIBaseModel):
|
||||
# ``None`` if (a) the request was aborted before any forward pass,
|
||||
# or (b) ``enable_return_routed_experts`` is off server-side.
|
||||
routed_experts: str | None = None
|
||||
# Contiguous weight version spans over output-token indexes.
|
||||
weight_versions: list[WeightVersionSpan] | None = None
|
||||
|
||||
|
||||
class CompletionResponse(OpenAIBaseModel):
|
||||
@@ -716,6 +728,8 @@ class CompletionResponseStreamChoice(OpenAIBaseModel):
|
||||
# prompt tokens is put into choice to align with CompletionResponseChoice
|
||||
prompt_token_ids: list[int] | None = None
|
||||
token_ids: list[int] | None = None
|
||||
# Set on the final chunk only; covers the whole output.
|
||||
weight_versions: list[WeightVersionSpan] | None = None
|
||||
|
||||
|
||||
class CompletionStreamResponse(OpenAIBaseModel):
|
||||
|
||||
@@ -376,6 +376,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
not delta_text
|
||||
and not delta_token_ids
|
||||
and not previous_num_tokens[i]
|
||||
and finish_reason is None
|
||||
):
|
||||
# Chunked prefill case, don't return empty chunks
|
||||
continue
|
||||
@@ -421,6 +422,11 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
if request.return_token_ids
|
||||
else None
|
||||
),
|
||||
weight_versions=(
|
||||
output.weight_versions
|
||||
if request.return_weight_versions
|
||||
else None
|
||||
),
|
||||
)
|
||||
],
|
||||
)
|
||||
@@ -594,6 +600,11 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
as_list(output.token_ids) if request.return_token_ids else None
|
||||
),
|
||||
routed_experts=routed_experts_b64,
|
||||
weight_versions=(
|
||||
output.weight_versions
|
||||
if request.return_weight_versions
|
||||
else None
|
||||
),
|
||||
)
|
||||
choices.append(choice_data)
|
||||
|
||||
|
||||
@@ -53,6 +53,7 @@ from openai_harmony import Message as OpenAIHarmonyMessage
|
||||
from pydantic import (
|
||||
BeforeValidator,
|
||||
Field,
|
||||
PrivateAttr,
|
||||
ValidationError,
|
||||
field_serializer,
|
||||
model_validator,
|
||||
@@ -469,6 +470,14 @@ class ResponsesRequest(OpenAIBaseModel):
|
||||
max_output_tokens_param="max_output_tokens",
|
||||
)
|
||||
|
||||
_grammar_from_parser: bool = PrivateAttr(default=False)
|
||||
"""CAUTION: Should only be set by the parser's adjust_request."""
|
||||
|
||||
_parser_structured_outputs: StructuredOutputsParams | None = PrivateAttr(
|
||||
default=None
|
||||
)
|
||||
"""Parser-derived constraints that preserve the public request's text metadata."""
|
||||
|
||||
_DEFAULT_SAMPLING_PARAMS = {
|
||||
"temperature": 1.0,
|
||||
"top_p": 1.0,
|
||||
@@ -478,6 +487,8 @@ class ResponsesRequest(OpenAIBaseModel):
|
||||
|
||||
def extract_structured_outputs(self) -> StructuredOutputsParams | None:
|
||||
"""Normalize request constraints into ``StructuredOutputsParams``."""
|
||||
if self._parser_structured_outputs is not None:
|
||||
return self._parser_structured_outputs
|
||||
if self.text is None or self.text.format is None:
|
||||
return self.structured_outputs
|
||||
|
||||
|
||||
@@ -526,6 +526,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
trace_headers=trace_headers,
|
||||
session_id=session_id,
|
||||
reasoning_parser_kwargs=reasoning_parser_kwargs,
|
||||
reasoning_ended=True if request._grammar_from_parser else None,
|
||||
)
|
||||
generators.append(generator)
|
||||
|
||||
@@ -647,6 +648,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
trace_headers: Mapping[str, str] | None = None,
|
||||
session_id: str | None = None,
|
||||
reasoning_parser_kwargs: dict[str, Any] | None = None,
|
||||
reasoning_ended: bool | None = None,
|
||||
):
|
||||
max_model_len = self.model_config.max_model_len
|
||||
cache_salt = cast(str | None, engine_input.get("cache_salt"))
|
||||
@@ -673,6 +675,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
priority=priority,
|
||||
session_id=session_id,
|
||||
reasoning_parser_kwargs=reasoning_parser_kwargs,
|
||||
reasoning_ended=reasoning_ended,
|
||||
)
|
||||
|
||||
async for res in generator:
|
||||
|
||||
@@ -325,8 +325,8 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
|
||||
it is used to create the progress bar.
|
||||
If `False`, no progress bar is created.
|
||||
lora_request: LoRA request to use for generation, if any.
|
||||
chat_template: The chat template to use for the scoring. If None, we
|
||||
use the model's default chat template.
|
||||
chat_template: The chat template to use for scoring. If None, use
|
||||
the chat template passed to the LLM constructor.
|
||||
tokenization_kwargs: Overrides for `tokenizer.encode`.
|
||||
|
||||
Returns:
|
||||
|
||||
@@ -548,7 +548,11 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
|
||||
yield ScoringRenderParams(
|
||||
data_1=data_1[i],
|
||||
data_2=data_2[i],
|
||||
chat_template=ctx.chat_template,
|
||||
chat_template=(
|
||||
self.chat_template
|
||||
if ctx.chat_template is None
|
||||
else ctx.chat_template
|
||||
),
|
||||
max_tokens_per_query=max_tokens_per_query,
|
||||
max_tokens_per_doc=max_tokens_per_doc,
|
||||
tok_params=tok_params,
|
||||
|
||||
@@ -247,6 +247,7 @@ if TYPE_CHECKING:
|
||||
VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: int = 163840
|
||||
VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS: int = 1
|
||||
VLLM_ENFORCE_STRICT_TOOL_CALLING: bool = True
|
||||
VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT: bool = True
|
||||
VLLM_MQ_MAX_CHUNK_BYTES_MB: int = 16
|
||||
VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: int = 300
|
||||
VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS: int = 5
|
||||
@@ -1854,6 +1855,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_ENFORCE_STRICT_TOOL_CALLING": lambda: (
|
||||
os.getenv("VLLM_ENFORCE_STRICT_TOOL_CALLING", "True").lower() in ("true", "1")
|
||||
),
|
||||
# Constrain every `beam` output to its response template's format, also
|
||||
# without tool or structured output constraints.
|
||||
"VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT": lambda: (
|
||||
os.getenv("VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT", "True").lower() in ("true", "1")
|
||||
),
|
||||
# Control the max chunk bytes (in MB) for the rpc message queue.
|
||||
# Object larger than this threshold will be broadcast to worker
|
||||
# processes via zmq.
|
||||
@@ -2376,6 +2382,7 @@ def compile_factors() -> dict[str, object]:
|
||||
"VLLM_RANDOMIZE_DP_DUMMY_INPUTS",
|
||||
"VLLM_MODEL_REDIRECT_PATH",
|
||||
"VLLM_HOST_IP",
|
||||
"VLLM_RAY_BUNDLE_INDICES",
|
||||
"VLLM_ELASTIC_EP_SCALE_UP_LAUNCH",
|
||||
"VLLM_FORCE_AOT_LOAD",
|
||||
"S3_ACCESS_KEY_ID",
|
||||
|
||||
@@ -15,6 +15,7 @@ from vllm.lora.ops.triton_ops.utils import (
|
||||
get_lora_op_configs,
|
||||
supports_pdl,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
@@ -231,6 +232,10 @@ def _lora_expand(
|
||||
torch.bfloat16,
|
||||
]:
|
||||
CAST_TYPE = True
|
||||
# Pipelined wgmma with a register A operand is miscompiled by
|
||||
# ptxas < 13.3 on SM90 (triton-lang/triton#9433).
|
||||
if current_platform.is_cuda() and current_platform.is_device_capability(90):
|
||||
NUM_STAGES = 1
|
||||
|
||||
# TODO (varun): This grid formulation maximizes parallelization at the
|
||||
# cost of wasteful thread block launch when only a few input tokens require
|
||||
|
||||
@@ -356,10 +356,12 @@ class LLBf16Dotprod:
|
||||
if tidx == 0:
|
||||
for m in cutlass.range_constexpr(M):
|
||||
partials = sm[m, None].load()
|
||||
gC[m, n_idx] = partials.reduce(
|
||||
cute.ReductionOp.ADD,
|
||||
init_val=cutlass.Float32(0.0),
|
||||
reduction_profile=0,
|
||||
)
|
||||
gC[m, n_idx] = cutlass.Float32(
|
||||
partials.reduce(
|
||||
cute.ReductionOp.ADD,
|
||||
init_val=cutlass.Float32(0.0),
|
||||
reduction_profile=0,
|
||||
)
|
||||
).to(gC.element_type)
|
||||
if const_expr(self.use_pdl):
|
||||
cute.arch.griddepcontrol_launch_dependents()
|
||||
|
||||
@@ -47,7 +47,7 @@ class LLBf16SplitK:
|
||||
|
||||
:param ab_dtype: Element type for A and B operands.
|
||||
:param acc_dtype: Accumulator type used by MMA and reductions.
|
||||
:param out_dtype: Output element type. The public wrapper uses fp32.
|
||||
:param out_dtype: Output element type (fp32 or bf16).
|
||||
:param tile_n: CTA tile size in N. M is fixed to 16 for router batches.
|
||||
:type tile_n: int
|
||||
:param tile_k: K tile size staged through shared memory.
|
||||
@@ -580,6 +580,6 @@ class LLBf16SplitK:
|
||||
reduction_profile=0,
|
||||
)
|
||||
)
|
||||
gC[local_coord] = acc
|
||||
gC[local_coord] = cutlass.Float32(acc).to(gC.element_type)
|
||||
|
||||
cute.arch.sync_threads()
|
||||
|
||||
@@ -76,8 +76,8 @@ def make_fake_gemm_tensors(
|
||||
a_dtype: torch.dtype,
|
||||
b_dtype: torch.dtype,
|
||||
divisibility: int,
|
||||
output_dtype: torch.dtype = torch.float32,
|
||||
):
|
||||
from cutlass import Float32
|
||||
from quack.compile_utils import make_fake_tensor
|
||||
|
||||
hidden_states = make_fake_tensor(
|
||||
@@ -86,7 +86,7 @@ def make_fake_gemm_tensors(
|
||||
router_weight = make_fake_tensor(
|
||||
cutlass_dtype(b_dtype), (N, K), divisibility=divisibility
|
||||
)
|
||||
output = make_fake_tensor(Float32, (M, N), divisibility=1)
|
||||
output = make_fake_tensor(cutlass_dtype(output_dtype), (M, N), divisibility=1)
|
||||
return hidden_states, router_weight, output
|
||||
|
||||
|
||||
@@ -97,6 +97,7 @@ def validate_common_gemm_inputs(
|
||||
*,
|
||||
op_name: str,
|
||||
k_multiple: int | None = None,
|
||||
output_dtypes: tuple[torch.dtype, ...] = (torch.float32,),
|
||||
) -> None:
|
||||
if hidden_states.dim() != 2 or router_weight.dim() != 2:
|
||||
raise ValueError("hidden_states and router_weight must be 2D tensors")
|
||||
@@ -106,8 +107,11 @@ def validate_common_gemm_inputs(
|
||||
raise ValueError(
|
||||
"hidden_states and router_weight must be on the same CUDA device"
|
||||
)
|
||||
if output_dtype != torch.float32:
|
||||
raise ValueError(f"{op_name} only supports output_dtype=torch.float32")
|
||||
if output_dtype not in output_dtypes:
|
||||
raise ValueError(
|
||||
f"{op_name} only supports output_dtype="
|
||||
+ " or ".join(str(dtype) for dtype in output_dtypes)
|
||||
)
|
||||
if hidden_states.shape[1] != router_weight.shape[1]:
|
||||
raise ValueError(
|
||||
"hidden_states and router_weight must have matching K dimensions"
|
||||
|
||||
@@ -47,6 +47,8 @@ _SM100F_TUNED_SPLITK_CONFIGS: dict[tuple[int, int], dict[int, tuple[int, int]]]
|
||||
**{M: (8, 5) for M in (5, 8)},
|
||||
9: (8, 2),
|
||||
},
|
||||
(5120, 1792): {M: (2, 4) for M in range(5, 17)},
|
||||
(5120, 3584): {M: (2, 4) for M in range(5, 17)},
|
||||
(7168, 256): {14: (8, 2)},
|
||||
(6144, 256): {M: (8, 2) for M in (9, 12, 16)},
|
||||
(7168, 384): {M: (7, 5) for M in (13, 16)},
|
||||
@@ -101,39 +103,52 @@ class LLBf16Gemm:
|
||||
bs: int = 0
|
||||
split_k: int = 0
|
||||
num_stages: int = 0
|
||||
output_dtype: torch.dtype = torch.float32
|
||||
|
||||
def __init__(self, *, prefetch_pdl_weights: bool = False) -> None:
|
||||
self._prefetch_pdl_weights = prefetch_pdl_weights
|
||||
# Dot-prod: keyed on (M, K, bs), because M and K are Constexpr.
|
||||
self._compiled_cache: dict[tuple[int, int, int], Any] = {}
|
||||
# Split-K: keyed on (split_k, num_stages), fully shape-dynamic.
|
||||
self._splitk_cache: dict[tuple[int, int], Any] = {}
|
||||
# M and K are compile-time constants for dot-product.
|
||||
self._compiled_cache: dict[tuple[int, int, int, torch.dtype], Any] = {}
|
||||
# Split-K is shape-dynamic; both caches distinguish output dtype.
|
||||
self._splitk_cache: dict[tuple[int, int, torch.dtype], Any] = {}
|
||||
|
||||
def dispatch(self, *, M: int, K: int, N: int) -> CompileKey:
|
||||
def dispatch(
|
||||
self, *, M: int, K: int, N: int, output_dtype: torch.dtype = torch.float32
|
||||
) -> CompileKey:
|
||||
tuned_bs, tuned_splitk = _arch_tuned_configs()
|
||||
if M <= _DEFAULT_DOTPROD_MAX_M or K < 2048:
|
||||
bs = tuned_bs.get((K, N), {}).get(M, _DEFAULT_DOTPROD_BS)
|
||||
return self.CompileKey(backend="dotprod", M=M, K=K, bs=bs)
|
||||
return self.CompileKey(
|
||||
backend="dotprod", M=M, K=K, bs=bs, output_dtype=output_dtype
|
||||
)
|
||||
|
||||
split_k, num_stages = tuned_splitk.get((K, N), {}).get(
|
||||
M, _DEFAULT_SPLITK_CONFIG
|
||||
)
|
||||
return self.CompileKey(backend="splitk", split_k=split_k, num_stages=num_stages)
|
||||
return self.CompileKey(
|
||||
backend="splitk",
|
||||
split_k=split_k,
|
||||
num_stages=num_stages,
|
||||
output_dtype=output_dtype,
|
||||
)
|
||||
|
||||
def get_warmup_keys(
|
||||
self,
|
||||
*,
|
||||
shapes: Iterable[tuple[int, int]],
|
||||
m_values: Iterable[int],
|
||||
output_dtype: torch.dtype = torch.float32,
|
||||
) -> list[CompileKey]:
|
||||
return list(
|
||||
dict.fromkeys(
|
||||
self.dispatch(M=M, K=K, N=N) for K, N in shapes for M in m_values
|
||||
self.dispatch(M=M, K=K, N=N, output_dtype=output_dtype)
|
||||
for K, N in shapes
|
||||
for M in m_values
|
||||
)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _fake_gemm_tensors(*, M, K, N, divisibility: int):
|
||||
def _fake_gemm_tensors(*, M, K, N, divisibility: int, output_dtype: torch.dtype):
|
||||
return make_fake_gemm_tensors(
|
||||
M=M,
|
||||
K=K,
|
||||
@@ -141,10 +156,13 @@ class LLBf16Gemm:
|
||||
a_dtype=torch.bfloat16,
|
||||
b_dtype=torch.bfloat16,
|
||||
divisibility=divisibility,
|
||||
output_dtype=output_dtype,
|
||||
)
|
||||
|
||||
def _compile_splitk(self, compile_key: CompileKey) -> None:
|
||||
cute, _ = cute_context()
|
||||
from cutlass import BFloat16, Float32
|
||||
|
||||
from ._ll_bf16_splitk import LLBf16SplitK
|
||||
|
||||
hidden_states, router_weight, output = self._fake_gemm_tensors(
|
||||
@@ -152,8 +170,12 @@ class LLBf16Gemm:
|
||||
K=cute.sym_int(),
|
||||
N=cute.sym_int(),
|
||||
divisibility=8,
|
||||
output_dtype=compile_key.output_dtype,
|
||||
)
|
||||
gemm = LLBf16SplitK(
|
||||
out_dtype=BFloat16
|
||||
if compile_key.output_dtype == torch.bfloat16
|
||||
else Float32,
|
||||
tile_n=16,
|
||||
tile_k=256,
|
||||
num_stages=compile_key.num_stages,
|
||||
@@ -169,7 +191,9 @@ class LLBf16Gemm:
|
||||
current_cuda_stream(),
|
||||
options="--enable-tvm-ffi",
|
||||
)
|
||||
self._splitk_cache[(compile_key.split_k, compile_key.num_stages)] = compiled
|
||||
self._splitk_cache[
|
||||
(compile_key.split_k, compile_key.num_stages, compile_key.output_dtype)
|
||||
] = compiled
|
||||
logger.debug(
|
||||
"Compiled ll_bf16_splitk: sk=%d ns=%d",
|
||||
compile_key.split_k,
|
||||
@@ -187,6 +211,7 @@ class LLBf16Gemm:
|
||||
K=compile_key.K,
|
||||
N=N,
|
||||
divisibility=stride_divisibility,
|
||||
output_dtype=compile_key.output_dtype,
|
||||
)
|
||||
gemm = LLBf16Dotprod(
|
||||
k=compile_key.K,
|
||||
@@ -205,7 +230,9 @@ class LLBf16Gemm:
|
||||
current_cuda_stream(),
|
||||
options="--enable-tvm-ffi --ptxas-options -maxrregcount=64",
|
||||
)
|
||||
self._compiled_cache[(compile_key.M, compile_key.K, compile_key.bs)] = compiled
|
||||
self._compiled_cache[
|
||||
(compile_key.M, compile_key.K, compile_key.bs, compile_key.output_dtype)
|
||||
] = compiled
|
||||
logger.debug(
|
||||
"Compiled ll_bf16_dotprod: M=%d, K=%d, bs=%d",
|
||||
compile_key.M,
|
||||
@@ -215,12 +242,21 @@ class LLBf16Gemm:
|
||||
|
||||
def compile(self, compile_key: CompileKey) -> None:
|
||||
if compile_key.backend == "splitk":
|
||||
splitk_cache_key = (compile_key.split_k, compile_key.num_stages)
|
||||
splitk_cache_key = (
|
||||
compile_key.split_k,
|
||||
compile_key.num_stages,
|
||||
compile_key.output_dtype,
|
||||
)
|
||||
if splitk_cache_key not in self._splitk_cache:
|
||||
self._compile_splitk(compile_key)
|
||||
return
|
||||
|
||||
dotprod_cache_key = (compile_key.M, compile_key.K, compile_key.bs)
|
||||
dotprod_cache_key = (
|
||||
compile_key.M,
|
||||
compile_key.K,
|
||||
compile_key.bs,
|
||||
compile_key.output_dtype,
|
||||
)
|
||||
if dotprod_cache_key not in self._compiled_cache:
|
||||
self._compile_dotprod(compile_key)
|
||||
|
||||
@@ -229,8 +265,11 @@ class LLBf16Gemm:
|
||||
*,
|
||||
shapes: Iterable[tuple[int, int]],
|
||||
m_values: Iterable[int],
|
||||
output_dtype: torch.dtype = torch.float32,
|
||||
) -> None:
|
||||
for compile_key in self.get_warmup_keys(shapes=shapes, m_values=m_values):
|
||||
for compile_key in self.get_warmup_keys(
|
||||
shapes=shapes, m_values=m_values, output_dtype=output_dtype
|
||||
):
|
||||
self.compile(compile_key)
|
||||
|
||||
@staticmethod
|
||||
@@ -251,6 +290,7 @@ class LLBf16Gemm:
|
||||
output_dtype,
|
||||
op_name="ll_bf16_gemm",
|
||||
k_multiple=8,
|
||||
output_dtypes=(torch.float32, torch.bfloat16),
|
||||
)
|
||||
|
||||
def __call__(
|
||||
@@ -258,19 +298,28 @@ class LLBf16Gemm:
|
||||
hidden_states: torch.Tensor, # [M, K] bf16
|
||||
router_weight: torch.Tensor, # [N, K] bf16
|
||||
output_dtype: torch.dtype = torch.float32,
|
||||
) -> torch.Tensor: # [M, N] fp32
|
||||
) -> torch.Tensor: # [M, N]
|
||||
self._validate_inputs(hidden_states, router_weight, output_dtype)
|
||||
|
||||
M, K = hidden_states.shape
|
||||
N = router_weight.shape[0]
|
||||
compile_key = self.dispatch(M=M, K=K, N=N)
|
||||
compile_key = self.dispatch(M=M, K=K, N=N, output_dtype=output_dtype)
|
||||
if compile_key.backend == "splitk":
|
||||
splitk_cache_key = (compile_key.split_k, compile_key.num_stages)
|
||||
splitk_cache_key = (
|
||||
compile_key.split_k,
|
||||
compile_key.num_stages,
|
||||
compile_key.output_dtype,
|
||||
)
|
||||
if splitk_cache_key not in self._splitk_cache:
|
||||
self.compile(compile_key)
|
||||
kernel = self._splitk_cache[splitk_cache_key]
|
||||
else:
|
||||
dotprod_cache_key = (compile_key.M, compile_key.K, compile_key.bs)
|
||||
dotprod_cache_key = (
|
||||
compile_key.M,
|
||||
compile_key.K,
|
||||
compile_key.bs,
|
||||
compile_key.output_dtype,
|
||||
)
|
||||
if dotprod_cache_key not in self._compiled_cache:
|
||||
self.compile(compile_key)
|
||||
kernel = self._compiled_cache[dotprod_cache_key]
|
||||
|
||||
@@ -6,19 +6,6 @@ from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
|
||||
def _torch_hc_prenorm_gemm(
|
||||
x: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
sqrsum: torch.Tensor,
|
||||
) -> None:
|
||||
assert out.shape[0] == 1
|
||||
assert sqrsum.shape[0] == 1
|
||||
x_float = x.float()
|
||||
out[0].copy_(x_float @ fn.t())
|
||||
sqrsum[0].copy_(x_float.square().sum(dim=-1))
|
||||
|
||||
|
||||
def _hc_prenorm_gemm_outputs(
|
||||
x: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
|
||||
@@ -2024,6 +2024,7 @@ def build_mla_chunked_context_metadata(
|
||||
dcp_local_block_size: int,
|
||||
dcp_virtual_block_size: int,
|
||||
dcp_manager: MLADCPManager | None = None,
|
||||
align_indptr: bool = False,
|
||||
) -> "MLACommonPrefillMetadata.ChunkedContextMetadata | None":
|
||||
"""Build chunked-context metadata for an MLA prefill.
|
||||
|
||||
@@ -2043,6 +2044,7 @@ def build_mla_chunked_context_metadata(
|
||||
dcp_local_block_size: Per-rank interleave block size for DCP.
|
||||
dcp_virtual_block_size: ``dcp_local_block_size * dcp_world_size``.
|
||||
dcp_manager: Shared MLA DCP collective manager.
|
||||
align_indptr: Align packed int32 indptr slices to 16 bytes for CuTe DSL.
|
||||
|
||||
Returns:
|
||||
The chunked-context metadata, or None when no prefill has any context.
|
||||
@@ -2123,6 +2125,15 @@ def build_mla_chunked_context_metadata(
|
||||
num_tokens = sum(plan.seq_lens)
|
||||
num_local_tokens = num_tokens
|
||||
|
||||
if align_indptr:
|
||||
# CuTe DSL requires 16-byte-aligned int32 indptrs.
|
||||
padding = -boundary_offset % 4
|
||||
cu_seq_lens_flat.extend([0] * padding)
|
||||
cu_seqlens_q_flat.extend([0] * padding)
|
||||
if use_dcp:
|
||||
padded_local_cu_seq_lens_flat.extend([0] * padding)
|
||||
boundary_offset += padding
|
||||
|
||||
seq_lens_flat.extend(plan.seq_lens)
|
||||
cu_seq_lens_flat.extend(itertools.accumulate(plan.seq_lens, initial=0))
|
||||
query_base = query_start_loc[plan.request_start]
|
||||
@@ -2616,6 +2627,14 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
query_start_loc_cpu[reqs_start:] - query_start_loc_cpu[reqs_start]
|
||||
)
|
||||
|
||||
align_indptr = False
|
||||
if (
|
||||
self._prefill_backend.get_name() == "TRTLLM_RAGGED"
|
||||
and self.vllm_config.attention_config.trtllm_ragged_backend
|
||||
== "cute-dsl"
|
||||
):
|
||||
align_indptr = True
|
||||
|
||||
chunked_context_metadata = build_mla_chunked_context_metadata(
|
||||
context_lens_cpu=context_lens_cpu,
|
||||
prefill_query_start_loc_cpu=prefill_query_start_loc_cpu,
|
||||
@@ -2628,6 +2647,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
dcp_local_block_size=self.dcp_local_block_size,
|
||||
dcp_virtual_block_size=self.dcp_virtual_block_size,
|
||||
dcp_manager=self.dcp_manager,
|
||||
align_indptr=align_indptr,
|
||||
)
|
||||
|
||||
prefill_metadata = MLACommonPrefillMetadata(
|
||||
|
||||
@@ -359,13 +359,6 @@ class ExpertMapManager:
|
||||
return True
|
||||
return self._expert_map[global_id] != -1
|
||||
|
||||
def get_local_expert_ids(self) -> list[int]:
|
||||
"""Get list of global IDs for experts on this rank."""
|
||||
if self._expert_map is None:
|
||||
return list(range(self.global_num_experts))
|
||||
|
||||
return torch.where(self._expert_map != -1)[0].tolist()
|
||||
|
||||
def update(
|
||||
self,
|
||||
moe_parallel_config: FusedMoEParallelConfig,
|
||||
|
||||
@@ -545,7 +545,12 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
return (workspace1, workspace2, output)
|
||||
|
||||
def _act_mul_quant(
|
||||
self, input: torch.Tensor, output: torch.Tensor, activation: MoEActivation
|
||||
self,
|
||||
input: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
expert_ends: torch.Tensor | None = None,
|
||||
expert_alignment: int = 0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
block_k = self._ACT_BLOCK_K
|
||||
scale_fmt = DeepGemmQuantScaleFMT.from_oracle()
|
||||
@@ -561,6 +566,8 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
output_q=output,
|
||||
group_size=block_k,
|
||||
clamp_limit=self.gemm1_clamp_limit,
|
||||
expert_ends=expert_ends,
|
||||
expert_alignment=expert_alignment,
|
||||
)
|
||||
use_ue8m0 = scale_fmt == DeepGemmQuantScaleFMT.FLOAT32_CEIL_UE8M0
|
||||
return silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
@@ -634,7 +641,8 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
a1q_perm = _resize_cache(
|
||||
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, K)
|
||||
)
|
||||
a1q, a1q_scale, expert_ids, inv_perm, align_used = deepgemm_moe_permute(
|
||||
use_psum_layout = current_platform.is_device_capability_family(100)
|
||||
a1q, a1q_scale, grouped_layout, inv_perm, align_used = deepgemm_moe_permute(
|
||||
aq=a1q,
|
||||
aq_scale=a1q_scale,
|
||||
topk_ids=topk_ids,
|
||||
@@ -642,6 +650,7 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
expert_map=expert_map,
|
||||
expert_tokens_meta=expert_tokens_meta,
|
||||
aq_out=a1q_perm,
|
||||
use_psum_layout=use_psum_layout,
|
||||
)
|
||||
assert a1q.size(0) == M_sum
|
||||
|
||||
@@ -655,18 +664,23 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
(a1q, a1q_scale),
|
||||
(w1.view(torch.int8), self.w1_scale),
|
||||
mm1_out,
|
||||
expert_ids,
|
||||
grouped_layout,
|
||||
use_psum_layout=use_psum_layout,
|
||||
recipe_a=(1, self._ACT_BLOCK_K),
|
||||
recipe_b=(1, self._WEIGHT_BLOCK_K),
|
||||
)
|
||||
|
||||
# SwiGLU activation + FP8 requant
|
||||
# SiLU+mul and FP8 requant use the same live expert ranges as GEMM.
|
||||
activation_out_dim = self.adjust_N_for_activation(N, activation)
|
||||
quant_out = _resize_cache(
|
||||
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, activation_out_dim)
|
||||
)
|
||||
a2q, a2q_scale = self._act_mul_quant(
|
||||
input=mm1_out.view(-1, N), output=quant_out, activation=activation
|
||||
input=mm1_out.view(-1, N),
|
||||
output=quant_out,
|
||||
activation=activation,
|
||||
expert_ends=grouped_layout if use_psum_layout else None,
|
||||
expert_alignment=align_used if use_psum_layout else 0,
|
||||
)
|
||||
|
||||
# FC2: FP8 activations x FP4 weights
|
||||
@@ -675,7 +689,8 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
(a2q, a2q_scale),
|
||||
(w2.view(torch.int8), self.w2_scale),
|
||||
mm2_out,
|
||||
expert_ids,
|
||||
grouped_layout,
|
||||
use_psum_layout=use_psum_layout,
|
||||
recipe_a=(1, self._ACT_BLOCK_K),
|
||||
recipe_b=(1, self._WEIGHT_BLOCK_K),
|
||||
)
|
||||
|
||||
@@ -517,6 +517,7 @@ _TILE_CONFIG_OVERRIDES = {
|
||||
256: (128, 32),
|
||||
512: (128, 19),
|
||||
},
|
||||
(5120, 256): {64: (64, 64)}, # GB200
|
||||
(6144, 128): { # MiniMax-M3
|
||||
64: (32, 64),
|
||||
128: (64, 64),
|
||||
@@ -533,7 +534,7 @@ def _pick_tile_config(N: int, K: int, M: int, num_sms: int) -> tuple[int, int]:
|
||||
if 32 < N <= 512:
|
||||
token_bucket = triton.next_power_of_2(N)
|
||||
overrides = _TILE_CONFIG_OVERRIDES.get((K, M))
|
||||
if overrides is not None:
|
||||
if overrides is not None and token_bucket in overrides:
|
||||
BN, split_k = overrides[token_bucket]
|
||||
return BN, min(split_k, k_tiles)
|
||||
|
||||
@@ -609,6 +610,8 @@ def _bf16x3_router_gemm(X: torch.Tensor, W: torch.Tensor) -> torch.Tensor:
|
||||
"""Return ``X @ W.T`` using the SM100 BF16x3 router GEMM kernel."""
|
||||
N, K = X.shape
|
||||
M, _ = W.shape
|
||||
if N == 0:
|
||||
return X.new_empty((N, M), dtype=torch.float32)
|
||||
num_sms = torch.cuda.get_device_properties(X.device).multi_processor_count
|
||||
|
||||
BN, split_k = _pick_tile_config(N, K, M, num_sms)
|
||||
|
||||
@@ -24,8 +24,8 @@ class GateLinear(ReplicatedLinear):
|
||||
|
||||
1. cuteDSL ll_bf16_gemm (SM90+, M<=16, bf16 in, fp32 out,
|
||||
any shape w/ K divisible by 8)
|
||||
2. fp32 specialized kernel (bf16/fp32 in, fp32 out, M<=32; SM90+
|
||||
model-specific shapes, gfx950 any shape)
|
||||
2. fp32 specialized kernel (bf16/fp32 in, fp32 out; SM90+
|
||||
model-specific shapes and batch limits, gfx950 any shape with M<=32)
|
||||
3. cuteDSL ll_fp32w_gemm (SM90+, M<=32, bf16/fp32 in, fp32 out,
|
||||
any shape)
|
||||
4. bf16x3 CuteDSL kernel (SM100, bf16 in, fp32 weight)
|
||||
@@ -46,9 +46,14 @@ class GateLinear(ReplicatedLinear):
|
||||
split-K from the token count and skip the batch-invariant linear path.
|
||||
"""
|
||||
|
||||
# (hidden_size, num_experts) pairs with an instantiated fp32 kernel:
|
||||
# (3072, 256) -> MiniMax-M2/M2.5, (6144, 128) -> MiniMax-M3
|
||||
FP32_SUPPORTED_SHAPES = {(3072, 256), (6144, 128)}
|
||||
# (hidden_size, num_experts) pairs with an instantiated fp32 kernel.
|
||||
FP32_SUPPORTED_SHAPES = {
|
||||
(3072, 256),
|
||||
(6144, 128),
|
||||
(5120, 256),
|
||||
(4096, 352),
|
||||
(8192, 256),
|
||||
}
|
||||
FP32_MAX_TOKENS = 32
|
||||
# Largest batch for which tier 1 beats the cuBLAS epilogue. Subclasses
|
||||
# whose router shape stays ahead further up the range may raise it.
|
||||
@@ -454,9 +459,9 @@ def fp32_router_gemm_dispatch_impl(
|
||||
weight: torch.Tensor,
|
||||
allow_bf16x3_router_gemm: bool,
|
||||
) -> torch.Tensor:
|
||||
"""Dynamically run fp32 specialized gemm if num_tokens <= FP32_MAX_TOKENS,
|
||||
otherwise optionally run the experimental BF16x3 kernel for medium/large
|
||||
SM100 router batches, then fall back to F.linear.
|
||||
"""Run the FP32 kernel for small batches, with shape-specific limits.
|
||||
|
||||
Larger SM100 BF16 batches use BF16x3; other inputs fall back to F.linear.
|
||||
This must be wrapped in a custom op because our torch.compile integration
|
||||
does not support runtime dispatching on num_tokens.
|
||||
"""
|
||||
@@ -473,7 +478,15 @@ def fp32_router_gemm_dispatch_impl(
|
||||
if weight.dtype == torch.bfloat16:
|
||||
return torch.mm(x, weight.T, out_dtype=torch.float32)
|
||||
return torch.nn.functional.linear(x.float(), weight)
|
||||
if x.shape[0] <= _FP32_ROUTER_GEMM_MAX_TOKENS:
|
||||
max_tokens = _FP32_ROUTER_GEMM_MAX_TOKENS
|
||||
if (
|
||||
allow_bf16x3_router_gemm
|
||||
and x.dtype == torch.bfloat16
|
||||
and weight.shape == (256, 5120)
|
||||
):
|
||||
# GB200: four token groups win through 48 for multiples of eight.
|
||||
max_tokens = 48 if x.shape[0] % 8 == 0 else 16
|
||||
if x.shape[0] <= max_tokens:
|
||||
return ops.fp32_router_gemm(x, weight)
|
||||
|
||||
if allow_bf16x3_router_gemm and x.dtype == torch.bfloat16:
|
||||
|
||||
@@ -9,139 +9,162 @@
|
||||
from packaging import version
|
||||
|
||||
from vllm.model_executor.layers.mamba.ops.triton_helpers import fast_exp
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
TRITON_22 = version.parse(triton.__version__) >= version.parse("2.2.0")
|
||||
|
||||
|
||||
def _drop_racy_wgmma_configs(configs: list[triton.Config]) -> list[triton.Config]:
|
||||
"""Drop the configs that Triton miscompiles on sm90.
|
||||
|
||||
With whole warpgroups (num_warps % 4 == 0) and BLOCK_SIZE_M % 64 == 0,
|
||||
`tl.dot(cb, x)` lowers to wgmma with `cb` in registers. With
|
||||
num_stages > 1, Triton leaves that wgmma running across the loop
|
||||
back-edge while the next iteration overwrites the registers of `cb`,
|
||||
which writes NaN/Inf into whole output rows.
|
||||
"""
|
||||
if not (current_platform.is_cuda() and current_platform.is_device_capability(90)):
|
||||
return configs
|
||||
return [
|
||||
c
|
||||
for c in configs
|
||||
if c.num_stages == 1
|
||||
or c.num_warps % 4 != 0
|
||||
or c.kwargs["BLOCK_SIZE_M"] % 64 != 0
|
||||
]
|
||||
|
||||
|
||||
@triton.autotune(
|
||||
configs=[
|
||||
# =================================================================
|
||||
# Higher warp count configs for better latency hiding
|
||||
# More warps = more instructions in flight = better memory latency hiding
|
||||
# =================================================================
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
|
||||
num_stages=2,
|
||||
num_warps=8,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=2,
|
||||
num_warps=8,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
|
||||
num_stages=2,
|
||||
num_warps=8,
|
||||
),
|
||||
# Smaller tiles with more stages for software pipelining
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
|
||||
num_stages=3,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 64},
|
||||
num_stages=2,
|
||||
num_warps=4,
|
||||
),
|
||||
# =================================================================
|
||||
# Low register pressure configs (num_stages=1) for large dstate
|
||||
# =================================================================
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
|
||||
num_stages=1,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=1,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=1,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=1,
|
||||
num_warps=4,
|
||||
),
|
||||
# num_stages=2 configs - moderate register pressure
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
|
||||
num_stages=2,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=2,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=2,
|
||||
num_warps=4,
|
||||
),
|
||||
# Original configs for larger dstate values
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 64},
|
||||
num_stages=3,
|
||||
num_warps=8,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
|
||||
num_stages=5,
|
||||
num_warps=2,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=5,
|
||||
num_warps=2,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=2,
|
||||
),
|
||||
],
|
||||
configs=_drop_racy_wgmma_configs(
|
||||
[
|
||||
# =================================================================
|
||||
# Higher warp count configs for better latency hiding
|
||||
# More warps = more instructions in flight = better memory latency hiding
|
||||
# =================================================================
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
|
||||
num_stages=2,
|
||||
num_warps=8,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=2,
|
||||
num_warps=8,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
|
||||
num_stages=2,
|
||||
num_warps=8,
|
||||
),
|
||||
# Smaller tiles with more stages for software pipelining
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
|
||||
num_stages=3,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 64},
|
||||
num_stages=2,
|
||||
num_warps=4,
|
||||
),
|
||||
# =================================================================
|
||||
# Low register pressure configs (num_stages=1) for large dstate
|
||||
# =================================================================
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
|
||||
num_stages=1,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=1,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=1,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=1,
|
||||
num_warps=4,
|
||||
),
|
||||
# num_stages=2 configs - moderate register pressure
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
|
||||
num_stages=2,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=2,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=2,
|
||||
num_warps=4,
|
||||
),
|
||||
# Original configs for larger dstate values
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 64},
|
||||
num_stages=3,
|
||||
num_warps=8,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
|
||||
num_stages=5,
|
||||
num_warps=2,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=5,
|
||||
num_warps=2,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=2,
|
||||
),
|
||||
]
|
||||
),
|
||||
key=["chunk_size", "hdim", "dstate", "IS_CAUSAL"],
|
||||
)
|
||||
@triton.jit
|
||||
|
||||
@@ -12,35 +12,6 @@ from vllm.triton_utils import tl, triton
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def block_dequant(
|
||||
x_q_block: torch.Tensor,
|
||||
x_s: torch.Tensor,
|
||||
block_size: list[int],
|
||||
) -> torch.Tensor:
|
||||
"""This function conducts block-wise dequantization.
|
||||
The inputs are block-wise quantization tensor `x_q_block`,
|
||||
block-wise quantization scale and the block size.
|
||||
The outputs are dequantized tensor.
|
||||
"""
|
||||
block_n, block_k = block_size[0], block_size[1]
|
||||
n, k = x_q_block.shape
|
||||
n_tiles = (n + block_n - 1) // block_n
|
||||
k_tiles = (k + block_k - 1) // block_k
|
||||
assert n_tiles == x_s.shape[0]
|
||||
assert k_tiles == x_s.shape[1]
|
||||
|
||||
x_dq_block = x_q_block.to(torch.float32)
|
||||
|
||||
for i in range(k_tiles):
|
||||
for j in range(n_tiles):
|
||||
x_dq_block[
|
||||
j * block_n : min((j + 1) * block_n, n),
|
||||
i * block_k : min((i + 1) * block_k, k),
|
||||
] *= x_s[j][i]
|
||||
|
||||
return x_dq_block
|
||||
|
||||
|
||||
if current_platform.is_rocm():
|
||||
|
||||
@triton.jit
|
||||
|
||||
@@ -1404,14 +1404,17 @@ class DiffusionSampler:
|
||||
# unaffected; only the canvas exploration is constrained. Applied
|
||||
# before canvas padding so phantom positions stay uniform.
|
||||
if num_decode > 0:
|
||||
top_k, top_p = self.sampling_states.get_top_k_top_p(
|
||||
top_k, top_p, has_topp_only_rows = self.sampling_states.get_top_k_top_p(
|
||||
decode_slots.repeat_interleave(
|
||||
valid_canvas_len, output_size=int(valid_canvas_len_np.sum())
|
||||
),
|
||||
decode_slots_np,
|
||||
logits.shape[-1],
|
||||
)
|
||||
if top_k is not None or top_p is not None:
|
||||
logits = apply_top_k_top_p(logits.float(), top_k, top_p)
|
||||
logits = apply_top_k_top_p(
|
||||
logits.float(), top_k, top_p, has_topp_only_rows=has_topp_only_rows
|
||||
)
|
||||
|
||||
# Where each decode request's rows start in the flat logits. Tiles
|
||||
# below gather and pad a request to the tile's width.
|
||||
|
||||
@@ -64,6 +64,7 @@ if TYPE_CHECKING:
|
||||
from vllm.tasks import ScoreType
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, KVCacheSpec
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import (
|
||||
EncoderCudaGraphCaptureInputs,
|
||||
EncoderCudaGraphConfig,
|
||||
@@ -670,6 +671,47 @@ def supports_score_template(
|
||||
return getattr(model, "supports_score_template", False)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsKVCacheGrouping(Protocol):
|
||||
"""The interface required for models that own their KV cache grouping.
|
||||
|
||||
.. note::
|
||||
Experimental. The classmethod contract may change as model-owned
|
||||
grouping stabilizes.
|
||||
|
||||
Implementing it lets a model replace the default KV cache grouping (which
|
||||
layers share a block table) while Core keeps owning placement, byte
|
||||
accounting, and admission.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def get_kv_cache_groups(
|
||||
cls,
|
||||
vllm_config: "VllmConfig",
|
||||
kv_cache_spec: dict[str, "KVCacheSpec"],
|
||||
) -> list["KVCacheGroupSpec"]:
|
||||
"""Split the model's layers into logical KV cache groups."""
|
||||
...
|
||||
|
||||
|
||||
@overload
|
||||
def supports_kv_cache_grouping(
|
||||
model: type[object],
|
||||
) -> TypeIs[type[SupportsKVCacheGrouping]]: ...
|
||||
|
||||
|
||||
@overload
|
||||
def supports_kv_cache_grouping(
|
||||
model: object,
|
||||
) -> TypeIs[SupportsKVCacheGrouping]: ...
|
||||
|
||||
|
||||
def supports_kv_cache_grouping(
|
||||
model: type[object] | object,
|
||||
) -> TypeIs[type[SupportsKVCacheGrouping]] | TypeIs[SupportsKVCacheGrouping]:
|
||||
return isinstance(model, SupportsKVCacheGrouping)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsLoRA(Protocol):
|
||||
"""The interface required for all models that support LoRA."""
|
||||
|
||||
@@ -50,6 +50,7 @@ from .interfaces import (
|
||||
is_hybrid,
|
||||
requires_raw_input_tokens,
|
||||
supports_hc_sp,
|
||||
supports_kv_cache_grouping,
|
||||
supports_multimodal,
|
||||
supports_multimodal_encoder_tp_data,
|
||||
supports_multimodal_raw_input_only,
|
||||
@@ -882,6 +883,7 @@ class _ModelInfo:
|
||||
supports_replayssm: bool
|
||||
supports_transcription: bool
|
||||
supports_transcription_only: bool
|
||||
supports_kv_cache_grouping: bool
|
||||
supported_video_pruning_methods: tuple[str, ...]
|
||||
supports_mm_device_do_normalize: bool
|
||||
|
||||
@@ -913,6 +915,7 @@ class _ModelInfo:
|
||||
supports_transcription_only=(
|
||||
supports_transcription(model) and model.supports_transcription_only
|
||||
),
|
||||
supports_kv_cache_grouping=supports_kv_cache_grouping(model),
|
||||
has_noops=has_noops(model),
|
||||
supported_video_pruning_methods=getattr(
|
||||
model, "supported_video_pruning_methods", ()
|
||||
|
||||
@@ -11,11 +11,14 @@ from typing import TYPE_CHECKING
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import CacheConfig, VllmConfig, replace
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.parallel_state import GroupCoordinator
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _normalize_cache_config_for_hash(cache_config: CacheConfig) -> CacheConfig:
|
||||
"""Normalize cache config for hashing"""
|
||||
@@ -55,6 +58,53 @@ def resolve_flashinfer_autotune_file(runner: "GPUModelRunner") -> Path:
|
||||
return output_dir / "autotune_configs.json"
|
||||
|
||||
|
||||
def use_flashinfer_autotune_v2(runner: "GPUModelRunner") -> bool:
|
||||
from vllm.distributed.parallel_state import get_node_count, get_world_group
|
||||
from vllm.utils.flashinfer import has_flashinfer_autotune_v2
|
||||
|
||||
# Elastic EP transfers tuning state to joining workers with v1
|
||||
# save_configs/load_configs, which do not export managed-store winners.
|
||||
if runner.vllm_config.parallel_config.enable_elastic_ep:
|
||||
return False
|
||||
# PP stages profile independently. Keep their group-scoped v1 files until
|
||||
# managed-cache reads and publishes can be isolated across tuning groups.
|
||||
if runner.vllm_config.parallel_config.pipeline_parallel_size > 1:
|
||||
return False
|
||||
if not has_flashinfer_autotune_v2():
|
||||
return False
|
||||
# Use the initialized topology: Ray may span nodes with config.nnodes=1.
|
||||
# A cache hit skips profiling collectives, so node-local stores with
|
||||
# different contents cannot safely participate in synchronized tuning.
|
||||
if get_node_count() != 1:
|
||||
logger.info_once(
|
||||
"Using legacy FlashInfer autotune cache synchronization for "
|
||||
"multi-node deployments; managed-cache synchronization is "
|
||||
"currently supported only within one node."
|
||||
)
|
||||
return False
|
||||
# Lift this gate after pinning a release with the fix below and validating
|
||||
# distributed managed-cache tuning: https://github.com/flashinfer-ai/flashinfer/pull/6185
|
||||
if get_world_group().world_size > 1:
|
||||
logger.info_once(
|
||||
"Using legacy FlashInfer autotune cache synchronization for "
|
||||
"multi-rank deployments until FlashInfer coordinates cache hits."
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def resolve_flashinfer_autotune_v2_root() -> Path | None:
|
||||
"""Placement root for FlashInfer's managed autotune store.
|
||||
|
||||
Unlike :func:`resolve_flashinfer_autotune_file`, no vLLM-side identity
|
||||
(config hash, flashinfer version, arch) is folded into the path: the
|
||||
managed store keys entries per op and hashes the software/hardware
|
||||
environment itself. ``None`` selects FlashInfer's default root.
|
||||
"""
|
||||
override_dir = envs.VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR
|
||||
return Path(override_dir).expanduser() if override_dir else None
|
||||
|
||||
|
||||
def sync_flashinfer_autotune_cache(
|
||||
runner: "GPUModelRunner",
|
||||
group: "GroupCoordinator",
|
||||
|
||||
@@ -9,6 +9,8 @@ import torch
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
|
||||
resolve_flashinfer_autotune_file,
|
||||
resolve_flashinfer_autotune_v2_root,
|
||||
use_flashinfer_autotune_v2,
|
||||
write_flashinfer_autotune_cache,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
@@ -145,7 +147,6 @@ def _run_flashinfer_sparse_mla_decode_autotune(
|
||||
|
||||
world = get_world_group()
|
||||
is_leader = world.rank_in_group == 0
|
||||
cache_path = resolve_flashinfer_autotune_file(runner)
|
||||
|
||||
dummy_run_kwargs = dict(
|
||||
num_tokens=num_tokens,
|
||||
@@ -155,6 +156,13 @@ def _run_flashinfer_sparse_mla_decode_autotune(
|
||||
create_mixed_batch=True,
|
||||
)
|
||||
|
||||
if use_flashinfer_autotune_v2(runner):
|
||||
return _run_sparse_mla_decode_autotune_v2(
|
||||
worker, num_tokens, log_label, dummy_run_kwargs
|
||||
)
|
||||
|
||||
cache_path = resolve_flashinfer_autotune_file(runner)
|
||||
|
||||
if is_leader:
|
||||
logger.info(
|
||||
"Autotuning FlashInfer SM120 sparse MLA %s decode with cache: %s",
|
||||
@@ -223,6 +231,58 @@ def _run_flashinfer_sparse_mla_decode_autotune(
|
||||
return True
|
||||
|
||||
|
||||
def _run_sparse_mla_decode_autotune_v2(
|
||||
worker: "Worker",
|
||||
num_tokens: int,
|
||||
log_label: str,
|
||||
dummy_run_kwargs: dict,
|
||||
) -> bool:
|
||||
"""Run every rank's sparse warmup in the managed autotune context.
|
||||
|
||||
FlashInfer owns persistence; some sparse kernels use a separate calibration
|
||||
cache instead of managed winner entries.
|
||||
"""
|
||||
from flashinfer import autotune_v2, autotune_v2_reload
|
||||
|
||||
from vllm.distributed.parallel_state import get_world_group
|
||||
|
||||
runner = worker.model_runner
|
||||
world = get_world_group()
|
||||
cache_root = resolve_flashinfer_autotune_v2_root()
|
||||
if world.rank_in_group == 0:
|
||||
logger.info(
|
||||
"Autotuning FlashInfer SM120 sparse MLA %s decode with the managed "
|
||||
"autotune context (root=%s)",
|
||||
log_label,
|
||||
cache_root if cache_root is not None else "flashinfer default",
|
||||
)
|
||||
|
||||
with torch.inference_mode():
|
||||
if _uses_v2_model_runner(runner) and runner.max_num_reqs >= 2:
|
||||
v2_runner = cast("V2GPUModelRunner", runner)
|
||||
warmup_executed = run_mixed_prefill_decode_warmup(
|
||||
v2_runner,
|
||||
worker.execute_model,
|
||||
worker.sample_tokens,
|
||||
num_tokens,
|
||||
mixed_step_context=autotune_v2(mode="tune", cache_root=cache_root),
|
||||
req_id_prefix="_sparse_mla_v2_warmup",
|
||||
)
|
||||
else:
|
||||
warmup_executed = True
|
||||
with autotune_v2(mode="tune", cache_root=cache_root):
|
||||
runner._dummy_run(**dummy_run_kwargs)
|
||||
|
||||
if not warmup_executed:
|
||||
return False
|
||||
|
||||
if world.world_size > 1:
|
||||
world.barrier()
|
||||
autotune_v2_reload()
|
||||
world.barrier()
|
||||
return True
|
||||
|
||||
|
||||
def _flashinfer_sparse_mla_decode_autotune(
|
||||
worker: "Worker",
|
||||
num_tokens: int,
|
||||
|
||||
@@ -1082,6 +1082,10 @@ class JitWarmupRegistry:
|
||||
from tqdm import tqdm
|
||||
|
||||
from vllm.distributed import is_global_first_rank
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
VllmTritonJitKernel,
|
||||
compile_triton_warmup,
|
||||
)
|
||||
|
||||
kernel_items: list[tuple[VllmJitKernel[Any], dict[Any, None]]] = []
|
||||
for kernel, registrations in self._registrations.items():
|
||||
@@ -1102,17 +1106,40 @@ class JitWarmupRegistry:
|
||||
if not kernel_items:
|
||||
return
|
||||
|
||||
parallel_items: list[tuple[VllmJitKernel[Any], dict[Any, None]]] = []
|
||||
serial_items = []
|
||||
for kernel, compile_keys in kernel_items:
|
||||
if (
|
||||
isinstance(kernel, VllmTritonJitKernel)
|
||||
and kernel.supports_parallel_compile
|
||||
):
|
||||
parallel_items.append((kernel, compile_keys))
|
||||
else:
|
||||
serial_items.append((kernel, compile_keys))
|
||||
|
||||
total_keys = sum(len(compile_keys) for _, compile_keys in kernel_items)
|
||||
with tqdm(
|
||||
kernel_items,
|
||||
total=len(kernel_items),
|
||||
desc=f"JIT kernel warmup ({total_keys} compile keys)",
|
||||
disable=not is_global_first_rank(),
|
||||
dynamic_ncols=True,
|
||||
unit="kernel",
|
||||
) as progress:
|
||||
for kernel, compile_keys in progress:
|
||||
progress.set_postfix_str(
|
||||
f"{kernel.__class__.__name__} ({len(compile_keys)} keys)",
|
||||
refresh=False,
|
||||
|
||||
def compile_kernels(
|
||||
items: list[tuple[VllmJitKernel[Any], dict[Any, None]]],
|
||||
) -> None:
|
||||
for kernel, compile_keys in items:
|
||||
progress.set_postfix_str(
|
||||
f"{kernel.__class__.__name__} ({len(compile_keys)} keys)",
|
||||
refresh=False,
|
||||
)
|
||||
kernel.compile_many(compile_keys)
|
||||
progress.update()
|
||||
|
||||
if parallel_items:
|
||||
compile_triton_warmup(
|
||||
lambda: compile_kernels(parallel_items),
|
||||
sum(len(keys) for _, keys in parallel_items),
|
||||
)
|
||||
kernel.compile_many(compile_keys)
|
||||
compile_kernels(serial_items)
|
||||
|
||||
@@ -6,7 +6,7 @@ from abc import abstractmethod
|
||||
from collections.abc import Callable, Hashable, Iterable, Mapping
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import nullcontext, suppress
|
||||
from contextvars import copy_context
|
||||
from contextvars import ContextVar, copy_context
|
||||
from dataclasses import dataclass, field
|
||||
from functools import cache, cached_property, update_wrapper
|
||||
from typing import Any, Generic, ParamSpec, Protocol, TypeVar, cast, overload
|
||||
@@ -207,6 +207,31 @@ class TritonWarmupTensor:
|
||||
return result if dim is None else result[dim]
|
||||
|
||||
|
||||
_shared_compile_pool: ContextVar[bool] = ContextVar(
|
||||
"shared_triton_warmup_compile_pool", default=False
|
||||
)
|
||||
|
||||
|
||||
def compile_triton_warmup(compile_all: Callable[[], None], num_keys: int) -> None:
|
||||
"""Share one compiler pool across all eligible warmup kernels."""
|
||||
num_threads = min(envs.VLLM_TRITON_JIT_WARMUP_NUM_THREADS, num_keys)
|
||||
if num_threads <= 1:
|
||||
compile_all()
|
||||
return
|
||||
|
||||
def compile_in_context() -> None:
|
||||
_shared_compile_pool.set(True)
|
||||
with (
|
||||
ThreadPoolExecutor(max_workers=num_threads) as executor,
|
||||
triton.AsyncCompileMode(executor),
|
||||
):
|
||||
# Dispatch mutates owner state and must stay on this thread.
|
||||
compile_all()
|
||||
|
||||
# Older Triton versions leave async mode active on compilation failure.
|
||||
copy_context().run(compile_in_context)
|
||||
|
||||
|
||||
class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
|
||||
"""Triton owner whose runtime launch specification is reused for warmup."""
|
||||
|
||||
@@ -230,32 +255,25 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
|
||||
self._warming = False
|
||||
self._warming_compile_key = None
|
||||
|
||||
@property
|
||||
def supports_parallel_compile(self) -> bool:
|
||||
# AMD LLVM code generation can abort under concurrent compilation.
|
||||
return (
|
||||
current_platform.is_cuda()
|
||||
and not self._run_autotune
|
||||
and getattr(triton, "AsyncCompileMode", None) is not None
|
||||
and envs.VLLM_TRITON_JIT_WARMUP_NUM_THREADS > 1
|
||||
)
|
||||
|
||||
def compile_many(self, compile_keys: Iterable[CompileKeyT]) -> None:
|
||||
"""Compile CUDA startup warmup variants in parallel, then wait."""
|
||||
keys = list(compile_keys)
|
||||
num_threads = min(envs.VLLM_TRITON_JIT_WARMUP_NUM_THREADS, len(keys))
|
||||
async_compile = getattr(triton, "AsyncCompileMode", None)
|
||||
# AMD LLVM code generation can abort under concurrent compilation.
|
||||
if (
|
||||
not current_platform.is_cuda()
|
||||
or self._run_autotune
|
||||
or async_compile is None
|
||||
or num_threads <= 1
|
||||
):
|
||||
if not self.supports_parallel_compile or _shared_compile_pool.get():
|
||||
return super().compile_many(keys)
|
||||
|
||||
def compile_all() -> None:
|
||||
with (
|
||||
ThreadPoolExecutor(max_workers=num_threads) as executor,
|
||||
async_compile(executor),
|
||||
):
|
||||
# Dispatch stays on this thread: compile() mutates owner state.
|
||||
# Only Triton's underlying compiler work runs in the pool.
|
||||
super(VllmTritonJitKernel, self).compile_many(keys)
|
||||
|
||||
# Older Triton versions leave async mode active on compilation failure.
|
||||
# Isolate that state so it cannot affect later runtime JIT compilation.
|
||||
copy_context().run(compile_all)
|
||||
compile_triton_warmup(
|
||||
lambda: super(VllmTritonJitKernel, self).compile_many(keys), len(keys)
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def _kernel_arg_names(self) -> tuple[str, ...]:
|
||||
|
||||
@@ -23,6 +23,8 @@ from vllm.model_executor.warmup.cutedsl_warmup import cutedsl_warmup
|
||||
from vllm.model_executor.warmup.deep_gemm_warmup import deep_gemm_warmup
|
||||
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
|
||||
resolve_flashinfer_autotune_file,
|
||||
resolve_flashinfer_autotune_v2_root,
|
||||
use_flashinfer_autotune_v2,
|
||||
)
|
||||
from vllm.model_executor.warmup.flashinfer_sparse_mla_warmup import (
|
||||
autotune_hisparse_flashinfer_attention,
|
||||
@@ -419,7 +421,7 @@ def flashinfer_autotune(runner: "GPUModelRunner", *, skip_attn: bool = False) ->
|
||||
otherwise the world group tunes together. Per-tactic timings are
|
||||
averaged over the tuning group so all its ranks select the same tactic.
|
||||
|
||||
Results are persisted per rank: FlashInfer keys MoE entries by tp/ep rank
|
||||
Legacy results are persisted per rank: FlashInfer keys MoE entries by tp/ep rank
|
||||
(``MoERunner.get_cache_key_extras``), so one rank's file only hits on that
|
||||
rank. A rank with a cache hit skips the per-tactic reduce the others block
|
||||
in, so ranks keep loaded configs only if every rank in the tuning group
|
||||
@@ -450,38 +452,45 @@ def flashinfer_autotune(runner: "GPUModelRunner", *, skip_attn: bool = False) ->
|
||||
)
|
||||
autotune_kwargs["skip_ops"] = skip_ops
|
||||
|
||||
cache_path = resolve_flashinfer_autotune_file(runner)
|
||||
# The world group only spans DP ranks when vLLM folds DP into it.
|
||||
dp_rank = runner.vllm_config.parallel_config.data_parallel_rank
|
||||
cache_path = cache_path.with_name(
|
||||
f"{cache_path.stem}_dp{dp_rank}_rank{world.rank_in_group}{cache_path.suffix}"
|
||||
)
|
||||
if is_leader:
|
||||
logger.info_once("Using FlashInfer autotune cache file: %s", cache_path)
|
||||
use_autotune_v2 = use_flashinfer_autotune_v2(runner)
|
||||
if use_autotune_v2:
|
||||
from flashinfer import autotune_v2, autotune_v2_reload
|
||||
|
||||
# We skip EPLB here since we don't want to record dummy metrics.
|
||||
# Randomize inputs to avoid every token pick the same experts,
|
||||
# which lead to some EP ranks receiving no tokens and skipping their
|
||||
# MoE kernel entirely, and cause hang due to all-reduce collective
|
||||
# during synchronized autotuning.
|
||||
if _all_ranks_have_matching_cache(cache_path, tune_group):
|
||||
loaded = tuner.load_configs(str(cache_path))
|
||||
if tune_group.world_size > 1:
|
||||
loaded_by_rank: list[bool | None] = [None] * tune_group.world_size
|
||||
torch.distributed.all_gather_object(
|
||||
loaded_by_rank, loaded, group=tune_group.cpu_group
|
||||
cache_root = resolve_flashinfer_autotune_v2_root()
|
||||
if is_leader:
|
||||
logger.info_once(
|
||||
"Using FlashInfer managed autotune cache (root=%s)",
|
||||
cache_root if cache_root is not None else "flashinfer default",
|
||||
)
|
||||
loaded = all(loaded_by_rank)
|
||||
if not loaded:
|
||||
tuner.clear_cache()
|
||||
autotune_context = autotune_v2(
|
||||
mode="tune", cache_root=cache_root, **autotune_kwargs
|
||||
)
|
||||
else:
|
||||
cache_path = resolve_flashinfer_autotune_file(runner)
|
||||
# The world group only spans DP ranks when vLLM folds DP into it.
|
||||
dp_rank = runner.vllm_config.parallel_config.data_parallel_rank
|
||||
cache_path = cache_path.with_name(
|
||||
f"{cache_path.stem}_dp{dp_rank}_rank{world.rank_in_group}{cache_path.suffix}"
|
||||
)
|
||||
if is_leader:
|
||||
logger.info_once("Using FlashInfer autotune cache file: %s", cache_path)
|
||||
|
||||
if _all_ranks_have_matching_cache(cache_path, tune_group):
|
||||
loaded = tuner.load_configs(str(cache_path))
|
||||
if tune_group.world_size > 1:
|
||||
loaded_by_rank: list[bool | None] = [None] * tune_group.world_size
|
||||
torch.distributed.all_gather_object(
|
||||
loaded_by_rank, loaded, group=tune_group.cpu_group
|
||||
)
|
||||
loaded = all(loaded_by_rank)
|
||||
if not loaded:
|
||||
tuner.clear_cache()
|
||||
autotune_context = fi_utils.autotune(tune_mode=True, **autotune_kwargs)
|
||||
|
||||
group = tune_group.cpu_group if tune_group.world_size > 1 else None
|
||||
set_autotune_process_group(group)
|
||||
try:
|
||||
with (
|
||||
torch.inference_mode(),
|
||||
fi_utils.autotune(tune_mode=True, **autotune_kwargs),
|
||||
):
|
||||
with torch.inference_mode(), autotune_context:
|
||||
hisparse_enabled = (
|
||||
runner.vllm_config.attention_config.hisparse_config is not None
|
||||
)
|
||||
@@ -498,7 +507,10 @@ def flashinfer_autotune(runner: "GPUModelRunner", *, skip_attn: bool = False) ->
|
||||
|
||||
if world.world_size > 1:
|
||||
world.barrier()
|
||||
# Skip the rewrite when nothing was tuned this start (every entry came from
|
||||
# the file). FlashInfer gates its own autotune(cache=...) save the same way.
|
||||
if tuner._dirty:
|
||||
if use_autotune_v2:
|
||||
if world.world_size > 1:
|
||||
# All ranks serve the shared store's final state after publishing.
|
||||
autotune_v2_reload()
|
||||
world.barrier()
|
||||
elif tuner._dirty:
|
||||
tuner.save_configs(str(cache_path))
|
||||
|
||||
@@ -19,9 +19,11 @@ if TYPE_CHECKING:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _get_kda_layer(worker: Worker) -> KimiK3DeltaAttention | None:
|
||||
def _get_kda_layer(
|
||||
worker: Worker, module: str = "vllm.models.kimi_k3.nvidia.kda"
|
||||
) -> KimiK3DeltaAttention | None:
|
||||
# Kimi model construction already imports kda. Avoid importing it here.
|
||||
kda = sys.modules.get("vllm.models.kimi_k3.nvidia.kda")
|
||||
kda = sys.modules.get(module)
|
||||
if kda is None:
|
||||
return None
|
||||
|
||||
@@ -172,9 +174,59 @@ def _warm_recurrent_kda(
|
||||
)
|
||||
|
||||
|
||||
def _warm_gluon_chunk_kda(layer, input_dtype: torch.dtype) -> None:
|
||||
from vllm.models.kimi_k3.amd.ops.kda_prefill import (
|
||||
chunk_kda_prefill,
|
||||
get_gluon_chunk_kda_warmup_profiles,
|
||||
)
|
||||
|
||||
if layer.kda_prefill_backend != "gluon":
|
||||
return
|
||||
kv_cache = layer.kv_cache
|
||||
if not isinstance(kv_cache, (list, tuple)) or len(kv_cache) < 2:
|
||||
return
|
||||
state = kv_cache[1]
|
||||
if not isinstance(state, torch.Tensor) or not state.numel():
|
||||
return
|
||||
|
||||
logger.info("Warming up Kimi-K3 Gluon chunk KDA prefill kernels.")
|
||||
h, d = int(layer.local_num_heads), int(layer.head_dim)
|
||||
for num_seqs, seq_len in get_gluon_chunk_kda_warmup_profiles():
|
||||
num_tokens = num_seqs * seq_len
|
||||
shape = (1, num_tokens, h, d)
|
||||
chunk_kda_prefill(
|
||||
q=torch.zeros(shape, dtype=input_dtype, device=state.device),
|
||||
k=torch.zeros(shape, dtype=input_dtype, device=state.device),
|
||||
v=torch.zeros(shape, dtype=input_dtype, device=state.device),
|
||||
raw_g=torch.zeros(shape, dtype=input_dtype, device=state.device),
|
||||
raw_beta=torch.zeros(
|
||||
(1, num_tokens, h), dtype=input_dtype, device=state.device
|
||||
),
|
||||
A_log=layer.A_log,
|
||||
g_bias=layer.dt_bias,
|
||||
lower_bound=layer.gate_lower_bound,
|
||||
state_cache=state,
|
||||
state_indices=torch.zeros(num_seqs, dtype=torch.int32, device=state.device),
|
||||
has_initial_state=torch.zeros(
|
||||
num_seqs, dtype=torch.bool, device=state.device
|
||||
),
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
cu_seqlens=torch.arange(
|
||||
0, num_tokens + 1, seq_len, dtype=torch.int32, device=state.device
|
||||
),
|
||||
backend="gluon",
|
||||
out=torch.empty(shape, dtype=input_dtype, device=state.device),
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def kimi_k3_triton_warmup(worker: Worker) -> None:
|
||||
"""Warm Kimi-K3 Triton kernels reachable by this server."""
|
||||
if current_platform.is_rocm():
|
||||
layer = _get_kda_layer(worker, "vllm.models.kimi_k3.amd.kda")
|
||||
if layer is not None:
|
||||
_warm_gluon_chunk_kda(layer, worker.model_config.dtype)
|
||||
return
|
||||
if not current_platform.is_cuda():
|
||||
return
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ from itertools import islice
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_pp_group
|
||||
from vllm.distributed import get_pp_group, get_tensor_model_parallel_rank
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
@@ -71,6 +71,7 @@ class DeepseekV32DecoderLayer(torch.nn.Module):
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
)
|
||||
|
||||
self.zero_mlp_output = False
|
||||
if (
|
||||
config.n_routed_experts is not None
|
||||
and layer_idx >= config.first_k_dense_replace
|
||||
@@ -82,7 +83,15 @@ class DeepseekV32DecoderLayer(torch.nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
)
|
||||
self.mlp.experts.moe_config.skip_final_all_reduce = True
|
||||
moe_cfg = self.mlp.experts.moe_config
|
||||
pc = moe_cfg.moe_parallel_config
|
||||
# Validated on MoRI only. DeepEP, pplx and the other combines
|
||||
# keep the previous always-skip behavior.
|
||||
if pc.use_mori_kernels:
|
||||
# Combine is already all-reduced; the next fused norm sums TP.
|
||||
self.zero_mlp_output = get_tensor_model_parallel_rank() > 0
|
||||
else:
|
||||
moe_cfg.skip_final_all_reduce = True
|
||||
else:
|
||||
self.mlp = DeepseekV2MLP(
|
||||
hidden_size=config.hidden_size,
|
||||
@@ -116,6 +125,8 @@ class DeepseekV32DecoderLayer(torch.nn.Module):
|
||||
hidden_states, residual, self.post_attention_layernorm
|
||||
)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
if self.zero_mlp_output:
|
||||
hidden_states = torch.zeros_like(hidden_states)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
|
||||
@@ -578,6 +578,9 @@ def _validate_aiter_mega_moe_config(vllm_config: VllmConfig) -> None:
|
||||
|
||||
|
||||
class DeepseekV4MoE(nn.Module):
|
||||
# None: use the padding vLLM added.
|
||||
experts_intermediate_pad: int | None = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
@@ -726,6 +729,7 @@ class DeepseekV4MoE(nn.Module):
|
||||
top_k=config.num_experts_per_tok,
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.moe_intermediate_size,
|
||||
intermediate_pad=self.experts_intermediate_pad,
|
||||
renormalize=config.norm_topk_prob,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.experts",
|
||||
|
||||
@@ -77,6 +77,10 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class DeepseekV4MoE(DeepseekV4MoEBase):
|
||||
# TODO: revert once https://github.com/ROCm/aiter/pull/6598 is in the
|
||||
# pinned AITER; with the padding set, its FlyDSL MoE can output NaN.
|
||||
experts_intermediate_pad = 0
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
Mirrors the reference ``mtp_model.py`` shipped with the checkpoint: each MTP
|
||||
depth ``i`` owns ``hidden_norm`` / ``embed_norm`` RMSNorms, an ``input_proj``
|
||||
(``2H -> H``) and a full Inkling transformer block (dense bf16 MLP, with the
|
||||
(``2H -> H``) and a full Inkling transformer block (dense MLP, with the
|
||||
same short convolutions as the backbone; its attention is full or sliding-window
|
||||
per depth, selected by ``mtp_config.local_layer_ids``). When enabled, a shared
|
||||
``chain_norm`` is applied after every depth; its output is both the logits input
|
||||
@@ -28,18 +28,23 @@ from torch import nn
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.model_loader.mtp_validation import (
|
||||
is_mtp_completeness_check_enabled,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.interfaces import SupportsMultiModalEmbeddings
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.model_executor.models.utils import WeightsMapper, maybe_prefix
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from ..configs import InklingModelConfig
|
||||
from .layernorm import InklingRMSNorm
|
||||
from .model import InklingDecoderLayer, InklingReplicatedEmbedding
|
||||
from .model import (
|
||||
InklingDecoderLayer,
|
||||
InklingReplicatedEmbedding,
|
||||
_TmlForCausalLMBase,
|
||||
)
|
||||
from .ops.norm import embed_dual_rmsnorm_cat, embed_rmsnorm
|
||||
|
||||
# Checkpoint attention projections (wq_du/wk_dv/wv_dv/wr_du) -> fused qkvr.
|
||||
@@ -71,7 +76,13 @@ def _select_mtp_depth_count(n_predict: int, num_spec: int | None) -> int:
|
||||
class InklingMTPDepthLayer(nn.Module):
|
||||
"""One MTP depth: norm both inputs, fuse (2H->H), run a Inkling block."""
|
||||
|
||||
def __init__(self, config: InklingModelConfig, prefix: str, is_local: bool) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
config: InklingModelConfig,
|
||||
prefix: str,
|
||||
is_local: bool,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.embed_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
@@ -80,16 +91,17 @@ class InklingMTPDepthLayer(nn.Module):
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.input_proj",
|
||||
)
|
||||
# A force-dense-MLP bf16 block; ``is_local`` selects sliding-window vs
|
||||
# A force-dense-MLP block; ``is_local`` selects sliding-window vs
|
||||
# full attention (the swa_* head config and sliding_window window) to
|
||||
# match this depth's checkpoint transformer_block weights.
|
||||
self.transformer_block = InklingDecoderLayer(
|
||||
config,
|
||||
layer_id=0,
|
||||
is_local=is_local,
|
||||
quant_config=None,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.transformer_block",
|
||||
force_dense_mlp=True,
|
||||
)
|
||||
@@ -131,7 +143,10 @@ class InklingMultiTokenPredictor(nn.Module):
|
||||
self.layers = nn.ModuleDict(
|
||||
{
|
||||
str(idx): InklingMTPDepthLayer(
|
||||
config, f"{prefix}.layers.{idx}", idx in local_ids
|
||||
config,
|
||||
f"{prefix}.layers.{idx}",
|
||||
idx in local_ids,
|
||||
quant_config=vllm_config.quant_config,
|
||||
)
|
||||
for idx in range(self.num_mtp_layers)
|
||||
}
|
||||
@@ -244,6 +259,11 @@ class InklingMultiTokenPredictor(nn.Module):
|
||||
|
||||
|
||||
class InklingMTP(nn.Module, SupportsMultiModalEmbeddings):
|
||||
hf_to_vllm_mapper = _TmlForCausalLMBase.hf_to_vllm_mapper | WeightsMapper(
|
||||
orig_to_new_prefix={"model.mtp.": "model."},
|
||||
)
|
||||
packed_modules_mapping = _TmlForCausalLMBase.packed_modules_mapping
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
assert vllm_config.speculative_config is not None
|
||||
@@ -418,9 +438,9 @@ def _load_inkling_mtp_weights(
|
||||
continue
|
||||
|
||||
# Dense MLP fused gate/up + down.
|
||||
if ".mlp.w13_dn.weight" in name:
|
||||
if ".mlp.w13_dn." in name:
|
||||
loaded_weight = _load(name.replace(".w13_dn.", ".gate_up_proj."), weight)
|
||||
elif ".mlp.w2_md.weight" in name:
|
||||
elif ".mlp.w2_md." in name:
|
||||
loaded_weight = _load(name.replace(".w2_md.", ".down_proj."), weight)
|
||||
else:
|
||||
if name.endswith(".bias") and name not in params:
|
||||
|
||||
@@ -4,12 +4,15 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import Any, cast
|
||||
|
||||
import numpy as np
|
||||
import regex as re
|
||||
import torch
|
||||
from PIL import Image
|
||||
from transformers import InklingProcessor
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
|
||||
from vllm.config.multimodal import (
|
||||
@@ -29,18 +32,19 @@ from vllm.multimodal.processing import (
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
)
|
||||
from vllm.transformers_utils.processors.inkling import (
|
||||
AUDIO_MARKER_ID,
|
||||
AUDIO_TOKEN_ID,
|
||||
IMAGE_MARKER_ID,
|
||||
IMAGE_TOKEN_ID,
|
||||
InklingAudioFeatureExtractor,
|
||||
InklingImageProcessor,
|
||||
InklingProcessor,
|
||||
)
|
||||
|
||||
from ..configs import InklingMMConfig
|
||||
|
||||
# Block-start markers (<|content_image|>, <|content_audio_input|>), kept verbatim
|
||||
IMAGE_MARKER_ID = 200005
|
||||
AUDIO_MARKER_ID = 200020
|
||||
# Per-patch / per-frame placeholders (<|unused_200054|>, <|unused_200053|>)
|
||||
IMAGE_TOKEN_ID = 200054
|
||||
AUDIO_TOKEN_ID = 200053
|
||||
|
||||
# Long-edge upscale applied before patchifying (upstream defaults to no rescale)
|
||||
DEFAULT_RESCALE_IMAGE_FRAC = 2.0
|
||||
|
||||
# Maximum audio tokens accepted per clip. At the dMel rate of 20 tokens/s
|
||||
# (50 ms hop) this is ~10 minutes of audio. It bounds the persistent per-request
|
||||
# buffers and the encoder/memory budget; longer clips are rejected up front.
|
||||
@@ -57,6 +61,24 @@ class InklingMultiModalDataParser(MultiModalDataParser):
|
||||
return super()._parse_audio_data(data)
|
||||
|
||||
|
||||
def _rescale_image(
|
||||
image: Image.Image, frac: float | None, max_upscaled_long_edge: int | None
|
||||
) -> Image.Image:
|
||||
"""Scale the long edge by `frac` with PIL LANCZOS, as the reference does."""
|
||||
image = image.convert("RGB")
|
||||
if frac is None:
|
||||
return image
|
||||
long_edge = max(image.size)
|
||||
target_long_edge = long_edge * frac
|
||||
if max_upscaled_long_edge is not None:
|
||||
target_long_edge = min(target_long_edge, max(max_upscaled_long_edge, long_edge))
|
||||
ratio = target_long_edge / long_edge
|
||||
if ratio == 1.0:
|
||||
return image
|
||||
size = tuple(max(1, math.floor(d * ratio + 0.5)) for d in image.size)
|
||||
return image.resize(size, resample=Image.Resampling.LANCZOS)
|
||||
|
||||
|
||||
def inkling_vision_enabled(config: InklingMMConfig) -> bool:
|
||||
return getattr(config.vision_config, "decoder_dmodel", None) is not None
|
||||
|
||||
@@ -70,38 +92,13 @@ class InklingProcessingInfo(BaseProcessingInfo):
|
||||
return self.ctx.get_hf_config(InklingMMConfig)
|
||||
|
||||
def get_hf_processor(self, **kwargs: object) -> InklingProcessor:
|
||||
config = self.get_hf_config()
|
||||
vision_config = config.vision_config
|
||||
audio_config = config.audio_config
|
||||
|
||||
image_processor = InklingImageProcessor(
|
||||
patch_size=getattr(vision_config, "patch_size", None) or 40,
|
||||
)
|
||||
|
||||
if inkling_audio_enabled(config):
|
||||
audio_params = {
|
||||
"n_mels": audio_config.n_mel_bins,
|
||||
"num_dmel_bins": audio_config.mel_vocab_size,
|
||||
"dmel_min_value": audio_config.dmel_min_value,
|
||||
"dmel_max_value": audio_config.dmel_max_value,
|
||||
}
|
||||
else:
|
||||
audio_params = {}
|
||||
audio_extractor = InklingAudioFeatureExtractor(params=audio_params)
|
||||
|
||||
return InklingProcessor(
|
||||
image_processor=image_processor,
|
||||
audio_feature_extractor=audio_extractor,
|
||||
tokenizer=self.get_tokenizer(),
|
||||
)
|
||||
return self.ctx.get_hf_processor(InklingProcessor, **kwargs)
|
||||
|
||||
def get_data_parser(self) -> MultiModalDataParser:
|
||||
# Audio inputs must be resampled to the dMel feature extractor's rate
|
||||
# before `process_audios` (see InklingAudioFeatureExtractor._decode_one).
|
||||
# Without a target_sr the default parser raises on any audio input.
|
||||
extractor = self.get_hf_processor().audio_feature_extractor
|
||||
# The feature extractor requires audio at its sampling rate
|
||||
feature_extractor = self.get_hf_processor().feature_extractor
|
||||
return InklingMultiModalDataParser(
|
||||
target_sr=extractor.params.sample_rate,
|
||||
target_sr=feature_extractor.sampling_rate,
|
||||
target_channels=1,
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
)
|
||||
@@ -163,9 +160,8 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]):
|
||||
if num_audios:
|
||||
# Size the dummy at the maximum allowed audio so memory/encoder
|
||||
# budgeting reflects the largest clip we accept (MAX_AUDIO_TOKENS).
|
||||
params = self.info.get_hf_processor().audio_feature_extractor.params
|
||||
hop = int(round(params.audio_token_duration_s * params.sample_rate))
|
||||
audio_len = MAX_AUDIO_TOKENS * hop
|
||||
feature_extractor = self.info.get_hf_processor().feature_extractor
|
||||
audio_len = MAX_AUDIO_TOKENS * feature_extractor.hop_length
|
||||
mm_data["audio"] = self._get_dummy_audios(
|
||||
length=audio_len,
|
||||
num_audios=num_audios,
|
||||
@@ -203,31 +199,37 @@ class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo])
|
||||
data: dict[str, Any] = {"input_ids": [prompt_ids]}
|
||||
|
||||
if images:
|
||||
img_feat = processor.process_images(images)
|
||||
data["pixel_values"] = img_feat["vision_patches_bthwc"]
|
||||
data["num_patches"] = torch.tensor(
|
||||
img_feat["num_patches"], dtype=torch.int64
|
||||
image_processor = processor.image_processor
|
||||
frac = image_processor.rescale_image_frac or DEFAULT_RESCALE_IMAGE_FRAC
|
||||
max_long_edge = image_processor.rescale_image_max_upscaled_long_edge
|
||||
# Resize with PIL: torchvision LANCZOS does not match the reference
|
||||
images = [_rescale_image(img, frac, max_long_edge) for img in images]
|
||||
img_feat = image_processor(
|
||||
images, rescale_image_frac=None, return_tensors="pt"
|
||||
)
|
||||
data["pixel_values"] = img_feat["pixel_values"].to(torch.bfloat16)
|
||||
data["num_patches"] = img_feat["num_patches"].to(torch.int64)
|
||||
|
||||
if audios:
|
||||
aud_feat = processor.process_audios(audios)
|
||||
per_clip = aud_feat["dmel_bins"]
|
||||
num_audio_tokens = aud_feat["num_audio_tokens"]
|
||||
for i, n in enumerate(num_audio_tokens):
|
||||
if int(n) > MAX_AUDIO_TOKENS:
|
||||
feature_extractor = processor.feature_extractor
|
||||
aud_feat = feature_extractor(
|
||||
audios,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
return_tensors="pt",
|
||||
)
|
||||
audio_input_ids = processor._extract_dmel_bins(aud_feat["input_features"])
|
||||
num_audio_tokens = aud_feat["input_features_mask"].sum(-1)
|
||||
for i, n in enumerate(num_audio_tokens.tolist()):
|
||||
if n > MAX_AUDIO_TOKENS:
|
||||
raise ValueError(
|
||||
f"Audio clip {i} produces {int(n)} tokens, exceeding the "
|
||||
f"Audio clip {i} produces {n} tokens, exceeding the "
|
||||
f"maximum of {MAX_AUDIO_TOKENS} (~10 min at 20 tokens/s). "
|
||||
"Provide a shorter clip."
|
||||
)
|
||||
if per_clip:
|
||||
input_audio_features = torch.cat(
|
||||
[torch.as_tensor(c) for c in per_clip], dim=0
|
||||
)
|
||||
else:
|
||||
input_audio_features = torch.empty(0)
|
||||
data["input_audio_features"] = input_audio_features
|
||||
data["num_audio_tokens"] = torch.tensor(num_audio_tokens, dtype=torch.int64)
|
||||
data["input_audio_features"] = torch.cat(
|
||||
[ids[:n] for ids, n in zip(audio_input_ids, num_audio_tokens)]
|
||||
)
|
||||
data["num_audio_tokens"] = num_audio_tokens.to(torch.int64)
|
||||
|
||||
processed_data = BatchFeature(data=data, tensor_type=None)
|
||||
return self._finalize_hf_mm_data(
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
Mirrors the reference ``mtp_model.py`` shipped with the checkpoint: each MTP
|
||||
depth ``i`` owns ``hidden_norm`` / ``embed_norm`` RMSNorms, an ``input_proj``
|
||||
(``2H -> H``) and a full Inkling transformer block (dense bf16 MLP, with the
|
||||
(``2H -> H``) and a full Inkling transformer block (dense MLP, with the
|
||||
same short convolutions as the backbone; its attention is full or sliding-window
|
||||
per depth, selected by ``mtp_config.local_layer_ids``). When enabled, a shared
|
||||
``chain_norm`` is applied after every depth; its output is both the logits input
|
||||
@@ -28,18 +28,23 @@ from torch import nn
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.model_loader.mtp_validation import (
|
||||
is_mtp_completeness_check_enabled,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.interfaces import SupportsMultiModalEmbeddings
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.model_executor.models.utils import WeightsMapper, maybe_prefix
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from ..configs import InklingModelConfig
|
||||
from .layernorm import InklingRMSNorm
|
||||
from .model import InklingDecoderLayer, InklingReplicatedEmbedding
|
||||
from .model import (
|
||||
InklingDecoderLayer,
|
||||
InklingReplicatedEmbedding,
|
||||
_TmlForCausalLMBase,
|
||||
)
|
||||
from .ops.norm import embed_dual_rmsnorm_cat, embed_rmsnorm
|
||||
|
||||
# Checkpoint attention projections (wq_du/wk_dv/wv_dv/wr_du) -> fused qkvr.
|
||||
@@ -71,7 +76,13 @@ def _select_mtp_depth_count(n_predict: int, num_spec: int | None) -> int:
|
||||
class InklingMTPDepthLayer(nn.Module):
|
||||
"""One MTP depth: norm both inputs, fuse (2H->H), run a Inkling block."""
|
||||
|
||||
def __init__(self, config: InklingModelConfig, prefix: str, is_local: bool) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
config: InklingModelConfig,
|
||||
prefix: str,
|
||||
is_local: bool,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.embed_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
@@ -80,16 +91,17 @@ class InklingMTPDepthLayer(nn.Module):
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.input_proj",
|
||||
)
|
||||
# A force-dense-MLP bf16 block; ``is_local`` selects sliding-window vs
|
||||
# A force-dense-MLP block; ``is_local`` selects sliding-window vs
|
||||
# full attention (the swa_* head config and sliding_window window) to
|
||||
# match this depth's checkpoint transformer_block weights.
|
||||
self.transformer_block = InklingDecoderLayer(
|
||||
config,
|
||||
layer_id=0,
|
||||
is_local=is_local,
|
||||
quant_config=None,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.transformer_block",
|
||||
force_dense_mlp=True,
|
||||
)
|
||||
@@ -131,7 +143,10 @@ class InklingMultiTokenPredictor(nn.Module):
|
||||
self.layers = nn.ModuleDict(
|
||||
{
|
||||
str(idx): InklingMTPDepthLayer(
|
||||
config, f"{prefix}.layers.{idx}", idx in local_ids
|
||||
config,
|
||||
f"{prefix}.layers.{idx}",
|
||||
idx in local_ids,
|
||||
quant_config=vllm_config.quant_config,
|
||||
)
|
||||
for idx in range(self.num_mtp_layers)
|
||||
}
|
||||
@@ -244,6 +259,11 @@ class InklingMultiTokenPredictor(nn.Module):
|
||||
|
||||
|
||||
class InklingMTP(nn.Module, SupportsMultiModalEmbeddings):
|
||||
hf_to_vllm_mapper = _TmlForCausalLMBase.hf_to_vllm_mapper | WeightsMapper(
|
||||
orig_to_new_prefix={"model.mtp.": "model."},
|
||||
)
|
||||
packed_modules_mapping = _TmlForCausalLMBase.packed_modules_mapping
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
assert vllm_config.speculative_config is not None
|
||||
@@ -418,9 +438,9 @@ def _load_inkling_mtp_weights(
|
||||
continue
|
||||
|
||||
# Dense MLP fused gate/up + down.
|
||||
if ".mlp.w13_dn.weight" in name:
|
||||
if ".mlp.w13_dn." in name:
|
||||
loaded_weight = _load(name.replace(".w13_dn.", ".gate_up_proj."), weight)
|
||||
elif ".mlp.w2_md.weight" in name:
|
||||
elif ".mlp.w2_md." in name:
|
||||
loaded_weight = _load(name.replace(".w2_md.", ".down_proj."), weight)
|
||||
else:
|
||||
if name.endswith(".bias") and name not in params:
|
||||
|
||||
@@ -95,7 +95,9 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
assert isinstance(spec, MambaSpec)
|
||||
# Only the fused path exports the checkpoint snapshots
|
||||
can_checkpoint = (
|
||||
self.use_prefill_checkpoint and self.use_fused_chunk and self.use_safe_gate
|
||||
self.use_prefill_checkpoint
|
||||
and self.kda_prefill_backend == "fused"
|
||||
and self.use_safe_gate
|
||||
)
|
||||
if can_checkpoint and vllm_config.cache_config.mamba_cache_mode == "align":
|
||||
logger.info_once("Kimi-K3 KDA prefill checkpoint enabled.")
|
||||
@@ -132,6 +134,14 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
self.conv_size = kda_config["short_conv_kernel_size"]
|
||||
self.use_full_rank_gate = True
|
||||
|
||||
self._fused_qkv_conv_fn = None
|
||||
if rocm_aiter_ops.is_enabled():
|
||||
from aiter.ops.triton.gated_delta_net import (
|
||||
causal_conv1d_split_qkv_triton_tile_fn,
|
||||
)
|
||||
|
||||
self._fused_qkv_conv_fn = causal_conv1d_split_qkv_triton_tile_fn
|
||||
|
||||
# Keep f_a before the narrow beta shard, then pad each TP-local row to
|
||||
# select the aligned BF16 GEMM path. The padding also avoids an Inductor
|
||||
# correctness issue seen with the row-strided G view.
|
||||
@@ -242,13 +252,15 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
"The fused KDA chunk kernel requires gfx950 and a build that "
|
||||
"includes it."
|
||||
)
|
||||
self.use_fused_chunk = backend == "fused" or (
|
||||
backend == "auto" and is_fused_kda_chunk_supported()
|
||||
)
|
||||
logger.info_once(
|
||||
"Kimi-K3 KDA prefill backend: %s",
|
||||
"fused" if self.use_fused_chunk else "triton",
|
||||
)
|
||||
self.kda_prefill_backend = backend
|
||||
if backend == "auto":
|
||||
if self.use_safe_gate and rocm_aiter_ops.is_kda_gluon_prefill_available():
|
||||
self.kda_prefill_backend = "gluon"
|
||||
elif is_fused_kda_chunk_supported():
|
||||
self.kda_prefill_backend = "fused"
|
||||
else:
|
||||
self.kda_prefill_backend = "triton"
|
||||
logger.info_once("Kimi-K3 KDA prefill backend: %s", self.kda_prefill_backend)
|
||||
# TODO: Remove the extra config arg once tested
|
||||
self.use_prefill_checkpoint = (
|
||||
additional_config.get("kda_prefill_checkpoint", True)
|
||||
@@ -576,35 +588,48 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
if mixed_qkv_ns is not None:
|
||||
assert g1_ns is not None and beta_ns is not None
|
||||
if m.num_prefills > 0:
|
||||
q_ns, k_ns, v_ns = mixed_qkv_ns.split(
|
||||
self.local_projection_size, dim=-1
|
||||
)
|
||||
|
||||
# Packed prefill conv would require copying V solely to make
|
||||
# it dense for KDA. Separate calls accept the strided inputs
|
||||
# and produce dense Q/K/V without that extra traffic.
|
||||
# TODO: Use packed conv once every KDA prefill backend accepts
|
||||
# row-strided Q/K/V directly.
|
||||
def _prefill_conv(
|
||||
x: torch.Tensor,
|
||||
state: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return causal_conv1d_fn(
|
||||
x.transpose(0, 1),
|
||||
weight,
|
||||
if self._fused_qkv_conv_fn is not None:
|
||||
# One launch (fused qkv) instead of 3
|
||||
q_ns, k_ns, v_ns = self._fused_qkv_conv_fn(
|
||||
mixed_qkv_ns.transpose(0, 1),
|
||||
conv_weights,
|
||||
None,
|
||||
activation="silu",
|
||||
conv_states=state,
|
||||
has_initial_state=has_initial_state,
|
||||
conv_state,
|
||||
non_spec_query_start_loc,
|
||||
self.local_projection_size,
|
||||
self.local_projection_size,
|
||||
cache_indices=non_spec_state_indices_tensor,
|
||||
query_start_loc=non_spec_query_start_loc,
|
||||
has_initial_state=has_initial_state,
|
||||
activation="silu",
|
||||
block_m=8,
|
||||
metadata=m,
|
||||
).transpose(0, 1)
|
||||
preserve_input_dtype=True,
|
||||
)
|
||||
else:
|
||||
q_ns, k_ns, v_ns = mixed_qkv_ns.split(
|
||||
self.local_projection_size, dim=-1
|
||||
)
|
||||
|
||||
q_ns = _prefill_conv(q_ns, q_conv_state, q_conv_weight)
|
||||
k_ns = _prefill_conv(k_ns, k_conv_state, k_conv_weight)
|
||||
v_ns = _prefill_conv(v_ns, v_conv_state, v_conv_weight)
|
||||
def _prefill_conv(
|
||||
x: torch.Tensor,
|
||||
state: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return causal_conv1d_fn(
|
||||
x.transpose(0, 1),
|
||||
weight,
|
||||
None,
|
||||
activation="silu",
|
||||
conv_states=state,
|
||||
has_initial_state=has_initial_state,
|
||||
cache_indices=non_spec_state_indices_tensor,
|
||||
query_start_loc=non_spec_query_start_loc,
|
||||
metadata=m,
|
||||
).transpose(0, 1)
|
||||
|
||||
q_ns = _prefill_conv(q_ns, q_conv_state, q_conv_weight)
|
||||
k_ns = _prefill_conv(k_ns, k_conv_state, k_conv_weight)
|
||||
v_ns = _prefill_conv(v_ns, v_conv_state, v_conv_weight)
|
||||
q_ns, k_ns, v_ns = (
|
||||
rearrange(x, "n (h d) -> 1 n h d", d=self.head_dim)
|
||||
for x in (q_ns, k_ns, v_ns)
|
||||
@@ -694,7 +719,7 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
cu_seqlens=prefill_query_start_loc,
|
||||
chunk_indices=m.chunk_indices,
|
||||
chunk_offsets=m.chunk_offsets,
|
||||
use_fused_chunk=self.use_fused_chunk,
|
||||
backend=self.kda_prefill_backend,
|
||||
out=core_attn_out[:, nd_tok:num_actual_tokens] if direct else None,
|
||||
checkpoint_state=checkpoint_state,
|
||||
checkpoint_offsets=checkpoint_offsets,
|
||||
@@ -748,15 +773,9 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
|
||||
# ---------- merge spec and non-spec outputs ----------
|
||||
if core_attn_out_spec is not None and core_attn_out_non_spec is not None:
|
||||
# Mixed batches require indexed placement in the original order.
|
||||
merged = torch.empty(
|
||||
(1, num_actual_tokens, *core_attn_out_spec.shape[2:]),
|
||||
dtype=core_attn_out_spec.dtype,
|
||||
device=core_attn_out_spec.device,
|
||||
)
|
||||
merged = core_attn_out[:, :num_actual_tokens]
|
||||
merged.index_copy_(1, spec_token_indx, core_attn_out_spec)
|
||||
merged.index_copy_(1, non_spec_token_indx, core_attn_out_non_spec)
|
||||
core_attn_out[0, :num_actual_tokens] = merged[0, :num_actual_tokens]
|
||||
elif core_attn_out_non_spec is not None:
|
||||
if core_attn_out_non_spec.data_ptr() != core_attn_out.data_ptr():
|
||||
core_attn_out[0, :num_actual_tokens] = core_attn_out_non_spec[
|
||||
|
||||
@@ -11,6 +11,7 @@ from dataclasses import replace
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointMetadata,
|
||||
@@ -18,12 +19,14 @@ from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
from vllm.v1.attention.backend import max_decode_query_len
|
||||
from vllm.v1.attention.backends.gdn_attn import (
|
||||
GDNAttentionBackend,
|
||||
GDNAttentionMetadata,
|
||||
GDNAttentionMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import CommonAttentionMetadata
|
||||
from vllm.v1.kv_cache_interface import KVCacheSpec
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -92,7 +95,24 @@ def prepare_chunk_metadata_device(
|
||||
return chunk_indices, chunk_offsets
|
||||
|
||||
|
||||
def _adaptive_verification_enabled(vllm_config: VllmConfig) -> bool:
|
||||
speculative_config = vllm_config.speculative_config
|
||||
return speculative_config is not None and bool(
|
||||
getattr(speculative_config, "enable_adaptive_verification", False)
|
||||
)
|
||||
|
||||
|
||||
class KimiK3ROCmKDAMetadataBuilder(GDNAttentionMetadataBuilder):
|
||||
@classmethod
|
||||
def get_varlen_cudagraph_max_query_len(
|
||||
cls: type["KimiK3ROCmKDAMetadataBuilder"],
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: KVCacheSpec,
|
||||
) -> int | None:
|
||||
if not _adaptive_verification_enabled(vllm_config):
|
||||
return None
|
||||
return max_decode_query_len(vllm_config)
|
||||
|
||||
def build( # type: ignore[override]
|
||||
self,
|
||||
common_prefix_len: int,
|
||||
@@ -108,6 +128,11 @@ class KimiK3ROCmKDAMetadataBuilder(GDNAttentionMetadataBuilder):
|
||||
num_decode_draft_tokens_cpu,
|
||||
fast_build,
|
||||
)
|
||||
if (
|
||||
attn_metadata.uniform_spec_sequence_length is not None
|
||||
and _adaptive_verification_enabled(self.vllm_config)
|
||||
):
|
||||
attn_metadata = replace(attn_metadata, uniform_spec_sequence_length=None)
|
||||
if attn_metadata.checkpoint is None:
|
||||
return attn_metadata
|
||||
# The shared builder keys checkpoint rows on every request; the ROCm
|
||||
@@ -163,3 +188,7 @@ class KimiK3ROCmKDABackend(GDNAttentionBackend):
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type[KimiK3ROCmKDAMetadataBuilder]:
|
||||
return KimiK3ROCmKDAMetadataBuilder
|
||||
|
||||
@classmethod
|
||||
def supports_device_cpu_query_lens_mismatch(cls) -> bool:
|
||||
return True
|
||||
|
||||
@@ -23,8 +23,8 @@ _OPT_MIN_SIZE = 2048
|
||||
|
||||
|
||||
class KimiK3MultiHeadLatentAttentionWrapper(MultiHeadLatentAttentionWrapper):
|
||||
"""Kimi-K3 MLA wrapper with eager AITER q/kv RMSNorm fusion and a fused
|
||||
decode Q-prep path."""
|
||||
"""Kimi-K3 MLA wrapper with eager AITER q/kv RMSNorm and output gate
|
||||
fusions and a fused decode Q-prep path."""
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
@@ -303,6 +303,12 @@ class KimiK3MultiHeadLatentAttentionWrapper(MultiHeadLatentAttentionWrapper):
|
||||
)
|
||||
|
||||
if self.g_proj is not None:
|
||||
attn_out = attn_out * self.g_proj(hidden_states)[0].sigmoid()
|
||||
gate = self.g_proj(hidden_states)[0]
|
||||
if rocm_aiter_ops.is_enabled():
|
||||
from aiter.ops.triton.fusions.fused_sigmoid_mul import fused_sigmoid_mul
|
||||
|
||||
attn_out = fused_sigmoid_mul(attn_out, gate)
|
||||
else:
|
||||
attn_out = attn_out * gate.sigmoid()
|
||||
|
||||
return self.o_proj(attn_out)[0]
|
||||
|
||||
@@ -9,6 +9,7 @@ path.
|
||||
|
||||
import torch
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.mamba.ops.gather_initial_states import (
|
||||
gather_initial_states,
|
||||
@@ -24,6 +25,11 @@ from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
# Consumed by kimi_k3_triton_warmup.py during kernel_warmup().
|
||||
def get_gluon_chunk_kda_warmup_profiles() -> tuple[tuple[int, int], ...]:
|
||||
return ((1, 64), (2, 64), (4, 64), (8, 64), (16, 64), (1, 4096), (2, 2048))
|
||||
|
||||
|
||||
def chunk_kda_prefill(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
@@ -40,7 +46,7 @@ def chunk_kda_prefill(
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
chunk_indices: torch.Tensor | None = None,
|
||||
chunk_offsets: torch.Tensor | None = None,
|
||||
use_fused_chunk: bool = False,
|
||||
backend: str = "triton",
|
||||
out: torch.Tensor | None = None,
|
||||
checkpoint_state: torch.Tensor | None = None,
|
||||
checkpoint_offsets: torch.Tensor | None = None,
|
||||
@@ -68,9 +74,10 @@ def chunk_kda_prefill(
|
||||
cu_seqlens: int32 cumulative sequence lengths.
|
||||
chunk_indices: precomputed chunk indices for ``cu_seqlens``.
|
||||
chunk_offsets: int32 per-sequence first chunk index.
|
||||
use_fused_chunk: request the two-kernel ROCm path. It is used only when
|
||||
every one of its preconditions holds; otherwise the Triton path
|
||||
runs unchanged.
|
||||
backend: ``"triton"`` (the vendored chunk path), ``"fused"`` (the
|
||||
two-kernel ROCm path; used only when every one of its
|
||||
preconditions holds, otherwise the Triton path runs unchanged) or
|
||||
``"gluon"`` (AITER's ``chunk_kda``).
|
||||
out: buffer the result must land in. Honoured by both backends, so the
|
||||
caller can hand in a slice of its own output and skip a copy.
|
||||
checkpoint_state: destination for mid-prefill recurrent state
|
||||
@@ -89,6 +96,11 @@ def chunk_kda_prefill(
|
||||
The output and, when requested, the final recurrent state.
|
||||
|
||||
"""
|
||||
if backend not in ("triton", "fused", "gluon"):
|
||||
raise ValueError(
|
||||
"chunk_kda_prefill backend must be 'triton', 'fused' or 'gluon', "
|
||||
f"got {backend!r}."
|
||||
)
|
||||
if scale is None:
|
||||
scale = k.shape[-1] ** -0.5
|
||||
|
||||
@@ -96,7 +108,7 @@ def chunk_kda_prefill(
|
||||
# needs the raw projections and a bounded gate rather than the pre-normalized
|
||||
# tensors the Triton path takes.
|
||||
fused = (
|
||||
use_fused_chunk
|
||||
backend == "fused"
|
||||
and use_qk_l2norm_in_kernel
|
||||
and cu_seqlens is not None
|
||||
and lower_bound is not None
|
||||
@@ -116,6 +128,28 @@ def chunk_kda_prefill(
|
||||
if state_indices is None or has_initial_state is None:
|
||||
raise ValueError("state_cache needs state_indices and has_initial_state")
|
||||
|
||||
if backend == "gluon":
|
||||
return rocm_aiter_ops.kda_chunk_prefill(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=raw_g,
|
||||
beta=raw_beta,
|
||||
A_log=A_log,
|
||||
dt_bias=g_bias,
|
||||
lower_bound=lower_bound,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
chunk_offsets=chunk_offsets,
|
||||
scale=scale,
|
||||
out=out,
|
||||
initial_state=initial_state,
|
||||
output_final_state=output_final_state,
|
||||
state_cache=state_cache,
|
||||
state_indices=state_indices,
|
||||
has_initial_state=has_initial_state,
|
||||
)
|
||||
|
||||
# Only the fused walk addresses the paged rows directly. The Triton path
|
||||
# gathers the rows it needs and scatters the results back here, so callers
|
||||
# pass the cache the same way for either backend.
|
||||
|
||||
@@ -104,12 +104,12 @@ def convert_image_mode(
|
||||
to_mode: str,
|
||||
background_color: tuple[int, int, int] | list[int] = (255, 255, 255),
|
||||
) -> Image.Image:
|
||||
if image.mode == to_mode:
|
||||
return image
|
||||
|
||||
if to_mode == "RGB" and _has_transparency(image):
|
||||
if image.mode != "RGBA":
|
||||
image = image.convert("RGBA")
|
||||
return rgba_to_rgb(image, background_color)
|
||||
|
||||
if image.mode == to_mode:
|
||||
return image
|
||||
|
||||
return image.convert(to_mode)
|
||||
|
||||
@@ -16,7 +16,7 @@ from vllm.utils.sparse_utils import (
|
||||
safe_to_dense,
|
||||
)
|
||||
|
||||
from ..image import convert_image_mode, normalize_image, open_image, rgba_to_rgb
|
||||
from ..image import convert_image_mode, normalize_image, open_image
|
||||
from .base import MediaDecodedBudget, MediaIO, MediaWithBytes
|
||||
|
||||
MAGIC_NUMPY_PREFIX = b"\x93NUMPY" # https://numpy.org/devdocs/reference/generated/numpy.lib.format.html#format-version-1-0
|
||||
@@ -74,14 +74,9 @@ class ImageMediaIO(MediaIO[Image.Image]):
|
||||
"""Convert image mode with custom background color."""
|
||||
if isinstance(image, MediaWithBytes):
|
||||
image = image.media
|
||||
if self.image_mode is None or image.mode == self.image_mode:
|
||||
if self.image_mode is None:
|
||||
return image
|
||||
elif image.mode == "RGBA" and self.image_mode == "RGB":
|
||||
return rgba_to_rgb(image, self.rgba_background_color)
|
||||
else:
|
||||
return convert_image_mode(
|
||||
image, self.image_mode, self.rgba_background_color
|
||||
)
|
||||
return convert_image_mode(image, self.image_mode, self.rgba_background_color)
|
||||
|
||||
def load_bytes(self, data: bytes) -> MediaWithBytes[Image.Image]:
|
||||
try:
|
||||
|
||||
@@ -769,6 +769,10 @@ class DelegatingParser(Parser):
|
||||
return False
|
||||
return self._reasoning_parser.is_reasoning_end(input_ids)
|
||||
|
||||
def set_prompt_token_ids(self, prompt_token_ids: Sequence[int]) -> None:
|
||||
if self._reasoning_parser is not None:
|
||||
self._reasoning_parser.set_prompt_token_ids(prompt_token_ids)
|
||||
|
||||
def _is_reasoning_end_streaming(
|
||||
self, input_ids: list[int], delta_ids: list[int]
|
||||
) -> bool:
|
||||
@@ -858,6 +862,7 @@ class DelegatingParser(Parser):
|
||||
|
||||
if not state.prompt_reasoning_checked and prompt_token_ids is not None:
|
||||
state.prompt_reasoning_checked = True
|
||||
self.set_prompt_token_ids(prompt_token_ids)
|
||||
if self._reasoning_parser is None or self.is_reasoning_end(
|
||||
prompt_token_ids
|
||||
):
|
||||
|
||||
@@ -0,0 +1,262 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Parser for Beam checkpoints."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
from openai.types.responses import ToolChoiceFunction
|
||||
|
||||
from vllm import envs
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionNamedToolChoiceParam,
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.parser.abstract_parser import (
|
||||
_xgrammar_supports,
|
||||
structured_outputs_to_format,
|
||||
)
|
||||
from vllm.parser.chat_parsing import ResponseParser
|
||||
from vllm.parser.chat_parsing.response_templates import ResponseTemplate
|
||||
from vllm.parser.response_template import (
|
||||
CONTENT_FIELD,
|
||||
THINKING_FIELD,
|
||||
TOOL_FIELD,
|
||||
ResponseTemplateParser,
|
||||
ResponseTemplateReasoningParser,
|
||||
ResponseTemplateToolParser,
|
||||
_as_tool_dicts,
|
||||
_is_named_tool_choice,
|
||||
_tool_parsing_enabled,
|
||||
)
|
||||
from vllm.parser.response_template_grammar import (
|
||||
_openers,
|
||||
answer_rest_structural_tag,
|
||||
message_start_anchors,
|
||||
tool_call_structural_tag,
|
||||
)
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
from vllm.tool_parsers.structural_tag_registry import resolve_tool_strictness
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _named_tool(request: ChatCompletionRequest | ResponsesRequest) -> str:
|
||||
choice = request.tool_choice
|
||||
if isinstance(choice, ToolChoiceFunction):
|
||||
return choice.name
|
||||
return cast(ChatCompletionNamedToolChoiceParam, choice).function.name
|
||||
|
||||
|
||||
def _starts_message(template: ResponseTemplate, prefix: str) -> bool:
|
||||
"""Whether output after `prefix` starts the assistant message."""
|
||||
ends = [match.end() for match in template.start_anchor_re.finditer(prefix)]
|
||||
return bool(ends) and ends[-1] == len(prefix)
|
||||
|
||||
|
||||
def _open_field(template: ResponseTemplate, prefix: str) -> str | None:
|
||||
"""The field that `prefix` leaves open with its explicit opener, if any."""
|
||||
field = None
|
||||
for event in ResponseParser(template, prefix=prefix).initial_events:
|
||||
if event["type"] == "region_open" and event["end"] > event["start"]:
|
||||
field = event["field"]
|
||||
elif event["type"] in {"region_close", "region_malformed"}:
|
||||
field = None
|
||||
return field
|
||||
|
||||
|
||||
class BeamReasoningParser(ResponseTemplateReasoningParser):
|
||||
"""Selects `BeamParser` as the reasoning parser."""
|
||||
|
||||
@classmethod
|
||||
def unified_parser_cls(cls) -> type[ResponseTemplateParser]:
|
||||
return BeamParser
|
||||
|
||||
@property
|
||||
def reasoning_start_str(self) -> str | None:
|
||||
if literal := super().reasoning_start_str:
|
||||
return literal
|
||||
thinking = self.response_template.fields.get(THINKING_FIELD)
|
||||
if thinking is None:
|
||||
return None
|
||||
openers = _openers(thinking, message_start_anchors(self.response_template))
|
||||
if not openers:
|
||||
return None
|
||||
# Budget tracking needs one marker for both a prompt continuation
|
||||
# and a complete assistant message in the checkpoint's pattern.
|
||||
marker = min(openers, key=len)
|
||||
return marker if all(opener.endswith(marker) for opener in openers) else None
|
||||
|
||||
|
||||
class BeamToolParser(ResponseTemplateToolParser):
|
||||
"""Selects `BeamParser` as the tool parser."""
|
||||
|
||||
@classmethod
|
||||
def unified_parser_cls(cls) -> type[ResponseTemplateParser]:
|
||||
return BeamParser
|
||||
|
||||
|
||||
class BeamParser(ResponseTemplateParser):
|
||||
"""Parse Beam output with the checkpoint's `response_template` metadata,
|
||||
and constrain it with a grammar derived from that template."""
|
||||
|
||||
reasoning_parser_cls = BeamReasoningParser
|
||||
tool_parser_cls = BeamToolParser
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
chat_template_kwargs = kwargs.get("chat_template_kwargs") or {}
|
||||
# From the message start, the grammar opens the message with non-empty
|
||||
# reasoning, unless the request turns thinking off.
|
||||
self._requires_reasoning = bool(
|
||||
chat_template_kwargs.get("enable_thinking", True)
|
||||
)
|
||||
|
||||
def _constrain_request(
|
||||
self,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> ChatCompletionRequest | ResponsesRequest:
|
||||
message_start = _starts_message(self.response_template, self._prefix)
|
||||
tag = self._template_tag(request, message_start=message_start)
|
||||
if tag is not None:
|
||||
grammar = StructuredOutputsParams(structural_tag=json.dumps(tag))
|
||||
if isinstance(request, ChatCompletionRequest):
|
||||
request.structured_outputs = grammar
|
||||
request.response_format = None
|
||||
else:
|
||||
request._parser_structured_outputs = grammar
|
||||
# From the message start, the tag also covers the reasoning.
|
||||
request._grammar_from_parser = message_start
|
||||
return request
|
||||
|
||||
def _template_tag(
|
||||
self,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
*,
|
||||
message_start: bool,
|
||||
) -> dict[str, Any] | None:
|
||||
"""The template grammar for `request`, or `None` to leave its output
|
||||
to the request's own constraint, if any."""
|
||||
fields = self.response_template.fields
|
||||
named = _is_named_tool_choice(request)
|
||||
tools_enabled = (
|
||||
self._tool_parser is not None
|
||||
and bool(request.tools)
|
||||
and _tool_parsing_enabled(
|
||||
request, enable_auto_tools=self._enable_auto_tools
|
||||
)
|
||||
)
|
||||
forced = tools_enabled and (request.tool_choice == "required" or named)
|
||||
# The prompt may still list the tools, so only the grammar keeps the
|
||||
# model from calling them.
|
||||
tools_forbidden = (
|
||||
bool(request.tools)
|
||||
and request.tool_choice == "none"
|
||||
and TOOL_FIELD in fields
|
||||
)
|
||||
constrained_tools = None
|
||||
if tools_enabled and (forced or envs.VLLM_ENFORCE_STRICT_TOOL_CALLING):
|
||||
assert self._tool_parser is not None
|
||||
strict_level = self.tool_strict_level
|
||||
if strict_level is None:
|
||||
strict_level = self._tool_parser.default_tool_strict_level
|
||||
constrained_tools = resolve_tool_strictness(
|
||||
request.tools or [], request.tool_choice, strict_level
|
||||
)
|
||||
output = request.extract_structured_outputs()
|
||||
if forced and output is not None:
|
||||
logger.warning_once(
|
||||
"structured outputs are ignored because tool_choice forces tool call."
|
||||
)
|
||||
output = None
|
||||
answer = fields.get(CONTENT_FIELD)
|
||||
if output is not None and (
|
||||
output.structural_tag is not None
|
||||
or answer is None
|
||||
or answer.open_re is None
|
||||
or not _xgrammar_supports(output)
|
||||
):
|
||||
if constrained_tools is not None:
|
||||
logger.warning_once(
|
||||
"response_template parsing cannot enforce strict tools "
|
||||
"alongside this structured output."
|
||||
)
|
||||
return None
|
||||
# Without a tool or structured output constraint, the output still
|
||||
# follows the template.
|
||||
always_constrain = envs.VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT and not forced
|
||||
if (
|
||||
constrained_tools is None
|
||||
and output is None
|
||||
and not tools_forbidden
|
||||
and not always_constrain
|
||||
):
|
||||
return None
|
||||
|
||||
# Inside an open answer, only the always-on grammar applies, finishing a
|
||||
# free answer. Inside open reasoning, the grammar needs the reasoning
|
||||
# gate to defer it to the reasoning's end.
|
||||
open_field = _open_field(self.response_template, self._prefix)
|
||||
reasoning_gated = (
|
||||
self._reasoning_parser is not None and request.include_reasoning
|
||||
)
|
||||
tag = None
|
||||
if open_field == CONTENT_FIELD:
|
||||
if always_constrain and output is None:
|
||||
tag = answer_rest_structural_tag(fields[CONTENT_FIELD])
|
||||
elif open_field != THINKING_FIELD or reasoning_gated:
|
||||
functions = []
|
||||
if tools_enabled:
|
||||
tools = _as_tool_dicts(constrained_tools or request.tools)
|
||||
functions = [tool.get("function", tool) for tool in tools]
|
||||
if named:
|
||||
name = _named_tool(request)
|
||||
functions = [f for f in functions if f.get("name") == name]
|
||||
answer_format = (
|
||||
None if output is None else structured_outputs_to_format(output)
|
||||
)
|
||||
try:
|
||||
tag = tool_call_structural_tag(
|
||||
fields.get(TOOL_FIELD),
|
||||
functions,
|
||||
parallel_tool_calls=(
|
||||
not named and request.parallel_tool_calls is not False
|
||||
),
|
||||
thinking=fields.get(THINKING_FIELD),
|
||||
answer=None if forced else answer,
|
||||
answer_format=(
|
||||
None if answer_format is None else answer_format.model_dump()
|
||||
),
|
||||
message_start=(
|
||||
message_start_anchors(self.response_template)
|
||||
if message_start
|
||||
else None
|
||||
),
|
||||
requires_reasoning=self._requires_reasoning,
|
||||
# Beam drifts from schemas the grammar does not enforce,
|
||||
# and also writes arguments sorted by name.
|
||||
sorted_arguments=True,
|
||||
strict_arguments=True,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise VLLMValidationError(str(exc), parameter="tools") from exc
|
||||
if tag is None and forced:
|
||||
kind = "named" if named else "required"
|
||||
raise VLLMValidationError(
|
||||
f"response_template parsing cannot guarantee {kind} tool choice "
|
||||
"for this request",
|
||||
parameter="tool_choice",
|
||||
)
|
||||
if tag is None and constrained_tools is not None:
|
||||
logger.warning_once(
|
||||
"response_template parsing cannot enforce strict tools for this "
|
||||
"request."
|
||||
)
|
||||
return tag
|
||||
@@ -68,6 +68,11 @@ class ParserEngineReasoningAdapter(ReasoningParser):
|
||||
def find_reasoning_end(self, token_ids: Sequence[int]) -> ReasoningEnd | None:
|
||||
return self._parser_engine.find_reasoning_end(token_ids)
|
||||
|
||||
def set_prompt_token_ids(self, prompt_token_ids: Sequence[int]) -> None:
|
||||
self._parser_engine.set_prompt_token_ids(prompt_token_ids)
|
||||
if self._counting_parser_engine is not None:
|
||||
self._counting_parser_engine.set_prompt_token_ids(prompt_token_ids)
|
||||
|
||||
def adjust_initial_state_from_prompt(self, prompt_token_ids: Sequence[int]) -> None:
|
||||
self._parser_engine.adjust_initial_state_from_prompt(prompt_token_ids)
|
||||
|
||||
@@ -148,6 +153,9 @@ class ParserEngineReasoningAdapter(ReasoningParser):
|
||||
self._counting_parser_engine = self._parser_engine_cls(
|
||||
self.model_tokenizer, **self._parser_engine_kwargs
|
||||
) # type: ignore[call-arg]
|
||||
self._counting_parser_engine._prompt_closes_reasoning = (
|
||||
self._parser_engine._prompt_closes_reasoning
|
||||
)
|
||||
self._counting_parser_engine._single_pass_parse(
|
||||
self.model_tokenizer.decode(token_ids), token_ids
|
||||
)
|
||||
|
||||
@@ -147,6 +147,7 @@ class ParserEngine(Parser):
|
||||
or parser_engine_config.initial_state == ParserState.REASONING
|
||||
)
|
||||
self._reasoning_ended: bool = not self._has_reasoning
|
||||
self._prompt_closes_reasoning: bool = False
|
||||
self._streaming_initialized: bool = False
|
||||
self._prompt_streaming_prepared: bool = False
|
||||
|
||||
@@ -231,6 +232,18 @@ class ParserEngine(Parser):
|
||||
self._streaming_initialized = True
|
||||
self._reset(initial_state=initial_state)
|
||||
|
||||
def set_prompt_token_ids(self, prompt_token_ids: Sequence[int]) -> None:
|
||||
"""Start parsing in content when the prompt already closes reasoning,
|
||||
as with continue_final_message or a template that switches thinking
|
||||
off. Only configs with turn boundaries can tell a closed block in
|
||||
this turn from one in an earlier turn, so others are left as is.
|
||||
"""
|
||||
self._prompt_closes_reasoning = (
|
||||
self.parser_engine_config.initial_state is ParserState.REASONING
|
||||
and bool(self._turn_boundary_token_ids)
|
||||
and self.is_reasoning_end(list(prompt_token_ids))
|
||||
)
|
||||
|
||||
def adjust_initial_state_from_prompt(self, prompt_token_ids: Sequence[int]) -> None:
|
||||
"""See :meth:`ReasoningParser.adjust_initial_state_from_prompt`."""
|
||||
return
|
||||
@@ -242,8 +255,11 @@ class ParserEngine(Parser):
|
||||
return None
|
||||
|
||||
def _reset(self, initial_state: ParserState | None = None) -> None:
|
||||
self._engine.reset(initial_state=initial_state)
|
||||
self._reasoning_ended = not self._has_reasoning
|
||||
prompt_closed = initial_state is None and self._prompt_closes_reasoning
|
||||
self._engine.reset(
|
||||
initial_state=ParserState.CONTENT if prompt_closed else initial_state
|
||||
)
|
||||
self._reasoning_ended = not self._has_reasoning or prompt_closed
|
||||
self._tool_slots.clear()
|
||||
self._deferred_content = ""
|
||||
self._deferred_reasoning = ""
|
||||
|
||||
+64
-14
@@ -22,6 +22,7 @@ import regex as re
|
||||
from vllm.parser.engine.events import EventType
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
from vllm.parser.engine.parser_engine_config import (
|
||||
ArgValueExtender,
|
||||
ParserEngineConfig,
|
||||
ParserState,
|
||||
Transition,
|
||||
@@ -57,37 +58,85 @@ _PARTIAL_PARAM_RE = re.compile(
|
||||
)
|
||||
|
||||
|
||||
def _minimax_m2_arg_converter(raw_args: str, partial: bool) -> str:
|
||||
# What ends an open value in ``_PARAM_RE``: a closing tag or the next opener.
|
||||
_PARAM_VALUE_END_RE = re.compile(r"<\s*/\s*parameter\s*>|<\s*parameter\s+name\s*=")
|
||||
|
||||
|
||||
def _param_name(match: re.Match) -> str:
|
||||
return (
|
||||
match.group("dq_name")
|
||||
or match.group("sq_name")
|
||||
or match.group("bare_name")
|
||||
or ""
|
||||
).strip()
|
||||
|
||||
|
||||
def _minimax_m2_parse_args(
|
||||
raw_args: str, partial: bool
|
||||
) -> tuple[dict[str, object], re.Match | None]:
|
||||
"""Return the arguments and the match of the open trailing value, if any."""
|
||||
params: dict[str, object] = {}
|
||||
|
||||
for match in _PARAM_RE.finditer(raw_args):
|
||||
name = (
|
||||
match.group("dq_name")
|
||||
or match.group("sq_name")
|
||||
or match.group("bare_name")
|
||||
or ""
|
||||
).strip()
|
||||
name = _param_name(match)
|
||||
if not name:
|
||||
continue
|
||||
# Keep the value verbatim (like the glm47 converter): it is inline
|
||||
# between `>` and `</parameter>`, so surrounding whitespace is data.
|
||||
params[name] = match.group("value")
|
||||
|
||||
open_match = None
|
||||
if partial:
|
||||
remaining = _PARAM_RE.sub("", raw_args)
|
||||
match = _PARTIAL_PARAM_RE.search(remaining)
|
||||
if match:
|
||||
name = (
|
||||
match.group("dq_name")
|
||||
or match.group("sq_name")
|
||||
or match.group("bare_name")
|
||||
or ""
|
||||
).strip()
|
||||
name = _param_name(match)
|
||||
if name:
|
||||
# Verbatim, same as the complete-match loop above.
|
||||
params[name] = match.group("value")
|
||||
open_match = match
|
||||
|
||||
return json.dumps(params, ensure_ascii=False)
|
||||
return params, open_match
|
||||
|
||||
|
||||
def _minimax_m2_arg_converter(raw_args: str, partial: bool) -> str:
|
||||
return json.dumps(_minimax_m2_parse_args(raw_args, partial)[0], ensure_ascii=False)
|
||||
|
||||
|
||||
def _value_end_window(text: str) -> str:
|
||||
"""Return the suffix of *text* that may still grow into a value end."""
|
||||
start = text.rfind("<")
|
||||
if start == -1:
|
||||
return ""
|
||||
tail = text[start:]
|
||||
return tail if _PARAM_VALUE_END_RE.fullmatch(tail, partial=True) else ""
|
||||
|
||||
|
||||
def _minimax_m2_arg_value_extender(raw_args: str) -> ArgValueExtender | None:
|
||||
# The open value runs to the end of raw_args and is the last JSON value,
|
||||
# so a delta appends its JSON-escaped text unless it closes the value.
|
||||
params, match = _minimax_m2_parse_args(raw_args, partial=True)
|
||||
if match is None or next(reversed(params)) != _param_name(match):
|
||||
return None
|
||||
# Later text could still re-parse the opener: an earlier unmatched one, or
|
||||
# a bare name that starts a quote, may close into a different name.
|
||||
bare_name = match.group("bare_name")
|
||||
if _PARAM_VALUE_END_RE.search(match.string, 0, match.start()) or (
|
||||
bare_name is not None and bare_name[0] in "\"'"
|
||||
):
|
||||
return None
|
||||
|
||||
window = _value_end_window(match.group("value"))
|
||||
|
||||
def extend(delta: str) -> str | None:
|
||||
nonlocal window
|
||||
text = window + delta
|
||||
if _PARAM_VALUE_END_RE.search(text):
|
||||
return None
|
||||
window = _value_end_window(text)
|
||||
return json.dumps(delta, ensure_ascii=False)[1:-1]
|
||||
|
||||
return extend
|
||||
|
||||
|
||||
@functools.cache
|
||||
@@ -191,6 +240,7 @@ def minimax_m2_config() -> ParserEngineConfig:
|
||||
},
|
||||
},
|
||||
arg_converter=_minimax_m2_arg_converter,
|
||||
arg_value_extender=_minimax_m2_arg_value_extender,
|
||||
stream_arg_deltas=True,
|
||||
tool_args_json=False,
|
||||
validate_tool_names=True,
|
||||
|
||||
@@ -16,8 +16,6 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
HF_PARSER = "hf"
|
||||
|
||||
|
||||
class ParserManager:
|
||||
"""Provides a unified Parser by composing reasoning and tool parser adapters."""
|
||||
@@ -122,8 +120,10 @@ class ParserManager:
|
||||
tags (``--tool-strict-level``), or ``None`` for the tool
|
||||
parser's default level.
|
||||
tokenizer: Tokenizer the composed parser is validated against
|
||||
at startup (see :meth:`validate_parser_tokenizer`). ``None``
|
||||
skips the check (e.g. ``skip_tokenizer_init``).
|
||||
at startup (see :meth:`validate_parser_tokenizer`), including
|
||||
its `response_template` metadata when a response-template
|
||||
parser is selected. ``None`` skips the check (e.g.
|
||||
``skip_tokenizer_init``).
|
||||
|
||||
Returns:
|
||||
A Parser class, or None if neither parser is specified.
|
||||
@@ -184,39 +184,40 @@ class ParserManager:
|
||||
HarmonyParser.tool_strict_level = strict_level
|
||||
return HarmonyParser
|
||||
|
||||
if HF_PARSER in (reasoning_parser_name, tool_parser_name):
|
||||
if {reasoning_parser_name, tool_parser_name} - {
|
||||
HF_PARSER,
|
||||
None,
|
||||
"",
|
||||
}:
|
||||
raise TypeError(
|
||||
"The hf parser cannot be combined with other "
|
||||
"reasoning or tool call parsers"
|
||||
)
|
||||
from vllm.parser.response_template import (
|
||||
ResponseTemplateParser,
|
||||
validate_tokenizer_response_template,
|
||||
)
|
||||
from vllm.parser.response_template import (
|
||||
response_template_parser_cls,
|
||||
validate_tokenizer_response_template,
|
||||
)
|
||||
|
||||
template_parsers = {
|
||||
response_template_parser_cls(parser_cls)
|
||||
for parser_cls in (reasoning_parser_cls, tool_parser_cls)
|
||||
if parser_cls is not None
|
||||
}
|
||||
template_parser_cls = next(iter(template_parsers - {None}), None)
|
||||
if template_parser_cls is not None:
|
||||
if len(template_parsers) > 1:
|
||||
raise TypeError(
|
||||
f"--reasoning-parser {reasoning_parser_name} and "
|
||||
f"--tool-call-parser {tool_parser_name} cannot be combined"
|
||||
)
|
||||
if tokenizer is not None:
|
||||
validate_tokenizer_response_template(
|
||||
tokenizer,
|
||||
reasoning=reasoning_parser_cls is not None,
|
||||
tools=tool_parser_cls is not None,
|
||||
reasoning_parser=reasoning_parser_name,
|
||||
# Without auto tool choice, no tool parser is selected.
|
||||
tool_parser=tool_parser_name if tool_parser_cls else None,
|
||||
)
|
||||
|
||||
r_cls = reasoning_parser_cls
|
||||
t_cls = tool_parser_cls
|
||||
auto_tools = enable_auto_tools
|
||||
|
||||
class _ResponseTemplateParser(ResponseTemplateParser):
|
||||
reasoning_parser_cls = r_cls
|
||||
tool_parser_cls = t_cls
|
||||
tool_strict_level = strict_level
|
||||
_enable_auto_tools = auto_tools
|
||||
|
||||
return _ResponseTemplateParser
|
||||
return type(
|
||||
template_parser_cls.__name__,
|
||||
(template_parser_cls,),
|
||||
{
|
||||
"reasoning_parser_cls": reasoning_parser_cls,
|
||||
"tool_parser_cls": tool_parser_cls,
|
||||
"tool_strict_level": strict_level,
|
||||
"_enable_auto_tools": enable_auto_tools,
|
||||
},
|
||||
)
|
||||
|
||||
if reasoning_parser_name == "kimi_k3" or tool_parser_name == "kimi_k3":
|
||||
from vllm.parser.kimi_k3 import KimiK3Parser
|
||||
|
||||
@@ -7,6 +7,7 @@ from __future__ import annotations
|
||||
import copy
|
||||
import json
|
||||
from collections.abc import Callable, Iterable, Sequence
|
||||
from math import inf
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
from openai.types.responses import ToolChoiceCustom, ToolChoiceFunction
|
||||
@@ -28,6 +29,7 @@ from vllm.parser.chat_parsing.response_templates import (
|
||||
ResponseTemplate,
|
||||
load_response_template,
|
||||
)
|
||||
from vllm.parser.response_template_grammar import message_start_anchors
|
||||
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
|
||||
from vllm.renderers.chat_utils import make_tool_call_id
|
||||
from vllm.tool_parsers.abstract_tool_parser import ToolParser
|
||||
@@ -135,27 +137,38 @@ def validate_response_template_for_serving(
|
||||
def validate_tokenizer_response_template(
|
||||
tokenizer: Any,
|
||||
*,
|
||||
reasoning: bool,
|
||||
tools: bool,
|
||||
reasoning_parser: str | None,
|
||||
tool_parser: str | None,
|
||||
) -> None:
|
||||
"""Fail at startup if the tokenizer cannot serve the requested parsing."""
|
||||
"""Fail at startup if the tokenizer cannot serve the selected parsers,
|
||||
named as given on the command line."""
|
||||
template = resolve_response_template(tokenizer)
|
||||
if template is None:
|
||||
selected = " ".join(
|
||||
f"{flag} {name}"
|
||||
for flag, name in (
|
||||
("--reasoning-parser", reasoning_parser),
|
||||
("--tool-call-parser", tool_parser),
|
||||
)
|
||||
if name
|
||||
)
|
||||
raise TypeError(
|
||||
"The hf parser requires `response_template` metadata "
|
||||
f"{selected} requires `response_template` metadata "
|
||||
"in the tokenizer configuration"
|
||||
)
|
||||
try:
|
||||
fields = validate_response_template_for_serving(template).fields
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise TypeError(f"Invalid response_template metadata: {exc}") from exc
|
||||
if reasoning and THINKING_FIELD not in fields:
|
||||
if reasoning_parser and THINKING_FIELD not in fields:
|
||||
raise TypeError(
|
||||
"--reasoning-parser hf requires a response_template with a thinking field"
|
||||
f"--reasoning-parser {reasoning_parser} requires a response_template "
|
||||
"with a thinking field"
|
||||
)
|
||||
if tools and TOOL_FIELD not in fields:
|
||||
if tool_parser and TOOL_FIELD not in fields:
|
||||
raise TypeError(
|
||||
"--tool-call-parser hf requires a response_template with a tool_calls field"
|
||||
f"--tool-call-parser {tool_parser} requires a response_template "
|
||||
"with a tool_calls field"
|
||||
)
|
||||
|
||||
|
||||
@@ -292,7 +305,7 @@ def _serving_template(
|
||||
template = resolve_response_template(tokenizer, template)
|
||||
if template is None:
|
||||
raise ValueError(
|
||||
"The hf parser requires `response_template` "
|
||||
"Parsing with a response template requires `response_template` "
|
||||
"metadata in the tokenizer configuration"
|
||||
)
|
||||
validate_response_template_for_serving(template)
|
||||
@@ -319,12 +332,17 @@ def _to_tool_calls(value: Any) -> list[tuple[str, str]]:
|
||||
|
||||
|
||||
class ResponseTemplateReasoningParser(ReasoningParser):
|
||||
"""Reasoning end detection for `--reasoning-parser hf`.
|
||||
"""Reasoning end detection for a response-template parser.
|
||||
|
||||
`ResponseTemplateParser` parses the output; this parser tells it and the
|
||||
structured-output engine where reasoning ends.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def unified_parser_cls(cls) -> type[ResponseTemplateParser]:
|
||||
"""The unified parser that selecting this reasoning parser stands for."""
|
||||
return ResponseTemplateParser
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
@@ -400,8 +418,26 @@ class ResponseTemplateReasoningParser(ReasoningParser):
|
||||
|
||||
|
||||
class ResponseTemplateToolParser(ToolParser):
|
||||
"""Registers `--tool-call-parser hf`; `ResponseTemplateParser` extracts the
|
||||
tool calls."""
|
||||
"""Selects a response-template parser as the tool parser;
|
||||
`ResponseTemplateParser` extracts the tool calls."""
|
||||
|
||||
@classmethod
|
||||
def unified_parser_cls(cls) -> type[ResponseTemplateParser]:
|
||||
"""The unified parser that selecting this tool parser stands for."""
|
||||
return ResponseTemplateParser
|
||||
|
||||
|
||||
def response_template_parser_cls(
|
||||
split_parser_cls: type[ReasoningParser] | type[ToolParser],
|
||||
) -> type[ResponseTemplateParser] | None:
|
||||
"""The response-template parser that a registered reasoning or tool parser
|
||||
stands for, or `None` for any other parser."""
|
||||
if issubclass(
|
||||
split_parser_cls,
|
||||
(ResponseTemplateReasoningParser, ResponseTemplateToolParser),
|
||||
):
|
||||
return split_parser_cls.unified_parser_cls()
|
||||
return None
|
||||
|
||||
|
||||
class _DeltaBuilder:
|
||||
@@ -441,23 +477,50 @@ class _ResponseStream:
|
||||
self.include_reasoning = include_reasoning
|
||||
self.parse_tools = parse_tools
|
||||
self.new_tool_call_id = new_tool_call_id
|
||||
self.template = template
|
||||
self.parser = ResponseParser(template, prefix=prefix, tools=tools)
|
||||
self._prefix_end = len(self.parser.input_text)
|
||||
self._next_tool_index = 0
|
||||
self._finalized = False
|
||||
self._open_region: dict[str, Any] | None = None
|
||||
self._track(self.parser.initial_events)
|
||||
|
||||
def feed(self, text: str, *, finished: bool) -> DeltaMessage | None:
|
||||
if self._finalized:
|
||||
return None
|
||||
delta = _DeltaBuilder()
|
||||
self._route(self.parser.feed(text), delta)
|
||||
self._route(self._track(self.parser.feed(text)), delta)
|
||||
if finished:
|
||||
self._finalized = True
|
||||
if len(self.parser.input_text) > self._prefix_end:
|
||||
_, events = self.parser.finalize()
|
||||
self._route(events, delta)
|
||||
events = self.parser.feed(self._closer_rest())
|
||||
self._route(events + self.parser.finalize()[1], delta)
|
||||
return delta.build()
|
||||
|
||||
def _track(self, events: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
for event in events:
|
||||
if event["type"] == "region_open":
|
||||
self._open_region = event
|
||||
elif event["type"] in ("region_close", "region_malformed"):
|
||||
self._open_region = None
|
||||
return events
|
||||
|
||||
def _closer_rest(self) -> str:
|
||||
"""The rest of a closer the output stopped in after ending its region,
|
||||
when only the next message's start anchor is missing."""
|
||||
if self._open_region is None:
|
||||
return ""
|
||||
field = self.template.fields[self._open_region["field"]]
|
||||
text = self.parser.input_text[self._open_region["end"] :]
|
||||
for closer in field.close_literals or ():
|
||||
for anchor in message_start_anchors(self.template):
|
||||
if not anchor or not closer.endswith(anchor) or closer == anchor:
|
||||
continue
|
||||
for cut in range(len(closer) - 1, len(closer) - len(anchor) - 1, -1):
|
||||
if text.endswith(closer[:cut]):
|
||||
return closer[cut:]
|
||||
return ""
|
||||
|
||||
def _route(self, events: Sequence[dict[str, Any]], delta: _DeltaBuilder) -> None:
|
||||
for event in events:
|
||||
field = event.get("field")
|
||||
@@ -499,6 +562,54 @@ class _ResponseStream:
|
||||
self._next_tool_index += 1
|
||||
|
||||
|
||||
class _ReasoningTokenCounter:
|
||||
"""Counts the reasoning tokens of one growing generation, parsing each
|
||||
token once."""
|
||||
|
||||
def __init__(
|
||||
self, template: ResponseTemplate, prefix: str, tokenizer: TokenizerLike
|
||||
) -> None:
|
||||
self.tokenizer = tokenizer
|
||||
self.parser = ResponseParser(template, prefix=prefix)
|
||||
self.start = len(self.parser.input_text)
|
||||
self.token_ids: list[int] = []
|
||||
self.token_ends: list[int] = []
|
||||
# Input spans between the delimiters of thinking regions.
|
||||
self.spans: list[tuple[int, float]] = []
|
||||
self.open: int | None = None
|
||||
self._track(self.parser.initial_events)
|
||||
|
||||
def continues(self, token_ids: Sequence[int]) -> bool:
|
||||
return list(token_ids[: len(self.token_ids)]) == self.token_ids
|
||||
|
||||
def count(self, token_ids: Sequence[int]) -> int:
|
||||
new_ids = list(token_ids[len(self.token_ids) :])
|
||||
texts = [_decode(self.tokenizer, [token_id]) for token_id in new_ids]
|
||||
end = self.token_ends[-1] if self.token_ends else self.start
|
||||
for text in texts:
|
||||
end += len(text)
|
||||
self.token_ends.append(end)
|
||||
self.token_ids.extend(new_ids)
|
||||
self._track(self.parser.feed("".join(texts)))
|
||||
# Text held back as a possible closer counts until a closer arrives.
|
||||
spans = self.spans if self.open is None else [*self.spans, (self.open, inf)]
|
||||
count, start = 0, self.start
|
||||
for end in self.token_ends:
|
||||
count += any(begin < end and start < close for begin, close in spans)
|
||||
start = end
|
||||
return count
|
||||
|
||||
def _track(self, events: Iterable[dict[str, Any]]) -> None:
|
||||
for event in events:
|
||||
if event["field"] != THINKING_FIELD or event["type"] == "region_chunk":
|
||||
continue
|
||||
if event["type"] == "region_open":
|
||||
self.open = event["end"]
|
||||
elif self.open is not None:
|
||||
self.spans.append((self.open, event["start"]))
|
||||
self.open = None
|
||||
|
||||
|
||||
class ResponseTemplateParser(Parser):
|
||||
"""Parse reasoning, content, and tool calls with one response template."""
|
||||
|
||||
@@ -526,6 +637,7 @@ class ResponseTemplateParser(Parser):
|
||||
self._prompt_token_ids: list[int] | None = None
|
||||
self._prefix = ""
|
||||
self._stream: _ResponseStream | None = None
|
||||
self._reasoning_counter: _ReasoningTokenCounter | None = None
|
||||
|
||||
def set_prompt_token_ids(self, prompt_token_ids: Sequence[int]) -> None:
|
||||
prompt_token_ids = list(prompt_token_ids)
|
||||
@@ -534,6 +646,7 @@ class ResponseTemplateParser(Parser):
|
||||
self._prompt_token_ids = prompt_token_ids
|
||||
self._prefix = _decode(self.model_tokenizer, prompt_token_ids)
|
||||
self._stream = None
|
||||
self._reasoning_counter = None
|
||||
|
||||
def adjust_request(
|
||||
self,
|
||||
@@ -547,6 +660,14 @@ class ResponseTemplateParser(Parser):
|
||||
request.skip_special_tokens = False
|
||||
if hasattr(request, "spaces_between_special_tokens"):
|
||||
request.spaces_between_special_tokens = False
|
||||
return self._constrain_request(request)
|
||||
|
||||
def _constrain_request(
|
||||
self,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> ChatCompletionRequest | ResponsesRequest:
|
||||
"""Constrain `request` to the tool guarantees it asks for, or reject
|
||||
those this parser cannot enforce."""
|
||||
if (
|
||||
self._tool_parser is not None
|
||||
and request.tools
|
||||
@@ -567,6 +688,21 @@ class ResponseTemplateParser(Parser):
|
||||
or self._reasoning_parser.is_reasoning_end(input_ids)
|
||||
)
|
||||
|
||||
def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
|
||||
"""Count the generated tokens whose text the template parses as
|
||||
reasoning, continuing reasoning left open by the prompt."""
|
||||
if (
|
||||
self._reasoning_parser is None
|
||||
or THINKING_FIELD not in self.response_template.fields
|
||||
):
|
||||
return 0
|
||||
counter = self._reasoning_counter
|
||||
if counter is None or not counter.continues(token_ids):
|
||||
counter = self._reasoning_counter = _ReasoningTokenCounter(
|
||||
self.response_template, self._prefix, self.model_tokenizer
|
||||
)
|
||||
return counter.count(token_ids)
|
||||
|
||||
def parse(
|
||||
self,
|
||||
model_output: str,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+16
-8
@@ -289,10 +289,9 @@ class CpuPlatform(Platform):
|
||||
cache_config.block_size,
|
||||
)
|
||||
cache_config.block_size = 256
|
||||
elif not cache_config.user_specified_block_size:
|
||||
cache_config.block_size = 128
|
||||
|
||||
if not reference_cpu_mla_enabled and cache_config.block_size % 32 != 0:
|
||||
elif (
|
||||
cache_config.user_specified_block_size and cache_config.block_size % 32 != 0
|
||||
):
|
||||
logger.warning(
|
||||
"CPU backend prefers block_size is multiples of 32, "
|
||||
"otherwise the performance is not optimized."
|
||||
@@ -532,15 +531,24 @@ class CpuPlatform(Platform):
|
||||
@classmethod
|
||||
def update_block_size_for_backend(cls, vllm_config: "VllmConfig") -> None:
|
||||
model_config = vllm_config.model_config
|
||||
if model_config is None or not model_config.is_hybrid:
|
||||
if model_config is None:
|
||||
return
|
||||
|
||||
# reconcile attention and mamba page sizes
|
||||
backend_classes = cls._find_non_ssm_backends(vllm_config)
|
||||
if not backend_classes:
|
||||
return
|
||||
|
||||
pre_block_size = vllm_config.cache_config.block_size
|
||||
# Not in check_and_update_config(), which reruns with every
|
||||
# VllmConfig.__post_init__ and would undo the hybrid alignment below.
|
||||
cache_config = vllm_config.cache_config
|
||||
if not cache_config.user_specified_block_size:
|
||||
cache_config.block_size = cls._preferred_block_size_for_backends(
|
||||
backend_classes, 128, vllm_config
|
||||
)
|
||||
if not model_config.is_hybrid:
|
||||
return
|
||||
|
||||
# reconcile attention and mamba page sizes
|
||||
pre_block_size = cache_config.block_size
|
||||
cls._align_hybrid_block_size(vllm_config, backend_classes[0])
|
||||
cls._check_aligned_block_size(vllm_config, backend_classes, pre_block_size)
|
||||
|
||||
|
||||
@@ -1467,6 +1467,28 @@ class Platform:
|
||||
"Unknown vLLM environment variable detected: %s", env
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_kv_cache_config_builder_cls(cls, vllm_config: "VllmConfig") -> str:
|
||||
"""Resolve the fully-qualified class path of the platform's KV cache
|
||||
config builder.
|
||||
|
||||
.. note::
|
||||
Experimental, like the ``KVCacheConfigBuilder`` interface it selects.
|
||||
|
||||
The builder supplies grouping (``get_kv_cache_groups``) and tensor
|
||||
placement (``get_kv_cache_config_from_groups``). Vendors may override
|
||||
this to return a platform-specific builder. The default builder honors a
|
||||
model implementing
|
||||
:class:`~vllm.model_executor.models.interfaces.SupportsKVCacheGrouping`;
|
||||
a builder that overrides ``get_kv_cache_groups`` owns grouping and
|
||||
bypasses that model hook unless it calls ``super()``.
|
||||
|
||||
Returns:
|
||||
Fully-qualified class path of a ``KVCacheConfigBuilder`` subclass.
|
||||
|
||||
"""
|
||||
return "vllm.v1.core.kv_cache_planning.DefaultKVCacheConfigBuilder"
|
||||
|
||||
|
||||
class UnspecifiedPlatform(Platform):
|
||||
_enum = PlatformEnum.UNSPECIFIED
|
||||
|
||||
@@ -84,6 +84,10 @@ _REASONING_PARSERS_TO_REGISTER = {
|
||||
"response_template_reasoning_parser",
|
||||
"ResponseTemplateReasoningParser",
|
||||
),
|
||||
"beam": (
|
||||
"beam_reasoning_parser",
|
||||
"BeamReasoningParser",
|
||||
),
|
||||
"holo2": (
|
||||
"deepseek_v3_reasoning_parser",
|
||||
"DeepSeekV3ReasoningWithThinkingParser",
|
||||
|
||||
@@ -192,6 +192,15 @@ class ReasoningParser:
|
||||
"""Adjust request parameters; override in subclasses as needed."""
|
||||
return request
|
||||
|
||||
def set_prompt_token_ids(self, prompt_token_ids: Sequence[int]) -> None:
|
||||
"""Provide the rendered prompt before the output is parsed.
|
||||
|
||||
Parsers that can tell whether the prompt already closes reasoning
|
||||
(e.g. continue_final_message) use it to classify the output.
|
||||
Default is a no-op; override in subclasses as needed.
|
||||
"""
|
||||
return
|
||||
|
||||
def adjust_initial_state_from_prompt(self, prompt_token_ids: Sequence[int]) -> None:
|
||||
"""Hook called once at the start of streaming with the prompt tokens.
|
||||
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.parser.beam import BeamReasoningParser
|
||||
|
||||
__all__ = ["BeamReasoningParser"]
|
||||
+10
-2
@@ -687,13 +687,17 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
prompt = self._tokenize_prompt(prompt, params)
|
||||
prompt = self._apply_prompt_char_offset(prompt, prompt_char_offset)
|
||||
|
||||
prompt = params.apply_post_tokenization(self.tokenizer, prompt) # type: ignore[arg-type]
|
||||
|
||||
# Detokenize after truncation so the text matches the tokens sent
|
||||
# to the engine.
|
||||
if params.needs_detokenization and "prompt" not in prompt:
|
||||
if "prompt_token_ids" not in prompt:
|
||||
raise RuntimeError("Cannot run detokenization on embeddings")
|
||||
|
||||
prompt = self._detokenize_prompt(prompt) # type: ignore[arg-type]
|
||||
|
||||
return params.apply_post_tokenization(self.tokenizer, prompt) # type: ignore[arg-type]
|
||||
return prompt
|
||||
|
||||
@overload
|
||||
async def _tokenize_singleton_prompt_async(
|
||||
@@ -727,13 +731,17 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
prompt = await self._tokenize_prompt_async(prompt, params)
|
||||
prompt = self._apply_prompt_char_offset(prompt, prompt_char_offset)
|
||||
|
||||
prompt = params.apply_post_tokenization(self.tokenizer, prompt) # type: ignore[arg-type]
|
||||
|
||||
# Detokenize after truncation so the text matches the tokens sent
|
||||
# to the engine.
|
||||
if params.needs_detokenization and "prompt" not in prompt:
|
||||
if "prompt_token_ids" not in prompt:
|
||||
raise RuntimeError("Cannot run detokenization on embeddings")
|
||||
|
||||
prompt = await self._detokenize_prompt_async(prompt) # type: ignore[arg-type]
|
||||
|
||||
return params.apply_post_tokenization(self.tokenizer, prompt) # type: ignore[arg-type]
|
||||
return prompt
|
||||
|
||||
def _tokenize_enc_dec_prompt(
|
||||
self,
|
||||
|
||||
@@ -47,14 +47,16 @@ from vllm.entrypoints.serve.utils.tool_calls_utils import (
|
||||
from vllm.logger import init_logger
|
||||
from vllm.parser import Parser, ParserManager
|
||||
from vllm.renderers import BaseRenderer
|
||||
from vllm.renderers.chat_utils import ChatTemplateContentFormatOption
|
||||
from vllm.renderers.chat_utils import (
|
||||
ChatTemplateContentFormatOption,
|
||||
make_tool_call_id,
|
||||
)
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tokenizers.detokenizer_utils import (
|
||||
convert_prompt_ids_to_tokens,
|
||||
detokenize_incrementally,
|
||||
get_leading_space_marker,
|
||||
)
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.utils.async_utils import make_async
|
||||
from vllm.v1.engine.detokenizer import uses_fast_detokenizer
|
||||
|
||||
@@ -202,13 +204,21 @@ class OnlineDerenderer:
|
||||
if has_parser and chat_request is not None
|
||||
else {}
|
||||
)
|
||||
|
||||
# Like the coupled chat endpoint, hide per-token metadata (logprobs,
|
||||
# token ids) when a parser is configured and reasoning is hidden:
|
||||
# either would otherwise leak the reasoning text.
|
||||
hide_metadata = (
|
||||
has_parser
|
||||
and chat_request is not None
|
||||
and not chat_request.include_reasoning
|
||||
)
|
||||
return_token_ids = (
|
||||
chat_request is not None
|
||||
and chat_request.return_token_ids
|
||||
and (not has_parser or chat_request.include_reasoning)
|
||||
and not hide_metadata
|
||||
)
|
||||
top_limit = _chat_top_logprobs_limit(chat_request)
|
||||
|
||||
for choice in generate_response.choices:
|
||||
if not choice.token_ids:
|
||||
raise ValueError(f"choice {choice.index} has empty or null token_ids")
|
||||
@@ -219,7 +229,7 @@ class OnlineDerenderer:
|
||||
tokenizer,
|
||||
top_limit=top_limit,
|
||||
)
|
||||
if choice.logprobs is not None
|
||||
if choice.logprobs is not None and not hide_metadata
|
||||
else None
|
||||
)
|
||||
|
||||
@@ -326,7 +336,10 @@ class OnlineDerenderer:
|
||||
reasoning = None
|
||||
|
||||
tc_items = (
|
||||
[ToolCall(id=random_uuid(), function=tc) for tc in tool_calls]
|
||||
[
|
||||
ToolCall(id=tc.id or make_tool_call_id(), function=tc)
|
||||
for tc in tool_calls
|
||||
]
|
||||
if tool_calls
|
||||
else []
|
||||
)
|
||||
@@ -508,13 +521,6 @@ class OnlineDerenderer:
|
||||
spaces_between_special_tokens=spaces_between,
|
||||
)
|
||||
|
||||
# NOTE: parser-configured servers dispatch to
|
||||
# _derender_chat_stream_parsed above and never reach this plain
|
||||
# path. That parsed path does not resolve logprobs yet; when it
|
||||
# does, mirror the generate chat streaming path, which suppresses
|
||||
# logprobs entirely when a parser is configured and reasoning is
|
||||
# hidden, because decoded logprob token text would leak hidden
|
||||
# reasoning.
|
||||
resolved_logprobs = None
|
||||
if choice.logprobs is not None:
|
||||
resolved_logprobs = _resolve_logprobs(
|
||||
@@ -595,8 +601,17 @@ class OnlineDerenderer:
|
||||
Text comes from a fresh incremental detokenizer with special tokens
|
||||
preserved (``skip_special_tokens=False``), seeded from the prompt
|
||||
tail on every call since replay starts from scratch.
|
||||
|
||||
Logprobs need no replay: a generate chunk's ``logprobs`` cover exactly
|
||||
that chunk's tokens, so they are resolved once and attached to
|
||||
whatever delta the parser emits for the chunk, as the coupled chat
|
||||
endpoint does. Only the byte-fallback context crosses chunks, via
|
||||
``state.logprob_context_token_ids``. With ``include_reasoning=False``
|
||||
logprobs are dropped on every chunk, again like the coupled endpoint.
|
||||
"""
|
||||
tokenizer = self.renderer.get_tokenizer()
|
||||
hide_metadata = not chat_request.include_reasoning
|
||||
top_limit = _chat_top_logprobs_limit(chat_request)
|
||||
|
||||
parser = parser_cls(
|
||||
tokenizer,
|
||||
@@ -651,10 +666,9 @@ class OnlineDerenderer:
|
||||
role_sent = state.role_sent
|
||||
tools_streamed = state.tools_streamed
|
||||
last_tool_call_ids = list(state.last_tool_call_ids)
|
||||
logprob_context = list(state.logprob_context_token_ids)
|
||||
|
||||
return_token_ids = (
|
||||
chat_request.return_token_ids and chat_request.include_reasoning
|
||||
)
|
||||
return_token_ids = chat_request.return_token_ids and not hide_metadata
|
||||
|
||||
# At most one choice: the caller (derender_chat_stream) already
|
||||
# rejects >1 before dispatching here. role_sent/tools_streamed/
|
||||
@@ -707,6 +721,18 @@ class OnlineDerenderer:
|
||||
output_token_ids.extend(delta_tids)
|
||||
output_chunk_lens.append(len(delta_tids))
|
||||
|
||||
resolved_logprobs = None
|
||||
if choice.logprobs is not None and not hide_metadata:
|
||||
resolved_logprobs = _resolve_logprobs(
|
||||
choice.logprobs,
|
||||
tokenizer,
|
||||
initial_context_token_ids=state.logprob_context_token_ids,
|
||||
top_limit=top_limit,
|
||||
)
|
||||
logprob_context = _logprob_context_tail(
|
||||
state.logprob_context_token_ids, delta_tids
|
||||
)
|
||||
|
||||
if delta_message is None:
|
||||
delta_message = DeltaMessage()
|
||||
|
||||
@@ -735,6 +761,7 @@ class OnlineDerenderer:
|
||||
stream_choice = ChatCompletionResponseStreamChoice(
|
||||
index=choice.index,
|
||||
delta=delta_message,
|
||||
logprobs=resolved_logprobs,
|
||||
finish_reason=finish_reason,
|
||||
stop_reason=choice.stop_reason,
|
||||
token_ids=choice.token_ids if return_token_ids else None,
|
||||
@@ -750,6 +777,7 @@ class OnlineDerenderer:
|
||||
"role_sent": role_sent,
|
||||
"tools_streamed": tools_streamed,
|
||||
"last_tool_call_ids": last_tool_call_ids,
|
||||
"logprob_context_token_ids": logprob_context,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -60,6 +60,7 @@ from vllm.renderers.chat_utils import (
|
||||
has_non_text_content,
|
||||
)
|
||||
from vllm.renderers.inputs.preprocess import (
|
||||
extract_prompt_components,
|
||||
parse_model_prompt,
|
||||
prompt_to_seq,
|
||||
)
|
||||
@@ -844,13 +845,17 @@ class OnlineRenderer:
|
||||
f"but got {type(request).__name__}"
|
||||
)
|
||||
raise NotImplementedError(msg)
|
||||
request = parser(
|
||||
request_parser = parser(
|
||||
tokenizer,
|
||||
request.tools,
|
||||
model_config=self.model_config,
|
||||
chat_template_kwargs=chat_params.chat_template_kwargs,
|
||||
).adjust_request(
|
||||
request=request,
|
||||
)
|
||||
prompt_token_ids = extract_prompt_components(
|
||||
self.model_config, engine_input
|
||||
).token_ids
|
||||
if prompt_token_ids is not None:
|
||||
request_parser.set_prompt_token_ids(prompt_token_ids)
|
||||
request = request_parser.adjust_request(request=request)
|
||||
|
||||
return conversation, [engine_input]
|
||||
|
||||
@@ -90,6 +90,10 @@ _TOOL_PARSERS_TO_REGISTER = {
|
||||
"response_template_tool_parser",
|
||||
"ResponseTemplateToolParser",
|
||||
),
|
||||
"beam": (
|
||||
"beam_tool_parser",
|
||||
"BeamToolParser",
|
||||
),
|
||||
"poolside_v1": (
|
||||
"poolside_v1_tool_parser",
|
||||
"PoolsideV1ToolParser",
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.parser.beam import BeamToolParser
|
||||
|
||||
__all__ = ["BeamToolParser"]
|
||||
@@ -39,9 +39,6 @@ __all__ = [
|
||||
"OvisProcessor",
|
||||
"Ovis2_5Processor",
|
||||
"Step3VLProcessor",
|
||||
"InklingProcessor",
|
||||
"InklingImageProcessor",
|
||||
"InklingAudioFeatureExtractor",
|
||||
]
|
||||
|
||||
_CLASS_TO_MODULE: dict[str, str] = {
|
||||
@@ -74,9 +71,6 @@ _CLASS_TO_MODULE: dict[str, str] = {
|
||||
"OvisProcessor": "vllm.transformers_utils.processors.ovis",
|
||||
"Ovis2_5Processor": "vllm.transformers_utils.processors.ovis2_5",
|
||||
"Step3VLProcessor": "vllm.transformers_utils.processors.step3_vl",
|
||||
"InklingProcessor": "vllm.transformers_utils.processors.inkling",
|
||||
"InklingImageProcessor": "vllm.transformers_utils.processors.inkling",
|
||||
"InklingAudioFeatureExtractor": "vllm.transformers_utils.processors.inkling",
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -1,504 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Vendored HuggingFace-convention processors for the Inkling Titan model.
|
||||
|
||||
Implements the image processor (numba patchifier), the audio feature extractor
|
||||
(STFT/dMel path), the composite processor, and the MM token-id constants.
|
||||
|
||||
Besides raw bytes / file paths, the extractors also accept the dummy inputs
|
||||
vLLM generates during profiling (PIL images / numpy audio arrays).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import math
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from numba import njit
|
||||
from transformers.feature_extraction_utils import (
|
||||
BatchFeature,
|
||||
FeatureExtractionMixin,
|
||||
)
|
||||
from transformers.image_processing_utils import BaseImageProcessor
|
||||
from transformers.image_utils import ImageInput
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MM token-id constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Block-start marker token ids. These are real tokens the model was trained on
|
||||
# (``<|content_image|>`` / ``<|content_audio_input|>``) that mark the start of an
|
||||
# image/audio embedding block; they are kept verbatim in ``input_ids``.
|
||||
IMAGE_MARKER_ID = 200005 # <|content_image|>
|
||||
AUDIO_MARKER_ID = 200020 # <|content_audio_input|>
|
||||
|
||||
# Per-patch / per-frame placeholder ids marking where tower embeddings are
|
||||
# scattered in. These are unused slots in the padded vocabulary and the
|
||||
# corresponding positions are always overwritten by tower embeddings.
|
||||
IMAGE_TOKEN_ID = 200054 # <|unused_200054|>
|
||||
AUDIO_TOKEN_ID = 200053 # <|unused_200053|>
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Image processing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
IMAGE_MEAN = np.array([0.48145466, 0.4578275, 0.40821073], dtype=np.float32)
|
||||
IMAGE_STD = np.array([0.26862954, 0.2613026, 0.2757771], dtype=np.float32)
|
||||
PAD_RAW_VALUE = np.float32(-1.0 / 255.0)
|
||||
PAD_NORM = (np.full((3,), PAD_RAW_VALUE, dtype=np.float32) - IMAGE_MEAN) / IMAGE_STD
|
||||
|
||||
|
||||
def _validate_image_rescale(
|
||||
rescale_image_frac: float | None,
|
||||
rescale_image_max_upscaled_long_edge: int | None,
|
||||
) -> None:
|
||||
if rescale_image_frac is not None and (
|
||||
not math.isfinite(rescale_image_frac) or rescale_image_frac <= 0
|
||||
):
|
||||
raise ValueError(
|
||||
"rescale_image_frac must be positive and finite or None, "
|
||||
f"got {rescale_image_frac}"
|
||||
)
|
||||
if rescale_image_max_upscaled_long_edge is None:
|
||||
return
|
||||
if rescale_image_max_upscaled_long_edge <= 0:
|
||||
raise ValueError(
|
||||
"rescale_image_max_upscaled_long_edge must be positive or None, "
|
||||
f"got {rescale_image_max_upscaled_long_edge}"
|
||||
)
|
||||
if rescale_image_frac is None or rescale_image_frac <= 1.0:
|
||||
raise ValueError(
|
||||
"rescale_image_max_upscaled_long_edge requires rescale_image_frac > 1, "
|
||||
f"got {rescale_image_frac}"
|
||||
)
|
||||
|
||||
|
||||
def _scaled_image_dimensions(
|
||||
width: int,
|
||||
height: int,
|
||||
rescale_image_frac: float | None,
|
||||
rescale_image_max_upscaled_long_edge: int | None,
|
||||
) -> tuple[int, int]:
|
||||
"""Return the long-edge-scaled ``(width, height)``."""
|
||||
if rescale_image_frac is None:
|
||||
return width, height
|
||||
|
||||
long_edge = max(width, height)
|
||||
if long_edge == 0:
|
||||
return width, height
|
||||
|
||||
target_long_edge = float(long_edge) * rescale_image_frac
|
||||
if rescale_image_max_upscaled_long_edge is not None:
|
||||
effective_cap = max(rescale_image_max_upscaled_long_edge, long_edge)
|
||||
target_long_edge = min(target_long_edge, float(effective_cap))
|
||||
|
||||
ratio = target_long_edge / float(long_edge)
|
||||
if ratio == 1.0:
|
||||
return width, height
|
||||
|
||||
def scale(value: int) -> int:
|
||||
return max(1, math.floor(float(value) * ratio + 0.5))
|
||||
|
||||
return scale(width), scale(height)
|
||||
|
||||
|
||||
def _load_image_bytes(image) -> bytes:
|
||||
"""Encode a PIL image as raw PNG bytes for preprocessing.
|
||||
|
||||
The HF processor is always handed ``PIL.Image`` instances by vLLM, so no
|
||||
other input types need to be supported here.
|
||||
"""
|
||||
if image.mode != "RGB":
|
||||
image = image.convert("RGB")
|
||||
buf = io.BytesIO()
|
||||
image.save(buf, format="PNG")
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
@njit(cache=True)
|
||||
def _fill_patches_numba(
|
||||
arr: np.ndarray,
|
||||
patch_size: int,
|
||||
patches: np.ndarray,
|
||||
mean: np.ndarray,
|
||||
std: np.ndarray,
|
||||
pad_norm: np.ndarray,
|
||||
) -> None:
|
||||
h = arr.shape[0]
|
||||
w = arr.shape[1]
|
||||
nph = (h + patch_size - 1) // patch_size
|
||||
npw = w // patch_size + 1
|
||||
inv255 = np.float32(1.0 / 255.0)
|
||||
|
||||
for k in range(nph * npw):
|
||||
i = k // npw
|
||||
j = k - i * npw
|
||||
y_base = i * patch_size
|
||||
x_base = j * patch_size
|
||||
|
||||
for y in range(patch_size):
|
||||
iy = y_base + y
|
||||
for x in range(patch_size):
|
||||
ix = x_base + x
|
||||
if iy < h and ix < w:
|
||||
for c in range(3):
|
||||
raw = np.float32(arr[iy, ix, c]) * inv255
|
||||
patches[k, y, x, c] = (raw - mean[c]) / std[c]
|
||||
else:
|
||||
for c in range(3):
|
||||
patches[k, y, x, c] = pad_norm[c]
|
||||
|
||||
|
||||
def _encode_image_bytes(
|
||||
image_bytes: bytes,
|
||||
*,
|
||||
patch_size: int,
|
||||
rescale_image_frac: float | None,
|
||||
rescale_image_max_upscaled_long_edge: int | None,
|
||||
) -> torch.Tensor:
|
||||
if patch_size <= 0:
|
||||
raise ValueError("patch_size must be greater than zero")
|
||||
_validate_image_rescale(
|
||||
rescale_image_frac,
|
||||
rescale_image_max_upscaled_long_edge,
|
||||
)
|
||||
|
||||
from PIL import Image
|
||||
|
||||
image = Image.open(io.BytesIO(image_bytes)).convert("RGB")
|
||||
scaled_size = _scaled_image_dimensions(
|
||||
image.width,
|
||||
image.height,
|
||||
rescale_image_frac=rescale_image_frac,
|
||||
rescale_image_max_upscaled_long_edge=rescale_image_max_upscaled_long_edge,
|
||||
)
|
||||
if scaled_size != image.size:
|
||||
image = image.resize(scaled_size, resample=Image.Resampling.LANCZOS)
|
||||
arr = np.array(image, dtype=np.uint8, copy=True)
|
||||
height, width, _ = arr.shape
|
||||
|
||||
nph = (height + patch_size - 1) // patch_size
|
||||
npw = width // patch_size + 1
|
||||
num_patches = nph * npw
|
||||
|
||||
patches = np.empty((num_patches, patch_size, patch_size, 3), dtype=np.float32)
|
||||
_fill_patches_numba(arr, patch_size, patches, IMAGE_MEAN, IMAGE_STD, PAD_NORM)
|
||||
|
||||
return (
|
||||
torch.from_numpy(patches)
|
||||
.to(torch.bfloat16)
|
||||
.view(num_patches, 1, patch_size, patch_size, 3)
|
||||
.expand(num_patches, 2, patch_size, patch_size, 3)
|
||||
)
|
||||
|
||||
|
||||
class InklingImageProcessor(BaseImageProcessor):
|
||||
r"""Turn raw images into ``vision_patches_bthwc`` for Inkling hMLP.
|
||||
|
||||
``rescale_image_frac`` scales the long edge while preserving aspect ratio.
|
||||
``rescale_image_max_upscaled_long_edge`` optionally caps only upscaling and
|
||||
therefore requires a scale factor greater than one. The defaults, ``2.0`` and
|
||||
``2048``, grow images toward a 2048-pixel long edge by at most 2x, while leaving
|
||||
images already at or above 2048 unchanged.
|
||||
"""
|
||||
|
||||
model_input_names = ["vision_patches_bthwc"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
patch_size: int = 40,
|
||||
rescale_image_frac: float | None = 2.0,
|
||||
rescale_image_max_upscaled_long_edge: int | None = 2048,
|
||||
**kwargs,
|
||||
):
|
||||
if patch_size <= 0:
|
||||
raise ValueError("patch_size must be greater than zero")
|
||||
_validate_image_rescale(
|
||||
rescale_image_frac,
|
||||
rescale_image_max_upscaled_long_edge,
|
||||
)
|
||||
super().__init__(**kwargs)
|
||||
self.patch_size = patch_size
|
||||
self.rescale_image_frac = rescale_image_frac
|
||||
self.rescale_image_max_upscaled_long_edge = rescale_image_max_upscaled_long_edge
|
||||
|
||||
def _encode_one(self, image) -> torch.Tensor:
|
||||
return _encode_image_bytes(
|
||||
_load_image_bytes(image),
|
||||
patch_size=self.patch_size,
|
||||
rescale_image_frac=self.rescale_image_frac,
|
||||
rescale_image_max_upscaled_long_edge=self.rescale_image_max_upscaled_long_edge,
|
||||
)
|
||||
|
||||
def preprocess(
|
||||
self,
|
||||
images: ImageInput | list,
|
||||
return_tensors: str | None = "pt",
|
||||
**kwargs,
|
||||
) -> BatchFeature:
|
||||
del return_tensors, kwargs
|
||||
if not isinstance(images, (list, tuple)):
|
||||
images = [images]
|
||||
|
||||
per_image_patches: list[torch.Tensor] = []
|
||||
num_patches: list[int] = []
|
||||
num_tokens: list[int] = []
|
||||
for img in images:
|
||||
vp = self._encode_one(img)
|
||||
n_patches = int(vp.shape[0])
|
||||
per_image_patches.append(vp)
|
||||
num_patches.append(n_patches)
|
||||
num_tokens.append(n_patches)
|
||||
|
||||
if len(per_image_patches) == 1:
|
||||
vision_patches_bthwc = per_image_patches[0]
|
||||
elif per_image_patches:
|
||||
vision_patches_bthwc = torch.cat(per_image_patches, dim=0)
|
||||
else:
|
||||
vision_patches_bthwc = torch.empty(0)
|
||||
|
||||
data = {
|
||||
"vision_patches_bthwc": vision_patches_bthwc,
|
||||
"num_patches": num_patches,
|
||||
"num_tokens": num_tokens,
|
||||
}
|
||||
return BatchFeature(data=data, tensor_type=None)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Audio feature extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class InklingAudioEncoderParams:
|
||||
"""Audio preprocessing parameters used to convert raw audio into dMel bins."""
|
||||
|
||||
sample_rate: int = 16_000
|
||||
window_size_multiplier: float = 2.0
|
||||
n_fft: int | None = None
|
||||
n_mels: int = 80
|
||||
num_dmel_bins: int = 16
|
||||
dmel_min_value: float = -7.0
|
||||
dmel_max_value: float = 2.0
|
||||
audio_token_duration_s: float = 0.05
|
||||
|
||||
|
||||
def _to_exact_int(value: float, name: str, tolerance: float = 1e-6) -> int:
|
||||
rounded = round(value)
|
||||
if abs(value - rounded) > tolerance:
|
||||
raise ValueError(f"{name} must resolve to an integer sample count, got {value}")
|
||||
return int(rounded)
|
||||
|
||||
|
||||
def _hz_to_mel(frequencies: np.ndarray) -> np.ndarray:
|
||||
"""Slaney mel scale, matching the librosa/torchaudio convention."""
|
||||
frequencies = np.asarray(frequencies, dtype=np.float64)
|
||||
f_sp = 200.0 / 3.0
|
||||
min_log_hz = 1000.0
|
||||
min_log_mel = min_log_hz / f_sp
|
||||
logstep = np.log(6.4) / 27.0
|
||||
linear = frequencies / f_sp
|
||||
log = (
|
||||
min_log_mel + np.log(np.maximum(frequencies, min_log_hz) / min_log_hz) / logstep
|
||||
)
|
||||
return np.where(frequencies >= min_log_hz, log, linear)
|
||||
|
||||
|
||||
def _mel_to_hz(mels: np.ndarray) -> np.ndarray:
|
||||
mels = np.asarray(mels, dtype=np.float64)
|
||||
f_sp = 200.0 / 3.0
|
||||
min_log_hz = 1000.0
|
||||
min_log_mel = min_log_hz / f_sp
|
||||
logstep = np.log(6.4) / 27.0
|
||||
linear = mels * f_sp
|
||||
log = min_log_hz * np.exp(logstep * (mels - min_log_mel))
|
||||
return np.where(mels >= min_log_mel, log, linear)
|
||||
|
||||
|
||||
_MEL_BASIS_CACHE: dict[tuple[int, int, int], torch.Tensor] = {}
|
||||
|
||||
|
||||
def _mel_basis(sample_rate: int, n_fft: int, n_mels: int) -> torch.Tensor:
|
||||
key = (sample_rate, n_fft, n_mels)
|
||||
cached = _MEL_BASIS_CACHE.get(key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
fft_bins = n_fft // 2 + 1
|
||||
fft_freqs = np.arange(fft_bins, dtype=np.float64) * sample_rate / n_fft
|
||||
mel_edges = _mel_to_hz(
|
||||
np.linspace(
|
||||
_hz_to_mel(np.array([0.0]))[0],
|
||||
_hz_to_mel(np.array([sample_rate / 2.0]))[0],
|
||||
n_mels + 2,
|
||||
dtype=np.float64,
|
||||
)
|
||||
)
|
||||
mel_widths = np.diff(mel_edges)
|
||||
lower = (fft_freqs[None, :] - mel_edges[:-2, None]) / mel_widths[:-1, None]
|
||||
upper = (mel_edges[2:, None] - fft_freqs[None, :]) / mel_widths[1:, None]
|
||||
weights = np.maximum(0.0, np.minimum(lower, upper))
|
||||
|
||||
# Slaney area normalization.
|
||||
weights *= (2.0 / (mel_edges[2:] - mel_edges[:-2]))[:, None]
|
||||
basis = torch.from_numpy(weights.astype(np.float32, copy=False)).contiguous()
|
||||
_MEL_BASIS_CACHE[key] = basis
|
||||
return basis
|
||||
|
||||
|
||||
def _dmel_bins(audio: torch.Tensor, params: InklingAudioEncoderParams) -> torch.Tensor:
|
||||
hop_length = _to_exact_int(
|
||||
params.audio_token_duration_s * params.sample_rate,
|
||||
"audio_token_duration_s * sample_rate",
|
||||
)
|
||||
window_size = _to_exact_int(
|
||||
params.audio_token_duration_s
|
||||
* params.window_size_multiplier
|
||||
* params.sample_rate,
|
||||
"audio_token_duration_s * window_size_multiplier * sample_rate",
|
||||
)
|
||||
n_fft = params.n_fft or window_size
|
||||
if hop_length <= 0 or window_size <= 0 or n_fft <= 0:
|
||||
raise ValueError("audio hop length, window size, and n_fft must be positive")
|
||||
if audio.numel() == 0:
|
||||
return torch.empty((0, params.n_mels), dtype=torch.int32)
|
||||
|
||||
right_pad = math.ceil(audio.numel() / hop_length) * hop_length - audio.numel()
|
||||
left_pad = max(n_fft - hop_length, 0)
|
||||
audio = F.pad(audio, (left_pad, right_pad))
|
||||
|
||||
window = torch.hann_window(window_size, periodic=True, dtype=torch.float32)
|
||||
spec = torch.stft(
|
||||
audio.unsqueeze(0),
|
||||
n_fft=n_fft,
|
||||
hop_length=hop_length,
|
||||
win_length=window_size,
|
||||
window=window,
|
||||
center=False,
|
||||
normalized=False,
|
||||
onesided=True,
|
||||
return_complex=True,
|
||||
)
|
||||
spec_ri = torch.view_as_real(spec)
|
||||
magnitude = (
|
||||
(spec_ri[..., 0].square() + spec_ri[..., 1].square())
|
||||
.clamp_min(1e-10)
|
||||
.sqrt()
|
||||
.squeeze(0)
|
||||
)
|
||||
|
||||
mel = (
|
||||
_mel_basis(params.sample_rate, n_fft, params.n_mels)
|
||||
.matmul(magnitude)
|
||||
.clamp_min(1e-10)
|
||||
.log10()
|
||||
)
|
||||
mel = mel.to(torch.float64).clamp(
|
||||
min=params.dmel_min_value, max=params.dmel_max_value
|
||||
)
|
||||
bin_centers = torch.linspace(
|
||||
params.dmel_min_value,
|
||||
params.dmel_max_value,
|
||||
params.num_dmel_bins,
|
||||
dtype=torch.float64,
|
||||
)
|
||||
dmel_bins = (mel.unsqueeze(-1) - bin_centers).abs().argmin(dim=-1)
|
||||
return dmel_bins.to(torch.int32).T.contiguous()
|
||||
|
||||
|
||||
class InklingAudioFeatureExtractor(FeatureExtractionMixin):
|
||||
"""Convert raw audio into Inkling dMel bins in the HF feature-extractor API."""
|
||||
|
||||
model_input_names = ["dmel_bins"]
|
||||
|
||||
def __init__(self, params: dict | None = None, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
merged = InklingAudioEncoderParams()
|
||||
if params:
|
||||
for k, v in params.items():
|
||||
if hasattr(merged, k):
|
||||
setattr(merged, k, v)
|
||||
# also accept flat kwargs (HF config style)
|
||||
for k in list(kwargs.keys()):
|
||||
if hasattr(merged, k):
|
||||
setattr(merged, k, kwargs[k])
|
||||
self.params = merged
|
||||
|
||||
def _decode_one(self, audio) -> torch.Tensor:
|
||||
# vLLM hands the feature extractor numpy arrays (the dummy-input builder
|
||||
# during profiling, and MultiModalDataParser after resampling to the
|
||||
# target sample rate), so no other input types need to be supported.
|
||||
return torch.from_numpy(
|
||||
np.ascontiguousarray(audio.astype(np.float32, copy=False))
|
||||
).flatten()
|
||||
|
||||
def _encode_one(self, audio) -> torch.Tensor:
|
||||
return _dmel_bins(self._decode_one(audio), self.params)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
audios: Sequence | None,
|
||||
return_tensors: str | None = None,
|
||||
**kwargs,
|
||||
) -> BatchFeature:
|
||||
del return_tensors, kwargs
|
||||
if audios is None:
|
||||
audios = []
|
||||
if not isinstance(audios, (list, tuple)):
|
||||
audios = [audios]
|
||||
|
||||
dmel_bins = [self._encode_one(a) for a in audios]
|
||||
data = {
|
||||
# per-clip feature: dmel bins as float32 [T, n_mels]
|
||||
"dmel_bins": [bins.to(torch.float32) for bins in dmel_bins],
|
||||
"num_audio_tokens": [int(bins.shape[0]) for bins in dmel_bins],
|
||||
}
|
||||
# return_tensors intentionally ignored: per-clip features have ragged T.
|
||||
return BatchFeature(data=data, tensor_type=None)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Composite processor
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class InklingProcessor:
|
||||
"""Bundle Inkling image + audio preprocessing with the MM token ids."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image_processor: InklingImageProcessor | None = None,
|
||||
audio_feature_extractor: InklingAudioFeatureExtractor | None = None,
|
||||
tokenizer=None,
|
||||
):
|
||||
self.image_processor = image_processor or InklingImageProcessor()
|
||||
self.audio_feature_extractor = (
|
||||
audio_feature_extractor or InklingAudioFeatureExtractor()
|
||||
)
|
||||
self.tokenizer = tokenizer
|
||||
|
||||
def process_images(self, images: list):
|
||||
"""Raw images -> BatchFeature(vision_patches_bthwc, num_patches, num_tokens)."""
|
||||
return self.image_processor.preprocess(images, return_tensors="pt")
|
||||
|
||||
def process_audios(self, audios: list):
|
||||
"""Raw audios -> BatchFeature(dmel_bins, num_audio_tokens)."""
|
||||
return self.audio_feature_extractor(audios)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"InklingImageProcessor",
|
||||
"InklingAudioFeatureExtractor",
|
||||
"InklingAudioEncoderParams",
|
||||
"InklingProcessor",
|
||||
"IMAGE_MARKER_ID",
|
||||
"AUDIO_MARKER_ID",
|
||||
"IMAGE_TOKEN_ID",
|
||||
"AUDIO_TOKEN_ID",
|
||||
]
|
||||
@@ -16,10 +16,8 @@ def _compute_slot_mapping_kernel_impl(
|
||||
positions: torch.Tensor, # [num_tokens], int64
|
||||
block_table: torch.Tensor, # [max_num_reqs, max_num_blocks_per_req], int32
|
||||
block_table_stride: int, # max_num_blocks_per_req
|
||||
block_size: int,
|
||||
slot_mapping: torch.Tensor, # [max_num_tokens], int64
|
||||
KV_CACHE_BLOCK_SIZE: int | None = None,
|
||||
BLOCKS_PER_KV_BLOCK: int = 1,
|
||||
KV_CACHE_BLOCK_SIZE: int,
|
||||
TOTAL_CP_WORLD_SIZE: int = 1,
|
||||
TOTAL_CP_RANK: int = 0,
|
||||
CP_KV_CACHE_INTERLEAVE_SIZE: int = 1,
|
||||
@@ -27,14 +25,12 @@ def _compute_slot_mapping_kernel_impl(
|
||||
BLOCK_SIZE: int = 1024,
|
||||
) -> None:
|
||||
assert TOTAL_CP_WORLD_SIZE == 1, "Context Parallelism is not supported on CPU."
|
||||
if BLOCKS_PER_KV_BLOCK != 1:
|
||||
assert block_size * BLOCKS_PER_KV_BLOCK == KV_CACHE_BLOCK_SIZE
|
||||
torch.ops._C.compute_slot_mapping_kernel_impl(
|
||||
query_start_loc,
|
||||
positions,
|
||||
block_table,
|
||||
slot_mapping,
|
||||
block_size,
|
||||
KV_CACHE_BLOCK_SIZE,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -358,6 +358,19 @@ autotune = _lazy_import_wrapper(
|
||||
)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def has_flashinfer_autotune_v2() -> bool:
|
||||
"""Return ``True`` if FlashInfer ships the managed-cache autotuner
|
||||
(``flashinfer.autotune_v2``, flashinfer-ai/flashinfer#3861)."""
|
||||
if not has_flashinfer():
|
||||
return False
|
||||
mod = _get_submodule("flashinfer")
|
||||
return mod is not None and all(
|
||||
callable(getattr(mod, name, None))
|
||||
for name in ("autotune_v2", "autotune_v2_reload")
|
||||
)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def has_flashinfer_comm() -> bool:
|
||||
"""Return `True` if FlashInfer comm module is available."""
|
||||
@@ -1269,6 +1282,7 @@ __all__ = [
|
||||
"flashinfer_fused_kda_decode",
|
||||
"autotune",
|
||||
"has_flashinfer_moe",
|
||||
"has_flashinfer_autotune_v2",
|
||||
"has_flashinfer_comm",
|
||||
"has_flashinfer_nvlink_two_sided",
|
||||
"has_flashinfer_nvlink_one_sided",
|
||||
|
||||
@@ -6,7 +6,7 @@ from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config import VllmConfig, get_current_vllm_config_or_none
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointBuilder,
|
||||
MambaPrefillCheckpointMetadata,
|
||||
@@ -15,12 +15,13 @@ from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
CommonAttentionMetadata,
|
||||
max_decode_query_len,
|
||||
)
|
||||
from vllm.v1.attention.backends.mamba_attn import (
|
||||
BaseMambaAttentionMetadata,
|
||||
BaseMambaAttentionMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
from vllm.v1.kv_cache_interface import KVCacheSpec, MambaSpec
|
||||
|
||||
|
||||
def compute_varlen_chunk_metadata(
|
||||
@@ -104,6 +105,14 @@ class Mamba2AttentionBackend(AttentionBackend):
|
||||
def is_ssm(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def supports_device_cpu_query_lens_mismatch(cls) -> bool:
|
||||
# Decode kernels take device offsets; ReplaySSM plans off CPU counts.
|
||||
vllm_config = get_current_vllm_config_or_none()
|
||||
if vllm_config is None:
|
||||
return True
|
||||
return not vllm_config.cache_config.use_replayssm
|
||||
|
||||
|
||||
@dataclass
|
||||
class Mamba2AttentionMetadata(BaseMambaAttentionMetadata):
|
||||
@@ -124,6 +133,14 @@ class Mamba2AttentionMetadataBuilder(
|
||||
):
|
||||
metadata_cls = Mamba2AttentionMetadata
|
||||
|
||||
@classmethod
|
||||
def get_varlen_cudagraph_max_query_len(
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: KVCacheSpec,
|
||||
) -> int | None:
|
||||
return max_decode_query_len(vllm_config)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_spec: MambaSpec,
|
||||
|
||||
@@ -22,6 +22,17 @@ if TYPE_CHECKING:
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
|
||||
|
||||
def is_cute_dsl_mla_prefill_supported(
|
||||
qk_head_dim: int, v_head_dim: int, query_dtype: torch.dtype
|
||||
) -> bool:
|
||||
# FlashInfer has no public compatibility API; replace these checks when available.
|
||||
return (
|
||||
query_dtype in (torch.float16, torch.bfloat16, torch.float8_e4m3fn)
|
||||
and (qk_head_dim, v_head_dim) in ((128, 128), (192, 128))
|
||||
and (qk_head_dim != 192 or query_dtype == torch.float8_e4m3fn)
|
||||
)
|
||||
|
||||
|
||||
class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
|
||||
"""TRT-LLM Ragged backend for MLA prefill."""
|
||||
|
||||
@@ -79,6 +90,7 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
|
||||
v_head_dim=v_head_dim,
|
||||
vllm_config=vllm_config,
|
||||
)
|
||||
self._context_backend = vllm_config.attention_config.trtllm_ragged_backend
|
||||
self._workspace_spec = (
|
||||
(envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE,),
|
||||
torch.uint8,
|
||||
@@ -198,6 +210,16 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
|
||||
"TRTLLM ragged context prefill contains an empty query or KV row"
|
||||
)
|
||||
|
||||
if self._context_backend == "cute-dsl":
|
||||
if not is_cute_dsl_mla_prefill_supported(q.shape[-1], v.shape[-1], q.dtype):
|
||||
raise ValueError(
|
||||
"CuTe DSL context prefill does not support "
|
||||
f"Q/K head dimension {q.shape[-1]}, "
|
||||
f"V head dimension {v.shape[-1]}, and query dtype {q.dtype}."
|
||||
)
|
||||
# Generic MLA supplies V as a projection slice; CuTe requires packed V.
|
||||
v = v.contiguous()
|
||||
|
||||
if out is None:
|
||||
out = torch.empty(
|
||||
q.shape[0],
|
||||
@@ -225,6 +247,7 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
|
||||
cum_seq_lens_kv=chunk.cu_seq_lens,
|
||||
enable_pdl=False,
|
||||
is_causal=False,
|
||||
backend=self._context_backend,
|
||||
return_lse=True,
|
||||
out=out,
|
||||
skip_all_rows_active_check=True,
|
||||
|
||||
@@ -35,6 +35,7 @@ from vllm.v1.attention.backend import (
|
||||
AttentionLayer,
|
||||
CommonAttentionMetadata,
|
||||
MultipleOf,
|
||||
max_decode_query_len,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.triton_mla import _compute_num_kv_splits
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
@@ -49,7 +50,11 @@ from vllm.v1.attention.ops.rocm_aiter_mla_prefill import (
|
||||
gather_compressed_context,
|
||||
)
|
||||
from vllm.v1.attention.ops.triton_decode_attention import decode_attention_fwd
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec, is_quantized_kv_cache
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
KVCacheSpec,
|
||||
is_quantized_kv_cache,
|
||||
)
|
||||
from vllm.v1.worker.workspace import (
|
||||
current_workspace_manager,
|
||||
is_workspace_manager_initialized,
|
||||
@@ -457,6 +462,16 @@ def _aiter_mla_small_head_mode() -> str:
|
||||
return mode
|
||||
|
||||
|
||||
def _adaptive_varlen_verify_enabled(vllm_config: VllmConfig) -> bool:
|
||||
speculative_config = vllm_config.speculative_config
|
||||
return (
|
||||
speculative_config is not None
|
||||
and bool(getattr(speculative_config, "enable_adaptive_verification", False))
|
||||
and vllm_config.parallel_config.decode_context_parallel_size == 1
|
||||
and max_decode_query_len(vllm_config) > 1
|
||||
)
|
||||
|
||||
|
||||
class AiterMLABackend(MLACommonBackend):
|
||||
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16]
|
||||
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
|
||||
@@ -620,6 +635,8 @@ class AiterMLADecodeMetadata(MLACommonDecodeMetadata):
|
||||
use_gluon_verify: bool = False
|
||||
# Whether persistent MLA metadata was computed
|
||||
has_persistent_metadata: bool = False
|
||||
varlen_q_rows: torch.Tensor | None = None
|
||||
varlen_o_rows: torch.Tensor | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -720,6 +737,21 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
|
||||
return uniform_qo_len
|
||||
|
||||
@classmethod
|
||||
def get_varlen_cudagraph_max_query_len(
|
||||
cls: type["AiterMLAMetadataBuilder"],
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: KVCacheSpec,
|
||||
) -> int | None:
|
||||
if not _adaptive_varlen_verify_enabled(vllm_config):
|
||||
return None
|
||||
if (
|
||||
cls.get_cudagraph_support(vllm_config, kv_cache_spec)
|
||||
!= AttentionCGSupport.UNIFORM_BATCH
|
||||
):
|
||||
return None
|
||||
return max_decode_query_len(vllm_config)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_spec: AttentionSpec,
|
||||
@@ -823,6 +855,26 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
# persistent gate below and makes aiter raise a KeyError mid-run.
|
||||
self._mtp_decode_qlen = self.reorder_batch_threshold or 1
|
||||
|
||||
self._varlen_decode_qlen = 0
|
||||
if _adaptive_varlen_verify_enabled(vllm_config):
|
||||
assert self._mtp_decode_qlen == max_decode_query_len(vllm_config), (
|
||||
self._mtp_decode_qlen,
|
||||
max_decode_query_len(vllm_config),
|
||||
)
|
||||
self._varlen_decode_qlen = self._mtp_decode_qlen
|
||||
self.query_len_support = QueryLenSupport.VARLEN # type: ignore[misc]
|
||||
max_rows = vllm_config.scheduler_config.max_num_seqs * (
|
||||
self._varlen_decode_qlen
|
||||
)
|
||||
row_ids = torch.arange(max_rows, dtype=torch.int64, device=device)
|
||||
self._varlen_row_req = row_ids // self._varlen_decode_qlen
|
||||
self._varlen_row_pos = row_ids % self._varlen_decode_qlen
|
||||
self._varlen_row_ids = row_ids
|
||||
self._varlen_q_rows = torch.zeros_like(row_ids)
|
||||
self._varlen_o_rows = torch.zeros(
|
||||
max_rows + 1, dtype=torch.int64, device=device
|
||||
)
|
||||
|
||||
# Store the kernel block size from the spec. When kernel_block_size=1
|
||||
# (no spec-dec), behavior is identical to the original. When > 1
|
||||
# (e.g. 16 with Eagle3), we expand block-level indices into per-token
|
||||
@@ -850,6 +902,12 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
max_num_pages_per_req = vllm_config.model_config.max_model_len
|
||||
max_num_reqs = vllm_config.scheduler_config.max_num_seqs
|
||||
max_num_pages = max_num_reqs * max_num_pages_per_req
|
||||
speculative_config = vllm_config.speculative_config
|
||||
if _adaptive_varlen_verify_enabled(vllm_config):
|
||||
assert speculative_config is not None
|
||||
max_num_pages += max_num_reqs * (
|
||||
speculative_config.num_speculative_tokens or 0
|
||||
)
|
||||
|
||||
# Preparing persistent buffers
|
||||
# TODO: we can disambiguate between decode and mixed-prefill decode here
|
||||
@@ -1437,13 +1495,18 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
device = self.device
|
||||
num_reqs = seq_lens_device.size(0)
|
||||
qo_len = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1]
|
||||
max_qo_len = qo_len.max().item()
|
||||
padded_mtp_qo_len = self._uniform_padded_mtp_qo_len(
|
||||
qo_len, max_qo_len, num_decode_tokens
|
||||
)
|
||||
if padded_mtp_qo_len > 0:
|
||||
max_qo_len = padded_mtp_qo_len
|
||||
pad_uniform_mtp = padded_mtp_qo_len > 0
|
||||
varlen_decode = self._varlen_decode_qlen > 0 and causal
|
||||
if varlen_decode:
|
||||
max_qo_len = self._varlen_decode_qlen
|
||||
pad_uniform_mtp = True
|
||||
else:
|
||||
max_qo_len = qo_len.max().item()
|
||||
padded_mtp_qo_len = self._uniform_padded_mtp_qo_len(
|
||||
qo_len, max_qo_len, num_decode_tokens
|
||||
)
|
||||
if padded_mtp_qo_len > 0:
|
||||
max_qo_len = padded_mtp_qo_len
|
||||
pad_uniform_mtp = padded_mtp_qo_len > 0
|
||||
|
||||
seq_lens_for_kernel = seq_lens_device
|
||||
# the global lengths need the same dummy-row
|
||||
@@ -1477,6 +1540,13 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
g_tot_seq_lens.new_full((), max_qo_len * self.dcp_world_size),
|
||||
)
|
||||
|
||||
varlen_q_rows = varlen_o_rows = None
|
||||
if varlen_decode:
|
||||
seq_lens_for_kernel = seq_lens_device + (max_qo_len - qo_lens_device)
|
||||
varlen_q_rows, varlen_o_rows = self._fill_varlen_rows(
|
||||
query_start_loc_device, qo_lens_device, num_reqs
|
||||
)
|
||||
|
||||
if self._graph_seq_lens is not None:
|
||||
self._graph_seq_lens[:num_kernel_reqs].copy_(
|
||||
seq_lens_for_kernel, non_blocking=True
|
||||
@@ -1537,6 +1607,8 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
# device synchronisation. Programs whose chunk lies past
|
||||
# num_tokens mask out entirely.
|
||||
max_tokens_per_req = block_table_tensor.shape[1] * self.kernel_block_size
|
||||
if varlen_decode:
|
||||
max_tokens_per_req += max_qo_len - 1
|
||||
num_chunks = max(1, cdiv(max_tokens_per_req, 1024))
|
||||
_expand_page_indices_kernel[(num_reqs, num_chunks)](
|
||||
self.paged_kv_indices,
|
||||
@@ -1544,8 +1616,10 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
block_table_tensor.stride(0),
|
||||
block_table_tensor.stride(1),
|
||||
paged_kv_indptr,
|
||||
seq_lens_device,
|
||||
KERNEL_BLOCK_SIZE=self.kernel_block_size,
|
||||
BLOCK_SIZE=1024,
|
||||
CLAMP_TAIL=varlen_decode,
|
||||
)
|
||||
paged_kv_indices = self.paged_kv_indices
|
||||
|
||||
@@ -1740,10 +1814,38 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
use_gluon_verify=use_gluon_verify,
|
||||
attn_out_dtype=self.decode_attn_out_dtype,
|
||||
has_persistent_metadata=has_persistent_metadata,
|
||||
varlen_q_rows=varlen_q_rows,
|
||||
varlen_o_rows=varlen_o_rows,
|
||||
)
|
||||
|
||||
return attn_metadata
|
||||
|
||||
def _fill_varlen_rows(
|
||||
self,
|
||||
query_start_loc_device: torch.Tensor,
|
||||
qo_lens_device: torch.Tensor,
|
||||
num_reqs: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
width = self._varlen_decode_qlen
|
||||
num_rows = num_reqs * width
|
||||
row_req = self._varlen_row_req[:num_rows]
|
||||
row_pos = self._varlen_row_pos[:num_rows]
|
||||
row_qlen = qo_lens_device.to(torch.int64).index_select(0, row_req)
|
||||
row_start = query_start_loc_device.to(torch.int64).index_select(0, row_req)
|
||||
is_real = row_pos < row_qlen
|
||||
packed_row = row_start + row_pos
|
||||
q_rows = self._varlen_q_rows[:num_rows]
|
||||
q_rows.copy_(torch.where(is_real, packed_row, 0))
|
||||
o_rows = self._varlen_o_rows
|
||||
o_rows.zero_()
|
||||
dump_slot = o_rows.size(0) - 1
|
||||
o_rows.scatter_(
|
||||
0,
|
||||
torch.where(is_real, packed_row, dump_slot),
|
||||
self._varlen_row_ids[:num_rows],
|
||||
)
|
||||
return q_rows, o_rows
|
||||
|
||||
def build(
|
||||
self,
|
||||
common_prefix_len: int,
|
||||
@@ -1795,8 +1897,10 @@ def _expand_page_indices_kernel(
|
||||
block_table_stride_0,
|
||||
block_table_stride_1,
|
||||
cu_num_tokens,
|
||||
num_real_tokens,
|
||||
KERNEL_BLOCK_SIZE: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
CLAMP_TAIL: tl.constexpr = False,
|
||||
):
|
||||
"""Expand block table entries into per-token flat page indices.
|
||||
|
||||
@@ -1810,6 +1914,10 @@ def _expand_page_indices_kernel(
|
||||
|
||||
When KERNEL_BLOCK_SIZE=K: block table entry b (covering K tokens)
|
||||
is expanded to flat indices b*K, b*K+1, ..., b*K+(K-1).
|
||||
|
||||
With CLAMP_TAIL, entries past num_real_tokens[req] (padding a request's
|
||||
page list out to its kernel length) repeat the request's first page, so
|
||||
they never index past its block-table row.
|
||||
"""
|
||||
# One program per (request, token-chunk). Parallelising over requests
|
||||
# alone degenerates at low concurrency: with num_reqs == 1 a single
|
||||
@@ -1819,6 +1927,8 @@ def _expand_page_indices_kernel(
|
||||
row_ptr = block_table + req_idx * block_table_stride_0
|
||||
start_idx = tl.load(cu_num_tokens + req_idx)
|
||||
num_tokens = tl.load(cu_num_tokens + req_idx + 1) - start_idx
|
||||
if CLAMP_TAIL:
|
||||
real_tokens = tl.load(num_real_tokens + req_idx)
|
||||
|
||||
# The grid is sized from the block table width, an upper bound over all
|
||||
# requests, so a ragged batch launches chunks past a short request's end.
|
||||
@@ -1830,11 +1940,14 @@ def _expand_page_indices_kernel(
|
||||
|
||||
token_offsets = chunk_start + tl.arange(0, BLOCK_SIZE)
|
||||
mask = token_offsets < num_tokens
|
||||
source_offsets = token_offsets
|
||||
if CLAMP_TAIL:
|
||||
source_offsets = tl.where(token_offsets < real_tokens, token_offsets, 0)
|
||||
|
||||
# Which block in the block table does this token belong to?
|
||||
block_idx = token_offsets // KERNEL_BLOCK_SIZE
|
||||
block_idx = source_offsets // KERNEL_BLOCK_SIZE
|
||||
# Offset within that block
|
||||
offset_in_block = token_offsets % KERNEL_BLOCK_SIZE
|
||||
offset_in_block = source_offsets % KERNEL_BLOCK_SIZE
|
||||
|
||||
# Load the block ID from the block table
|
||||
# Both strides are taken from the caller: the block table is a view owned
|
||||
@@ -2600,6 +2713,28 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
|
||||
kv_c_and_k_pe_cache: torch.Tensor,
|
||||
attn_metadata: AiterMLAMetadata,
|
||||
layer: AttentionLayer,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
decode = attn_metadata.decode
|
||||
if decode is None or decode.varlen_q_rows is None:
|
||||
return self._forward_mqa(q, kv_c_and_k_pe_cache, attn_metadata, layer)
|
||||
assert decode.varlen_o_rows is not None
|
||||
q_rows = decode.varlen_q_rows
|
||||
if isinstance(q, tuple):
|
||||
num_tokens = q[0].shape[0]
|
||||
q = (q[0].index_select(0, q_rows), q[1].index_select(0, q_rows))
|
||||
else:
|
||||
num_tokens = q.shape[0]
|
||||
q = q.index_select(0, q_rows)
|
||||
o, lse = self._forward_mqa(q, kv_c_and_k_pe_cache, attn_metadata, layer)
|
||||
assert lse is None
|
||||
return o.index_select(0, decode.varlen_o_rows[:num_tokens]), None
|
||||
|
||||
def _forward_mqa(
|
||||
self,
|
||||
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
kv_c_and_k_pe_cache: torch.Tensor,
|
||||
attn_metadata: AiterMLAMetadata,
|
||||
layer: AttentionLayer,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
assert kv_c_and_k_pe_cache.numel() > 0
|
||||
assert attn_metadata.decode is not None
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Interface and resolution for pluggable KV cache config builders."""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.utils.import_utils import resolve_obj_by_qualname
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheSpec,
|
||||
)
|
||||
|
||||
|
||||
class KVCacheConfigBuilder(ABC):
|
||||
"""Interface for platform KV cache placement.
|
||||
|
||||
.. note::
|
||||
This interface is experimental and subject to change. The exact hooks
|
||||
and the ``num_blocks`` contract may evolve before it stabilizes.
|
||||
|
||||
Subclasses normally inherit from ``DefaultKVCacheConfigBuilder`` and
|
||||
override only the hooks they need. The cross-worker planning flow
|
||||
(:func:`vllm.v1.core.kv_cache_planning.get_kv_cache_configs`) is Core-owned;
|
||||
this builder supplies the placement decisions: how layers are split into
|
||||
groups and how a group list is materialized into tensors for a given block
|
||||
count. Grouping coordination lives in ``DefaultKVCacheConfigBuilder`` (a
|
||||
model implementing :class:`~vllm.model_executor.models.interfaces.
|
||||
SupportsKVCacheGrouping` owns its grouping there); a platform that replaces
|
||||
the builder fully owns its own grouping.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def get_kv_cache_groups(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
kv_cache_spec: dict[str, "KVCacheSpec"],
|
||||
) -> list["KVCacheGroupSpec"]:
|
||||
"""Split the given layers into KV cache groups.
|
||||
|
||||
Planning passes all workers' layers; profiling passes one worker's.
|
||||
Core rejects a result that does not put each layer in exactly one group.
|
||||
|
||||
The default builder honors a model's
|
||||
:class:`~vllm.model_executor.models.interfaces.SupportsKVCacheGrouping`
|
||||
declaration here before falling back to the default grouping; a
|
||||
platform-specific builder decides its own grouping.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def get_kv_cache_config_from_groups(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
kv_cache_groups: list["KVCacheGroupSpec"],
|
||||
num_blocks: int,
|
||||
) -> "KVCacheConfig":
|
||||
"""Materialize the groups for exactly ``num_blocks`` blocks.
|
||||
|
||||
Each device backing's size is ``bytes_per_block * num_blocks``. All
|
||||
device tensors share one backing allocation unless the builder
|
||||
overrides ``_get_pool_bytes_per_block``. Host-resident tensors, such as
|
||||
HiSparse source storage, are excluded from the device byte accounting
|
||||
and sized independently.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def _get_pool_bytes_per_block(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
kv_cache_groups: list["KVCacheGroupSpec"],
|
||||
) -> int | None:
|
||||
"""Return the device bytes one block ID consumes, or ``None``.
|
||||
|
||||
Private and transitional; expected to go away with worker-driven
|
||||
allocation (RFC #42449). With ``None`` (the default), Core derives the
|
||||
value from a one-block placement with one device backing. Override it
|
||||
only for placements with several device allocations.
|
||||
"""
|
||||
return None
|
||||
|
||||
|
||||
def get_kv_cache_config_builder(
|
||||
vllm_config: "VllmConfig",
|
||||
) -> KVCacheConfigBuilder:
|
||||
"""Resolve the builder selected by the current platform.
|
||||
|
||||
Resolution priority is owned by the platform hook
|
||||
(:meth:`vllm.platforms.interface.Platform.get_kv_cache_config_builder_cls`).
|
||||
Builders are stateless (every method takes ``vllm_config``), so a fresh
|
||||
instance is returned per call and engines with different configs in the
|
||||
same process always get the correct builder.
|
||||
"""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
builder_cls = resolve_obj_by_qualname(
|
||||
current_platform.get_kv_cache_config_builder_cls(vllm_config)
|
||||
)
|
||||
return builder_cls()
|
||||
@@ -14,6 +14,7 @@ from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv, round_up
|
||||
from vllm.utils.mem_utils import format_gib
|
||||
from vllm.utils.torch_utils import get_dtype_size
|
||||
from vllm.v1.core.kv_cache_config_builder import KVCacheConfigBuilder
|
||||
from vllm.v1.core.kv_cache_utils import _kv_cache_device_name
|
||||
from vllm.v1.hisparse.layout import (
|
||||
get_hisparse_host_pool_bytes,
|
||||
@@ -261,6 +262,48 @@ def _pool_bytes_per_block(
|
||||
)
|
||||
|
||||
|
||||
def _pool_bytes_per_block_from_builder(
|
||||
builder: KVCacheConfigBuilder,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
) -> int:
|
||||
"""Bytes consumed by one block in the worker's device KV cache pool.
|
||||
|
||||
Core places one block with the builder and reads the size of its single
|
||||
device backing. A placement with several device allocations declares the
|
||||
value through ``KVCacheConfigBuilder._get_pool_bytes_per_block``. HiSparse
|
||||
keeps the pure computation, since probing its placement would check the host
|
||||
pool against ``max_model_len`` before auto-fit shortens it.
|
||||
``num_gpu_blocks_override`` is suppressed so it cannot inflate the probe.
|
||||
"""
|
||||
if not kv_cache_groups:
|
||||
return 0
|
||||
if vllm_config.attention_config.hisparse_config is not None:
|
||||
return _pool_bytes_per_block(vllm_config, kv_cache_groups)
|
||||
pool_bytes = builder._get_pool_bytes_per_block(vllm_config, kv_cache_groups)
|
||||
if pool_bytes is not None:
|
||||
return pool_bytes
|
||||
saved_override = vllm_config.cache_config.num_gpu_blocks_override
|
||||
vllm_config.cache_config.num_gpu_blocks_override = None
|
||||
try:
|
||||
probe = builder.get_kv_cache_config_from_groups(
|
||||
vllm_config, kv_cache_groups, num_blocks=1
|
||||
)
|
||||
finally:
|
||||
vllm_config.cache_config.num_gpu_blocks_override = saved_override
|
||||
backing_sizes = {
|
||||
tensor.size for tensor in probe.kv_cache_tensors if not tensor.host_resident
|
||||
}
|
||||
if len(backing_sizes) != 1:
|
||||
raise ValueError(
|
||||
"KV cache tensors must share one device backing allocation, but got "
|
||||
f"sizes {sorted(backing_sizes)}. Override "
|
||||
"KVCacheConfigBuilder._get_pool_bytes_per_block() for placements "
|
||||
"with several device allocations."
|
||||
)
|
||||
return backing_sizes.pop()
|
||||
|
||||
|
||||
def get_uniform_page_size(kv_cache_specs: Iterable[KVCacheSpec]) -> int:
|
||||
"""Get the page size of the KV cache."""
|
||||
page_sizes = {layer.page_size_bytes for layer in kv_cache_specs}
|
||||
@@ -1264,6 +1307,36 @@ def get_kv_cache_groups(
|
||||
return groups
|
||||
|
||||
|
||||
def get_kv_cache_groups_from_builder(
|
||||
builder: KVCacheConfigBuilder,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: dict[str, KVCacheSpec],
|
||||
) -> list[KVCacheGroupSpec]:
|
||||
"""Group the layers with the builder and check the result.
|
||||
|
||||
Core checks every grouping, whether it comes from the default grouping, a
|
||||
model's ``SupportsKVCacheGrouping`` or a platform builder.
|
||||
"""
|
||||
groups = builder.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
_check_exact_partition(groups, kv_cache_spec)
|
||||
return groups
|
||||
|
||||
|
||||
def _check_exact_partition(
|
||||
groups: list[KVCacheGroupSpec],
|
||||
kv_cache_spec: dict[str, KVCacheSpec],
|
||||
) -> None:
|
||||
"""Ensure the groups partition every input layer exactly once."""
|
||||
grouped_layers: set[str] = set()
|
||||
for group in groups:
|
||||
assert not set(group.layer_names) & grouped_layers, (
|
||||
f"KV cache group overlaps: {set(group.layer_names) & grouped_layers}."
|
||||
)
|
||||
grouped_layers |= set(group.layer_names)
|
||||
missing = set(kv_cache_spec) - grouped_layers
|
||||
assert not missing, f"KV cache groups miss layers: {sorted(missing)}."
|
||||
|
||||
|
||||
def _layer_tp_replicas(spec: KVCacheSpec, tp_size: int, dcp_size: int) -> int:
|
||||
if not isinstance(spec, AttentionSpec) or spec.max_tp_shards is None:
|
||||
return 1
|
||||
@@ -1285,6 +1358,7 @@ def kv_cache_groups_tp_replicas(
|
||||
|
||||
|
||||
def _max_memory_usage_bytes_from_groups(
|
||||
builder: KVCacheConfigBuilder,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
) -> int:
|
||||
@@ -1301,7 +1375,9 @@ def _max_memory_usage_bytes_from_groups(
|
||||
if not kv_cache_groups:
|
||||
return 0
|
||||
|
||||
bytes_per_block = _pool_bytes_per_block(vllm_config, kv_cache_groups)
|
||||
bytes_per_block = _pool_bytes_per_block_from_builder(
|
||||
builder, vllm_config, kv_cache_groups
|
||||
)
|
||||
total_blocks = 0
|
||||
for group in kv_cache_groups:
|
||||
spec = group.kv_cache_spec
|
||||
@@ -1317,6 +1393,7 @@ def _max_memory_usage_bytes_from_groups(
|
||||
|
||||
|
||||
def _estimate_max_model_len_from_groups(
|
||||
builder: KVCacheConfigBuilder,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
available_memory: int,
|
||||
@@ -1334,13 +1411,16 @@ def _estimate_max_model_len_from_groups(
|
||||
vllm_config.model_config.max_model_len = model_len
|
||||
if hisparse_enabled:
|
||||
try:
|
||||
get_kv_cache_config_from_groups(
|
||||
vllm_config, kv_cache_groups, available_memory
|
||||
num_blocks = available_memory // _pool_bytes_per_block_from_builder(
|
||||
builder, vllm_config, kv_cache_groups
|
||||
)
|
||||
builder.get_kv_cache_config_from_groups(
|
||||
vllm_config, kv_cache_groups, num_blocks
|
||||
)
|
||||
except ValueError:
|
||||
return False
|
||||
return (
|
||||
_max_memory_usage_bytes_from_groups(vllm_config, kv_cache_groups)
|
||||
_max_memory_usage_bytes_from_groups(builder, vllm_config, kv_cache_groups)
|
||||
<= available_memory
|
||||
)
|
||||
|
||||
@@ -1362,6 +1442,7 @@ def _estimate_max_model_len_from_groups(
|
||||
|
||||
|
||||
def _auto_fit_max_model_len(
|
||||
builder: KVCacheConfigBuilder,
|
||||
vllm_config: VllmConfig,
|
||||
projected_groups_per_worker: list[list[KVCacheGroupSpec]],
|
||||
available_memory: list[int],
|
||||
@@ -1372,6 +1453,7 @@ def _auto_fit_max_model_len(
|
||||
workers.
|
||||
|
||||
Args:
|
||||
builder: The builder whose hooks drive block accounting.
|
||||
vllm_config: The global VllmConfig (will be modified in-place)
|
||||
projected_groups_per_worker: KV cache groups projected to each worker.
|
||||
available_memory: Memory available for KV cache in bytes for each
|
||||
@@ -1395,7 +1477,9 @@ def _auto_fit_max_model_len(
|
||||
for groups, avail_mem in zip(projected_groups_per_worker, available_memory):
|
||||
if not groups:
|
||||
continue
|
||||
worker_max = _estimate_max_model_len_from_groups(vllm_config, groups, avail_mem)
|
||||
worker_max = _estimate_max_model_len_from_groups(
|
||||
builder, vllm_config, groups, avail_mem
|
||||
)
|
||||
if worker_max < auto_fit_max:
|
||||
auto_fit_max = worker_max
|
||||
limiting_worker_mem = avail_mem
|
||||
@@ -1473,6 +1557,9 @@ def _project_kv_cache_groups_to_worker(
|
||||
worker_layer_names,
|
||||
group_spec,
|
||||
is_eagle_group=group.is_eagle_group and bool(worker_layer_names),
|
||||
host_resident=group.host_resident,
|
||||
enable_kv_transfer=group.enable_kv_transfer,
|
||||
role=group.role,
|
||||
)
|
||||
)
|
||||
return projected_groups
|
||||
@@ -1512,6 +1599,10 @@ def get_kv_cache_configs(
|
||||
The generated KVCacheConfigs for each worker.
|
||||
|
||||
"""
|
||||
from vllm.v1.core.kv_cache_config_builder import get_kv_cache_config_builder
|
||||
|
||||
builder = get_kv_cache_config_builder(vllm_config)
|
||||
|
||||
# Merge the KV cache specs of all workers. Different PP stages may have
|
||||
# different layer names, and different TP ranks of the same PP stage should
|
||||
# have the same KV cache spec.
|
||||
@@ -1542,7 +1633,9 @@ def get_kv_cache_configs(
|
||||
# Get global KV cache groups. This also handles spec unification for
|
||||
# hybrid models when disable_hybrid_kv_cache_manager is enabled.
|
||||
# After this call, merged_kv_cache_specs may be modified in-place.
|
||||
global_kv_cache_groups = get_kv_cache_groups(vllm_config, merged_kv_cache_specs)
|
||||
global_kv_cache_groups = get_kv_cache_groups_from_builder(
|
||||
builder, vllm_config, merged_kv_cache_specs
|
||||
)
|
||||
|
||||
# If original_max_model_len was -1, automatically
|
||||
# determine the maximum model length that fits in available GPU memory.
|
||||
@@ -1553,11 +1646,10 @@ def get_kv_cache_configs(
|
||||
]
|
||||
|
||||
# If `num_gpu_blocks_override` is set, the cache size that will actually
|
||||
# be allocated is decoupled from the profiled `available_memory`:
|
||||
# `may_override_num_blocks` in `get_kv_cache_config_from_groups` clamps
|
||||
# `num_blocks` to the override. Reflect that in `available_memory` here so
|
||||
# auto-fit, the admission check, and the per-worker config builder all
|
||||
# plan against the same effective capacity.
|
||||
# be allocated is decoupled from the profiled `available_memory`. Reflect
|
||||
# that in `available_memory` here so auto-fit, the admission check, and
|
||||
# the per-worker config builder all plan against the same effective
|
||||
# capacity.
|
||||
override = vllm_config.cache_config.num_gpu_blocks_override
|
||||
if override is not None:
|
||||
adjusted_memory: list[int] = []
|
||||
@@ -1565,7 +1657,9 @@ def get_kv_cache_configs(
|
||||
if not groups:
|
||||
adjusted_memory.append(avail_mem)
|
||||
continue
|
||||
bytes_per_block = _pool_bytes_per_block(vllm_config, groups)
|
||||
bytes_per_block = _pool_bytes_per_block_from_builder(
|
||||
builder, vllm_config, groups
|
||||
)
|
||||
logger.info(
|
||||
"Overriding num_gpu_blocks=%d with num_gpu_blocks_override=%d",
|
||||
avail_mem // bytes_per_block,
|
||||
@@ -1578,12 +1672,18 @@ def get_kv_cache_configs(
|
||||
# the capacity check both plan against usable blocks. Allocation below
|
||||
# still uses the full memory.
|
||||
check_memory = [
|
||||
avail_mem - _pool_bytes_per_block(vllm_config, groups) if groups else avail_mem
|
||||
(
|
||||
avail_mem - _pool_bytes_per_block_from_builder(builder, vllm_config, groups)
|
||||
if groups
|
||||
else avail_mem
|
||||
)
|
||||
for groups, avail_mem in zip(projected_groups_per_worker, available_memory)
|
||||
]
|
||||
|
||||
if vllm_config.model_config.original_max_model_len == -1:
|
||||
_auto_fit_max_model_len(vllm_config, projected_groups_per_worker, check_memory)
|
||||
_auto_fit_max_model_len(
|
||||
builder, vllm_config, projected_groups_per_worker, check_memory
|
||||
)
|
||||
|
||||
# Check if the available memory is enough per worker.
|
||||
for groups, avail_mem in zip(projected_groups_per_worker, check_memory):
|
||||
@@ -1591,9 +1691,9 @@ def get_kv_cache_configs(
|
||||
continue
|
||||
_check_enough_kv_cache_memory(
|
||||
avail_mem,
|
||||
partial(_max_memory_usage_bytes_from_groups, vllm_config, groups),
|
||||
partial(_max_memory_usage_bytes_from_groups, builder, vllm_config, groups),
|
||||
vllm_config.model_config.max_model_len,
|
||||
partial(_estimate_max_model_len_from_groups, vllm_config, groups),
|
||||
partial(_estimate_max_model_len_from_groups, builder, vllm_config, groups),
|
||||
)
|
||||
|
||||
kv_cache_configs: list[KVCacheConfig] = []
|
||||
@@ -1603,9 +1703,15 @@ def get_kv_cache_configs(
|
||||
assert sum(len(group.layer_names) for group in projected_groups) == len(
|
||||
kv_cache_spec_one_worker
|
||||
), "Some layers are not assigned to any group."
|
||||
bytes_per_block = _pool_bytes_per_block_from_builder(
|
||||
builder, vllm_config, projected_groups
|
||||
)
|
||||
num_blocks = (
|
||||
available_memory_one_worker // bytes_per_block if projected_groups else 1
|
||||
)
|
||||
kv_cache_configs.append(
|
||||
get_kv_cache_config_from_groups(
|
||||
vllm_config, projected_groups, available_memory_one_worker
|
||||
builder.get_kv_cache_config_from_groups(
|
||||
vllm_config, projected_groups, num_blocks
|
||||
)
|
||||
)
|
||||
|
||||
@@ -1618,13 +1724,12 @@ def get_kv_cache_configs(
|
||||
for i, kv_cache_config in enumerate(kv_cache_configs):
|
||||
if kv_cache_config.num_blocks == min_num_blocks:
|
||||
continue
|
||||
# Re-plan with exactly the memory the smallest rank can afford, so
|
||||
# strides and offsets stay consistent with the shrunken allocation.
|
||||
groups = kv_cache_config.kv_cache_groups
|
||||
kv_cache_configs[i] = get_kv_cache_config_from_groups(
|
||||
# Re-plan with exactly the smallest rank's block count, so strides and
|
||||
# offsets stay consistent with the shrunken allocation.
|
||||
kv_cache_configs[i] = builder.get_kv_cache_config_from_groups(
|
||||
vllm_config,
|
||||
groups,
|
||||
min_num_blocks * _pool_bytes_per_block(vllm_config, groups),
|
||||
kv_cache_config.kv_cache_groups,
|
||||
min_num_blocks,
|
||||
)
|
||||
|
||||
for kv_cache_config in kv_cache_configs:
|
||||
@@ -1635,3 +1740,64 @@ def get_kv_cache_configs(
|
||||
)
|
||||
|
||||
return kv_cache_configs
|
||||
|
||||
|
||||
class DefaultKVCacheConfigBuilder(KVCacheConfigBuilder):
|
||||
"""Default grouping and placement, backed by the module-level functions here.
|
||||
|
||||
.. note::
|
||||
This class is part of the experimental ``KVCacheConfigBuilder``
|
||||
interface and is subject to change.
|
||||
|
||||
Subclasses override only the hooks they need. The planning flow and its
|
||||
invariants (``num_gpu_blocks_override``, null-block reservation, auto-fit,
|
||||
admission checks, cross-rank convergence) stay in the Core-owned
|
||||
:func:`get_kv_cache_configs`.
|
||||
"""
|
||||
|
||||
def get_kv_cache_groups(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: dict[str, KVCacheSpec],
|
||||
) -> list[KVCacheGroupSpec]:
|
||||
"""See ``KVCacheConfigBuilder.get_kv_cache_groups``.
|
||||
|
||||
A model implementing ``SupportsKVCacheGrouping`` owns the grouping
|
||||
decision; otherwise the default grouping is used.
|
||||
"""
|
||||
if not vllm_config.model_config.supports_kv_cache_grouping:
|
||||
return get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
if vllm_config.attention_config.hisparse_config is not None:
|
||||
raise ValueError(
|
||||
"A model-declared get_kv_cache_groups cannot be combined "
|
||||
"with HiSparse, which owns its own KV cache grouping. "
|
||||
"Remove one of them."
|
||||
)
|
||||
model_cls, _ = vllm_config.model_config.registry.resolve_model_cls(
|
||||
vllm_config.model_config.architectures,
|
||||
vllm_config.model_config,
|
||||
)
|
||||
return model_cls.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
|
||||
def get_kv_cache_config_from_groups(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
num_blocks: int,
|
||||
) -> KVCacheConfig:
|
||||
"""See ``KVCacheConfigBuilder.get_kv_cache_config_from_groups``."""
|
||||
if len(kv_cache_groups) == 0:
|
||||
# Attention free models do not have KV cache.
|
||||
# Return num_blocks=1 as BlockPool always needs a null_block.
|
||||
return KVCacheConfig(
|
||||
num_blocks=1,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=kv_cache_groups,
|
||||
prefix_cache_retention_interval=(
|
||||
vllm_config.cache_config.prefix_cache_retention_interval
|
||||
),
|
||||
)
|
||||
bytes_per_block = _pool_bytes_per_block(vllm_config, kv_cache_groups)
|
||||
return get_kv_cache_config_from_groups(
|
||||
vllm_config, kv_cache_groups, num_blocks * bytes_per_block
|
||||
)
|
||||
|
||||
@@ -41,12 +41,9 @@ class DPCoordinator:
|
||||
DPEngineCoreProc._has_global_unfinished_reqs method.
|
||||
|
||||
* Broadcasts the START_DP_WAVE message to engines to move them from paused
|
||||
to running state when one engine receives a new request. This can happen
|
||||
in two cases:
|
||||
1) A front-end sending a new request while the engines are paused will
|
||||
concurrently notify the coordinator.
|
||||
2) An engine receiving a request for a stale request wave while in paused
|
||||
state will notify the coordinator.
|
||||
to running state when an idle engine receives a new request and notifies
|
||||
the coordinator. Only the engines start waves: a front-end's view of the
|
||||
running state can be stale, and a paused engine discards START_DP_WAVE.
|
||||
|
||||
Engines will move into running state when receiving a new request or
|
||||
START_DP_WAVE message.
|
||||
@@ -353,27 +350,9 @@ class DPCoordinatorProc:
|
||||
)
|
||||
continue # Skip normal engine notification processing
|
||||
|
||||
# Wave coordination: handle new-request messages from front-end.
|
||||
# Only process these when wave coordination is enabled
|
||||
if self.enable_wave_coordination:
|
||||
# We received a message on the front-end XPUB socket,
|
||||
# from an API server sending a new request while the
|
||||
# engines are paused, so that we can wake the other
|
||||
# engines.
|
||||
engine_to_exclude, wave = decoded
|
||||
if not engines_running:
|
||||
if wave < current_wave:
|
||||
# If the wave number is stale, ensure the message
|
||||
# is handled by all the engines.
|
||||
engine_to_exclude = None
|
||||
|
||||
# engines_running is only set from the engines'
|
||||
# own notifications; a paused engine discards
|
||||
# START_DP_WAVE, so sending it is not evidence
|
||||
# that the engines are running.
|
||||
self._send_start_wave(
|
||||
publish_back, current_wave, engine_to_exclude
|
||||
)
|
||||
logger.error(
|
||||
"DP Coordinator received unexpected message from front-end"
|
||||
)
|
||||
|
||||
if output_back in events:
|
||||
# We received a message from one of the engines.
|
||||
@@ -448,13 +427,12 @@ class DPCoordinatorProc:
|
||||
wave > current_wave
|
||||
or (wave == current_wave and not engines_running)
|
||||
):
|
||||
# 3. The engine received request for a non-current wave
|
||||
# so we must ensure that other engines progress to the
|
||||
# next wave (race condition handling).
|
||||
# 3. An engine started a wave (e.g. an idle engine
|
||||
# received a request), so the others must join it.
|
||||
logger.debug(
|
||||
"Starting wave %d after notification of "
|
||||
"stale wave request from engine.",
|
||||
"Starting wave %d on notification from engine %d.",
|
||||
wave,
|
||||
eng_index,
|
||||
)
|
||||
current_wave = wave
|
||||
engines_running = True
|
||||
|
||||
@@ -2269,15 +2269,14 @@ class DPEngineCoreProc(EngineCoreProc):
|
||||
|
||||
def add_request(self, request: Request, request_wave: int = 0):
|
||||
super().add_request(request, request_wave)
|
||||
if self.has_coordinator and request_wave != self.current_wave:
|
||||
if self.has_coordinator:
|
||||
if request_wave > self.current_wave:
|
||||
self.current_wave = request_wave
|
||||
elif (
|
||||
if (
|
||||
not self.engines_running
|
||||
and self.scheduler.pause_state == PauseState.UNPAUSED
|
||||
):
|
||||
# Request received for an already-completed wave, notify
|
||||
# front-end that we need to start the next one.
|
||||
# Engines alone start waves: have the coordinator wake the others.
|
||||
self.engines_running = True
|
||||
self.output_queue.put_nowait(
|
||||
(-1, EngineCoreOutputs(start_wave=self.current_wave))
|
||||
|
||||
@@ -506,8 +506,8 @@ class BackgroundResources:
|
||||
coordinator: DPCoordinator | None = None
|
||||
output_socket: zmq.Socket | zmq.asyncio.Socket | None = None
|
||||
input_socket: zmq.Socket | zmq.asyncio.Socket | None = None
|
||||
first_req_send_socket: zmq.asyncio.Socket | None = None
|
||||
first_req_rcv_socket: zmq.asyncio.Socket | None = None
|
||||
scale_send_socket: zmq.asyncio.Socket | None = None
|
||||
scale_rcv_socket: zmq.asyncio.Socket | None = None
|
||||
stats_update_socket: zmq.asyncio.Socket | None = None
|
||||
output_queue_task: asyncio.Task | None = None
|
||||
stats_update_task: asyncio.Task | None = None
|
||||
@@ -535,8 +535,8 @@ class BackgroundResources:
|
||||
sockets = (
|
||||
self.output_socket,
|
||||
self.input_socket,
|
||||
self.first_req_send_socket,
|
||||
self.first_req_rcv_socket,
|
||||
self.scale_send_socket,
|
||||
self.scale_rcv_socket,
|
||||
self.stats_update_socket,
|
||||
)
|
||||
|
||||
@@ -1493,9 +1493,9 @@ class DPAsyncMPClient(AsyncMPClient):
|
||||
|
||||
self.eep_scaling_cache: ElasticScalingCache | None = None
|
||||
|
||||
self.first_req_sock_addr = get_open_zmq_inproc_path()
|
||||
self.first_req_send_socket = self.resources.first_req_send_socket = (
|
||||
make_zmq_socket(self.ctx, self.first_req_sock_addr, zmq.PAIR, bind=True)
|
||||
self.scale_sock_addr = get_open_zmq_inproc_path()
|
||||
self.scale_send_socket = self.resources.scale_send_socket = make_zmq_socket(
|
||||
self.ctx, self.scale_sock_addr, zmq.PAIR, bind=True
|
||||
)
|
||||
try:
|
||||
# If we are running in an asyncio event loop, start the stats task.
|
||||
@@ -1518,75 +1518,48 @@ class DPAsyncMPClient(AsyncMPClient):
|
||||
with (
|
||||
make_zmq_socket(self.ctx, stats_addr, zmq.XSUB, linger=0) as socket,
|
||||
make_zmq_socket(
|
||||
self.ctx, self.first_req_sock_addr, zmq.PAIR, bind=False, linger=0
|
||||
) as first_req_rcv_socket,
|
||||
self.ctx, self.scale_sock_addr, zmq.PAIR, bind=False, linger=0
|
||||
) as scale_rcv_socket,
|
||||
):
|
||||
assert isinstance(socket, zmq.asyncio.Socket)
|
||||
assert isinstance(first_req_rcv_socket, zmq.asyncio.Socket)
|
||||
assert isinstance(scale_rcv_socket, zmq.asyncio.Socket)
|
||||
self.resources.stats_update_socket = socket
|
||||
self.resources.first_req_rcv_socket = first_req_rcv_socket
|
||||
self.resources.scale_rcv_socket = scale_rcv_socket
|
||||
# Send subscription message.
|
||||
await socket.send(b"\x01")
|
||||
|
||||
poller = zmq.asyncio.Poller()
|
||||
poller.register(socket, zmq.POLLIN)
|
||||
poller.register(first_req_rcv_socket, zmq.POLLIN)
|
||||
poller.register(scale_rcv_socket, zmq.POLLIN)
|
||||
|
||||
while True:
|
||||
events = await poller.poll()
|
||||
if (
|
||||
not self.engines_running
|
||||
and len(events) == 2
|
||||
or (events[0][0] == first_req_rcv_socket)
|
||||
):
|
||||
# Check if this is a regular request notification or
|
||||
# scale up notification
|
||||
buf = first_req_rcv_socket.recv(flags=zmq.NOBLOCK).result()
|
||||
|
||||
decoded = msgspec.msgpack.decode(buf)
|
||||
if (
|
||||
isinstance(decoded, (list, tuple))
|
||||
and len(decoded) == 2
|
||||
and decoded[0] == "SCALE_ELASTIC_EP"
|
||||
):
|
||||
# Extract new engine count from the decoded message
|
||||
new_engine_count = decoded[1]
|
||||
# Update engine_ranks_managed and count_slice
|
||||
parallel_config = self.vllm_config.parallel_config
|
||||
dp_size = parallel_config.data_parallel_size
|
||||
dp_rank = parallel_config.data_parallel_rank
|
||||
assert dp_rank == 0
|
||||
assert dp_size == new_engine_count
|
||||
assert not parallel_config.data_parallel_hybrid_lb
|
||||
# External LB clients continue to manage only local ranks.
|
||||
if not parallel_config.data_parallel_external_lb:
|
||||
self.engine_ranks_managed = list(range(dp_size))
|
||||
if len(self.lb_engines) < new_engine_count:
|
||||
self.lb_engines += [
|
||||
[0, 0, 0.0]
|
||||
for _ in range(
|
||||
new_engine_count - len(self.lb_engines)
|
||||
)
|
||||
]
|
||||
else:
|
||||
self.lb_engines = self.lb_engines[:new_engine_count]
|
||||
# Send scale up notification to coordinator
|
||||
scale_msg = msgspec.msgpack.encode(
|
||||
("SCALE_ELASTIC_EP", new_engine_count)
|
||||
)
|
||||
await socket.send(scale_msg)
|
||||
continue
|
||||
|
||||
# we're sending a request while the engines are
|
||||
# paused, so that it can wake the others up
|
||||
# (to run dummy EP loop).
|
||||
assert decoded[0] == "FIRST_REQ"
|
||||
target_eng_index = decoded[1]
|
||||
self.engines_running = True
|
||||
msg = msgspec.msgpack.encode(
|
||||
(target_eng_index, self.current_wave)
|
||||
)
|
||||
await socket.send(msg)
|
||||
events = dict(await poller.poll())
|
||||
if scale_rcv_socket in events:
|
||||
buf = scale_rcv_socket.recv(flags=zmq.NOBLOCK).result()
|
||||
msg_type, new_engine_count = msgspec.msgpack.decode(buf)
|
||||
assert msg_type == "SCALE_ELASTIC_EP"
|
||||
# Update engine_ranks_managed and count_slice
|
||||
parallel_config = self.vllm_config.parallel_config
|
||||
dp_size = parallel_config.data_parallel_size
|
||||
dp_rank = parallel_config.data_parallel_rank
|
||||
assert dp_rank == 0
|
||||
assert dp_size == new_engine_count
|
||||
assert not parallel_config.data_parallel_hybrid_lb
|
||||
# External LB clients continue to manage only local ranks.
|
||||
if not parallel_config.data_parallel_external_lb:
|
||||
self.engine_ranks_managed = list(range(dp_size))
|
||||
if len(self.lb_engines) < new_engine_count:
|
||||
self.lb_engines += [
|
||||
[0, 0, 0.0]
|
||||
for _ in range(
|
||||
new_engine_count - len(self.lb_engines)
|
||||
)
|
||||
]
|
||||
else:
|
||||
self.lb_engines = self.lb_engines[:new_engine_count]
|
||||
# Forward the scale notification to the coordinator.
|
||||
await socket.send(buf)
|
||||
continue
|
||||
|
||||
buf = None
|
||||
while True:
|
||||
@@ -1624,14 +1597,9 @@ class DPAsyncMPClient(AsyncMPClient):
|
||||
request.current_wave = self.current_wave
|
||||
request.client_index = self.client_index
|
||||
|
||||
# The engine that receives it wakes the other ranks if they are idle.
|
||||
chosen_engine = self.get_core_engine_for_request(request)
|
||||
to_await = self._send_input(EngineCoreRequestType.ADD, request, chosen_engine)
|
||||
if not self.engines_running:
|
||||
# Notify coordinator that we're sending a request
|
||||
req_msg = msgspec.msgpack.encode(("FIRST_REQ", chosen_engine))
|
||||
await self.first_req_send_socket.send(req_msg)
|
||||
|
||||
await to_await
|
||||
await self._send_input(EngineCoreRequestType.ADD, request, chosen_engine)
|
||||
|
||||
self._ensure_output_queue_task()
|
||||
|
||||
@@ -2100,7 +2068,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
scale_up_marker = msgspec.msgpack.encode(
|
||||
("SCALE_ELASTIC_EP", new_data_parallel_size)
|
||||
)
|
||||
await self.first_req_send_socket.send(scale_up_marker)
|
||||
await self.scale_send_socket.send(scale_up_marker)
|
||||
|
||||
logger.info(
|
||||
"[Elastic EP] Scale up completed, new data parallel size: %s",
|
||||
@@ -2176,7 +2144,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
scale_down_marker = msgspec.msgpack.encode(
|
||||
("SCALE_ELASTIC_EP", new_data_parallel_size)
|
||||
)
|
||||
await self.first_req_send_socket.send(scale_down_marker)
|
||||
await self.scale_send_socket.send(scale_down_marker)
|
||||
await wait_future
|
||||
await self.resume_scheduler_async()
|
||||
except Exception:
|
||||
|
||||
@@ -483,24 +483,24 @@ class TopKTopPSampler(nn.Module):
|
||||
return xpu_sample(logits, k, p, self.logprobs_mode)
|
||||
|
||||
|
||||
# Note: this is a workaround for
|
||||
# https://github.com/pytorch/pytorch/pull/151218
|
||||
@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend)
|
||||
def compiled_random_sample(logits: torch.Tensor) -> torch.Tensor:
|
||||
probs = logits.softmax(dim=-1, dtype=torch.float32)
|
||||
q = torch.empty_like(probs)
|
||||
q.exponential_()
|
||||
return probs.div(q).argmax(dim=-1).view(-1)
|
||||
|
||||
|
||||
def apply_top_k_top_p(
|
||||
logits: torch.Tensor, k: torch.Tensor | None, p: torch.Tensor | None
|
||||
logits: torch.Tensor,
|
||||
k: torch.Tensor | None,
|
||||
p: torch.Tensor | None,
|
||||
has_topp_only_rows: bool = True,
|
||||
) -> torch.Tensor:
|
||||
"""Apply top-k and/or top-p masking to logits.
|
||||
|
||||
has_topp_only_rows must be True if any row has p < 1 and
|
||||
k >= logits.shape[-1] (no active top-k).
|
||||
"""
|
||||
if p is None and k is None:
|
||||
return logits
|
||||
|
||||
if HAS_TRITON:
|
||||
return apply_top_k_top_p_triton(logits, k, p)
|
||||
return apply_top_k_top_p_triton(
|
||||
logits, k, p, has_topp_only_rows=has_topp_only_rows
|
||||
)
|
||||
|
||||
is_cpu = current_platform.is_cpu()
|
||||
return apply_top_k_top_p_pytorch(logits, k, p, allow_cpu_sync=is_cpu)
|
||||
|
||||
@@ -987,13 +987,19 @@ _SPLIT_MAX_BATCH = 64
|
||||
_SPLIT_MAX_SPLITS = 32 # power of two
|
||||
_SPLIT_FANOUT = 8
|
||||
_SPLIT_ROUNDS = 5
|
||||
# The step kernels are latency-bound at one program per SM, so keep splitting
|
||||
# past that while slices span at least this many entries; shorter slices no
|
||||
# longer amortize each program's row stats and combine replay.
|
||||
_SPLIT_MIN_SLICE = 7168
|
||||
|
||||
_TRITON_SPLIT_CACHE: dict[torch.device, dict[str, torch.Tensor]] = {}
|
||||
|
||||
|
||||
def _topp_split_count(batch_size: int, num_sm: int) -> int:
|
||||
def _topp_split_count(batch_size: int, vocab_size: int, num_sm: int) -> int:
|
||||
splits = 1
|
||||
while splits < _SPLIT_MAX_SPLITS and splits * 2 * batch_size <= num_sm:
|
||||
while splits < _SPLIT_MAX_SPLITS and (
|
||||
splits * 2 * batch_size <= num_sm or splits * 2 * _SPLIT_MIN_SLICE <= vocab_size
|
||||
):
|
||||
splits *= 2
|
||||
return splits
|
||||
|
||||
@@ -1079,7 +1085,7 @@ def _topp_split_stats(
|
||||
num_sm: int,
|
||||
) -> DispatchSpec:
|
||||
batch_size, vocab_size = logits.shape
|
||||
splits = _topp_split_count(batch_size, num_sm)
|
||||
splits = _topp_split_count(batch_size, vocab_size, num_sm)
|
||||
return (logits.shape[0] * splits,), dict(
|
||||
K=k if k is not None else logits,
|
||||
HAS_K=k is not None,
|
||||
@@ -1373,7 +1379,7 @@ def _topp_split_step(
|
||||
num_sm: int,
|
||||
) -> DispatchSpec:
|
||||
batch_size, vocab_size = logits.shape
|
||||
splits = _topp_split_count(batch_size, num_sm)
|
||||
splits = _topp_split_count(batch_size, vocab_size, num_sm)
|
||||
return (logits.shape[0] * splits,), dict(
|
||||
K=k if k is not None else logits,
|
||||
HAS_K=k is not None,
|
||||
@@ -1513,6 +1519,7 @@ def _topp_sb_mask_kernel(
|
||||
cum = tl.cumsum(dmask.to(tl.int32)).to(tl.float32) + num_preceding
|
||||
keep_dup = dmask & (cum <= numkeep)
|
||||
keep = keep & (~dmask | keep_dup)
|
||||
num_preceding += tl.sum(dmask.to(tl.float32))
|
||||
tl.store(ROW + i + offs, tl.where(keep, x, MASK_VALUE), mask=mask_n)
|
||||
|
||||
|
||||
@@ -1554,7 +1561,7 @@ def _topp_split_mask(
|
||||
num_sm: int,
|
||||
) -> DispatchSpec:
|
||||
batch_size, vocab_size = logits.shape
|
||||
splits = _topp_split_count(batch_size, num_sm)
|
||||
splits = _topp_split_count(batch_size, vocab_size, num_sm)
|
||||
return (logits.shape[0] * splits,), dict(
|
||||
K=k if k is not None else logits,
|
||||
HAS_K=k is not None,
|
||||
@@ -1618,6 +1625,7 @@ def apply_top_k_top_p_triton(
|
||||
k: torch.Tensor | None,
|
||||
p: torch.Tensor | None,
|
||||
mask_value: float = float("-inf"),
|
||||
has_topp_only_rows: bool = True,
|
||||
) -> torch.Tensor:
|
||||
"""Apply combined top-k and top-p masking using Triton.
|
||||
|
||||
@@ -1632,6 +1640,9 @@ def apply_top_k_top_p_triton(
|
||||
p: [batch_size] float32 tensor of top-p values per row (0 to 1),
|
||||
or None to disable top-p
|
||||
mask_value: Value for masked positions (default: -inf)
|
||||
has_topp_only_rows: Whether any row has p < 1 and k >= vocab_size.
|
||||
False lets small batches skip the split top-p pipeline; pass it
|
||||
only if no such row exists, as such a row would get no top-p.
|
||||
|
||||
Returns:
|
||||
The masked logits tensor. It may or may not be modified in-place.
|
||||
@@ -1714,7 +1725,7 @@ def apply_top_k_top_p_triton(
|
||||
mask_value,
|
||||
num_sm,
|
||||
)
|
||||
if use_split:
|
||||
if use_split and has_topp_only_rows:
|
||||
_apply_topp_split(logits, k_ptr, p_ptr, mask_value, num_sm)
|
||||
|
||||
return logits
|
||||
|
||||
@@ -151,6 +151,7 @@ class SimpleCPUOffloadScheduler:
|
||||
lazy_offload: bool = False,
|
||||
disk_capacity_bytes: int = 0,
|
||||
use_page_cache: bool = False,
|
||||
num_cpu_blocks: int | None = None,
|
||||
):
|
||||
self.vllm_config = vllm_config
|
||||
self.kv_cache_config = kv_cache_config
|
||||
@@ -170,7 +171,7 @@ class SimpleCPUOffloadScheduler:
|
||||
# Derive a CPU KVCacheConfig from the GPU config and build a coordinator
|
||||
assert kv_cache_config is not None
|
||||
self.cpu_kv_cache_config = self._derive_cpu_config(
|
||||
kv_cache_config, offload_capacity
|
||||
kv_cache_config, offload_capacity, num_cpu_blocks
|
||||
)
|
||||
self.num_cpu_blocks = self.cpu_kv_cache_config.num_blocks
|
||||
self.prefix_cacheable_group_ids = (
|
||||
@@ -278,7 +279,9 @@ class SimpleCPUOffloadScheduler:
|
||||
|
||||
@staticmethod
|
||||
def _derive_cpu_config(
|
||||
gpu_config: "KVCacheConfig", cpu_capacity_bytes: int
|
||||
gpu_config: "KVCacheConfig",
|
||||
cpu_capacity_bytes: int,
|
||||
num_cpu_blocks: int | None = None,
|
||||
) -> "KVCacheConfig":
|
||||
"""Derive a CPU KVCacheConfig from the GPU config.
|
||||
Same kv_cache_groups, num_blocks scaled by CPU/GPU memory ratio."""
|
||||
@@ -291,7 +294,16 @@ class SimpleCPUOffloadScheduler:
|
||||
# so its size is the total GPU KV cache size.
|
||||
gpu_total_bytes = gpu_config.kv_cache_tensors[0].size
|
||||
num_gpu_blocks = gpu_config.num_blocks
|
||||
num_cpu_blocks = max(1, num_gpu_blocks * cpu_capacity_bytes // gpu_total_bytes)
|
||||
local_capacity = num_gpu_blocks * cpu_capacity_bytes // gpu_total_bytes
|
||||
if num_cpu_blocks is None:
|
||||
num_cpu_blocks = max(1, local_capacity)
|
||||
elif (
|
||||
type(num_cpu_blocks) is not int or not 0 < num_cpu_blocks <= local_capacity
|
||||
):
|
||||
raise ValueError(
|
||||
f"num_cpu_blocks={num_cpu_blocks!r} must be between 1 and "
|
||||
f"local offload capacity {local_capacity}"
|
||||
)
|
||||
# Create CPU kv_cache_tensors mirroring GPU by scaling size proportionally.
|
||||
cpu_tensors = [
|
||||
KVCacheTensor(
|
||||
|
||||
@@ -36,10 +36,12 @@ class SimpleCPUOffloadWorker:
|
||||
disk_capacity_bytes: int = 0,
|
||||
disk_buffer_slots: int = 2,
|
||||
use_page_cache: bool = False,
|
||||
num_cpu_blocks: int | None = None,
|
||||
):
|
||||
self.vllm_config = vllm_config
|
||||
self.kv_cache_config = kv_cache_config
|
||||
self.cpu_capacity_bytes = cpu_capacity_bytes
|
||||
self._num_cpu_blocks_override = num_cpu_blocks
|
||||
self.disk_path = disk_path
|
||||
self.disk_capacity_bytes = disk_capacity_bytes
|
||||
self.disk_buffer_slots = disk_buffer_slots
|
||||
@@ -159,7 +161,10 @@ class SimpleCPUOffloadWorker:
|
||||
]
|
||||
total_bytes_per_block = sum(per_tensor_bpb)
|
||||
|
||||
self.num_cpu_blocks = max(1, self.cpu_capacity_bytes // total_bytes_per_block)
|
||||
self.num_cpu_blocks = self._resolve_num_cpu_blocks(
|
||||
self.disk_capacity_bytes if self.disk_mode else self.cpu_capacity_bytes,
|
||||
total_bytes_per_block,
|
||||
)
|
||||
|
||||
# Use lowest priority so KV cache I/O yields to compute streams.
|
||||
low_pri, _ = torch.cuda.Stream.priority_range()
|
||||
@@ -173,13 +178,27 @@ class SimpleCPUOffloadWorker:
|
||||
else:
|
||||
self._init_cpu_mode(unique_gpu_caches, total_bytes_per_block, self.device)
|
||||
|
||||
def _resolve_num_cpu_blocks(self, capacity_bytes: int, bytes_per_block: int) -> int:
|
||||
local_capacity = capacity_bytes // bytes_per_block
|
||||
requested = self._num_cpu_blocks_override
|
||||
if requested is None:
|
||||
return max(1, local_capacity)
|
||||
if type(requested) is not int or not 0 < requested <= local_capacity:
|
||||
raise ValueError(
|
||||
f"num_cpu_blocks={requested!r} must be between 1 and "
|
||||
f"local offload capacity {local_capacity}"
|
||||
)
|
||||
return requested
|
||||
|
||||
def _init_disk_mode(
|
||||
self,
|
||||
unique_gpu_caches: dict[str, torch.Tensor],
|
||||
total_bytes_per_block: int,
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
num_disk_slots = max(1, self.disk_capacity_bytes // total_bytes_per_block)
|
||||
num_disk_slots = self._resolve_num_cpu_blocks(
|
||||
self.disk_capacity_bytes, total_bytes_per_block
|
||||
)
|
||||
self.num_cpu_blocks = num_disk_slots
|
||||
|
||||
logger.info(
|
||||
|
||||
@@ -58,6 +58,7 @@ class GPUWatermarkSampler(Sampler):
|
||||
pos: torch.Tensor,
|
||||
top_k: torch.Tensor | None,
|
||||
top_p: torch.Tensor | None,
|
||||
has_topp_only_rows: bool,
|
||||
use_fused_sampler: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
enabled = self.watermarking.np[idx_mapping_np] & (
|
||||
@@ -71,10 +72,13 @@ class GPUWatermarkSampler(Sampler):
|
||||
pos,
|
||||
top_k,
|
||||
top_p,
|
||||
has_topp_only_rows,
|
||||
use_fused_sampler,
|
||||
)
|
||||
|
||||
processed_logits = apply_top_k_top_p(processed_logits, top_k, top_p)
|
||||
processed_logits = apply_top_k_top_p(
|
||||
processed_logits, top_k, top_p, has_topp_only_rows=has_topp_only_rows
|
||||
)
|
||||
contexts = self._get_contexts(expanded_idx_mapping)
|
||||
repeated_contexts = None
|
||||
if self.deduplicate_contexts != "none":
|
||||
|
||||
@@ -1036,26 +1036,22 @@ def _extrapolate_full_graph_memory(mem_samples: list[int], total_graphs: int) ->
|
||||
|
||||
def _init_minimal_kv_cache_for_profiling(runner: "GPUModelRunner") -> None:
|
||||
"""Allocate the smallest KV cache that still lets every graph be captured."""
|
||||
from vllm.v1.core.kv_cache_planning import (
|
||||
get_kv_cache_config_from_groups,
|
||||
get_kv_cache_groups,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_config_builder import get_kv_cache_config_builder
|
||||
from vllm.v1.core.kv_cache_planning import get_kv_cache_groups_from_builder
|
||||
|
||||
kv_cache_spec = runner.get_kv_cache_spec()
|
||||
kv_cache_groups = get_kv_cache_groups(runner.vllm_config, kv_cache_spec)
|
||||
builder = get_kv_cache_config_builder(runner.vllm_config)
|
||||
kv_cache_groups = get_kv_cache_groups_from_builder(
|
||||
builder, runner.vllm_config, kv_cache_spec
|
||||
)
|
||||
# At least one block per sequence is required to capture the graphs.
|
||||
min_blocks = (
|
||||
min(runner.max_num_reqs, runner.compilation_config.max_cudagraph_capture_size)
|
||||
or 1
|
||||
)
|
||||
saved_override = runner.cache_config.num_gpu_blocks_override
|
||||
runner.cache_config.num_gpu_blocks_override = min_blocks
|
||||
try:
|
||||
minimal_config = get_kv_cache_config_from_groups(
|
||||
runner.vllm_config, kv_cache_groups, available_memory=0
|
||||
)
|
||||
finally:
|
||||
runner.cache_config.num_gpu_blocks_override = saved_override
|
||||
minimal_config = builder.get_kv_cache_config_from_groups(
|
||||
runner.vllm_config, kv_cache_groups, num_blocks=min_blocks
|
||||
)
|
||||
|
||||
runner.initialize_kv_cache(minimal_config, is_profiling=True)
|
||||
runner.cache_config.num_gpu_blocks = minimal_config.num_blocks
|
||||
|
||||
@@ -298,8 +298,8 @@ class Sampler:
|
||||
seq_lens_upper_bound_np,
|
||||
skip_top_k_top_p=True,
|
||||
)
|
||||
top_k, top_p = self.sampling_states.get_top_k_top_p(
|
||||
expanded_idx_mapping, idx_mapping_np
|
||||
top_k, top_p, has_topp_only_rows = self.sampling_states.get_top_k_top_p(
|
||||
expanded_idx_mapping, idx_mapping_np, logits.shape[-1]
|
||||
)
|
||||
# Don't use a fused sampler if no requests use top_k/top_p, if there are
|
||||
# any greedy requests or per-request seeds, or if post-processed
|
||||
@@ -321,6 +321,7 @@ class Sampler:
|
||||
pos,
|
||||
top_k,
|
||||
top_p,
|
||||
has_topp_only_rows,
|
||||
use_fused_sampler,
|
||||
)
|
||||
|
||||
@@ -332,6 +333,7 @@ class Sampler:
|
||||
pos: torch.Tensor,
|
||||
top_k: torch.Tensor | None,
|
||||
top_p: torch.Tensor | None,
|
||||
has_topp_only_rows: bool,
|
||||
use_fused_sampler: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if use_fused_sampler:
|
||||
@@ -342,7 +344,9 @@ class Sampler:
|
||||
else: # Use XPU sampler
|
||||
sampled, _ = xpu_sample(processed_logits, top_k, top_p)
|
||||
else:
|
||||
processed_logits = apply_top_k_top_p(processed_logits, top_k, top_p)
|
||||
processed_logits = apply_top_k_top_p(
|
||||
processed_logits, top_k, top_p, has_topp_only_rows=has_topp_only_rows
|
||||
)
|
||||
sampled = gumbel_sample(
|
||||
processed_logits,
|
||||
expanded_idx_mapping,
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user