[AuxOutput] Add block-keyed storage for routed-expert outputs (#45635)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Hongxin Xu
2026-09-21 10:25:05 +00:00
committed by GitHub
co-authored by aoshen02 OpenAI Codex Roger Wang Nick Hill Codex
parent 0aee727ff6
commit 0b7f11a1ee
32 changed files with 3604 additions and 877 deletions
+119
View File
@@ -0,0 +1,119 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from typing import Any
import pytest
from vllm.config import AuxOutputConfig, VllmConfig
from vllm.engine.arg_utils import EngineArgs
pytestmark = pytest.mark.cpu_test
def _config(
*,
use_v2: bool = True,
pp: int = 1,
dcp: int = 1,
pcp: int = 1,
connector: str | None = None,
runner_type: str = "generate",
is_moe: bool = True,
sliding_window: int | None = None,
attention_chunk_size: int | None = None,
enable_prefix_caching: bool = True,
adaptive_verification: bool = False,
):
return SimpleNamespace(
model_config=SimpleNamespace(
runner_type=runner_type,
is_moe=is_moe,
get_sliding_window=lambda: sliding_window,
attention_chunk_size=attention_chunk_size,
),
use_v2_model_runner=use_v2,
parallel_config=SimpleNamespace(
pipeline_parallel_size=pp,
decode_context_parallel_size=dcp,
prefill_context_parallel_size=pcp,
),
aux_output_config=AuxOutputConfig(enable_return_routed_experts=True),
cache_config=SimpleNamespace(
enable_prefix_caching=enable_prefix_caching,
),
speculative_config=(
SimpleNamespace(enable_adaptive_verification=True)
if adaptive_verification
else None
),
kv_transfer_config=(
None
if connector is None
else SimpleNamespace(
is_kv_transfer_instance=True,
kv_connector=connector,
)
),
)
def test_aux_output_config_defaults():
config = AuxOutputConfig()
assert not config.enabled
assert not config.enable_return_routed_experts
assert config.max_bytes is None
def test_aux_output_capture_changes_compilation_hash():
disabled = AuxOutputConfig()
enabled = AuxOutputConfig(enable_return_routed_experts=True)
assert disabled.compute_hash() != enabled.compute_hash()
def test_legacy_routed_experts_flag_updates_aux_output_config():
args = EngineArgs(enable_return_routed_experts=True)
assert args.aux_output_config.enabled
assert args.aux_output_config.enable_return_routed_experts
@pytest.mark.parametrize(
("kwargs", "error"),
[
({"use_v2": False}, "requires Model Runner V2"),
({"runner_type": "pooling"}, "only supports generate runners"),
({"is_moe": False}, "only supports MoE models"),
({"enable_prefix_caching": False}, "requires prefix caching"),
(
{"adaptive_verification": True},
"adaptive speculative verification",
),
({"pp": 2}, "pipeline parallelism"),
({"dcp": 2}, "context parallelism"),
({"pcp": 2}, "context parallelism"),
({"connector": "MooncakeConnector"}, "incompatible with KV connectors"),
],
)
def test_aux_output_connector_rejects_unsupported_configuration(kwargs, error):
with pytest.raises(ValueError, match=error):
VllmConfig._verify_aux_output_compatibility(_config(**kwargs))
@pytest.mark.parametrize(
"kwargs",
[{"sliding_window": 4096}, {"attention_chunk_size": 4096}],
)
def test_aux_output_config_defers_attention_layout_to_kv_config(kwargs):
VllmConfig._verify_aux_output_compatibility(_config(**kwargs))
def test_aux_output_guards_are_inactive_when_capture_is_disabled():
config: Any = VllmConfig.__new__(VllmConfig)
config.model_config = SimpleNamespace()
config.aux_output_config = AuxOutputConfig()
config._verify_aux_output_compatibility()
File diff suppressed because it is too large Load Diff
@@ -12,15 +12,13 @@ from ...utils import RemoteOpenAIServer
MODEL_NAME = "TitanML/tiny-mixtral"
# tiny-mixtral config: 8 local experts, top-2 routing, 2 hidden layers.
# The published config has sliding_window=4096, which produces
# SlidingWindowSpec kv-cache groups; RoutedExpertsManager requires a
# FullAttentionSpec group, so we override sliding_window=null below.
# Its published sliding window is incompatible with AuxOutput retention.
NUM_LOCAL_EXPERTS = 8
NUM_EXPERTS_PER_TOK = 2
NUM_HIDDEN_LAYERS = 2
def assert_valid_routed_experts(encoded: str | None) -> None:
def decode_routed_experts(encoded: str | None) -> np.ndarray:
assert encoded is not None
routed_experts = np.load(io.BytesIO(base64.b64decode(encoded)))
assert routed_experts.ndim == 3
@@ -30,6 +28,7 @@ def assert_valid_routed_experts(encoded: str | None) -> None:
assert topk == NUM_EXPERTS_PER_TOK
assert (routed_experts >= 0).all()
assert (routed_experts < NUM_LOCAL_EXPERTS).all()
return routed_experts
@pytest.fixture(scope="module")
@@ -44,7 +43,11 @@ def server():
"--hf-overrides",
'{"sliding_window": null}',
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
with RemoteOpenAIServer(
MODEL_NAME,
args,
env_dict={"VLLM_USE_V2_MODEL_RUNNER": "1"},
) as remote_server:
yield remote_server
@@ -63,4 +66,8 @@ async def test_routed_experts(server):
choice = result.model_dump()["choices"][0]
assert choice["token_ids"] is not None
assert_valid_routed_experts(choice["routed_experts"])
assert result.usage is not None
routed_experts = decode_routed_experts(choice["routed_experts"])
assert len(routed_experts) == (
result.usage.prompt_tokens + len(choice["token_ids"]) - 1
)
@@ -15,9 +15,7 @@ MODEL_NAME = "TitanML/tiny-mixtral"
GEN_ENDPOINT = "/inference/v1/generate"
# tiny-mixtral config: 8 local experts, top-2 routing, 2 hidden layers.
# The published config has sliding_window=4096, which produces
# SlidingWindowSpec kv-cache groups; RoutedExpertsManager requires a
# FullAttentionSpec group, so we override sliding_window=null below.
# Its published sliding window is incompatible with AuxOutput retention.
NUM_LOCAL_EXPERTS = 8
NUM_EXPERTS_PER_TOK = 2
NUM_HIDDEN_LAYERS = 2
@@ -36,7 +34,13 @@ def server():
'{"sliding_window": null}',
"--enable-scale-out",
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
with RemoteOpenAIServer(
MODEL_NAME,
args,
env_dict={
"VLLM_USE_V2_MODEL_RUNNER": "1",
},
) as remote_server:
yield remote_server
@@ -56,9 +60,10 @@ async def client(server: RemoteOpenAIServer):
@pytest.mark.asyncio
async def test_generate_routed_experts(client):
"""Test that /inference/v1/generate returns routed_experts when enabled."""
prompt_token_ids = [1, 2, 3]
payload = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"token_ids": prompt_token_ids,
"sampling_params": {"max_tokens": 10, "temperature": 0.0},
"stream": False,
}
@@ -80,3 +85,4 @@ async def test_generate_routed_experts(client):
assert topk == NUM_EXPERTS_PER_TOK
assert (routed_experts >= 0).all()
assert (routed_experts < NUM_LOCAL_EXPERTS).all()
assert num_tokens == len(prompt_token_ids) + len(choice["token_ids"]) - 1
@@ -7,26 +7,19 @@ from unittest.mock import Mock, patch
import pytest
import torch
from vllm.config import ModelConfig, VllmConfig
from vllm.config import VllmConfig
from vllm.config.compilation import CompilationMode
from vllm.distributed.eplb.eplb_state import EplbLayerState
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
RoutedExpertsCapturer,
RoutedExpertsManager,
bind_routed_experts_capturer,
get_routed_experts_attn_gid,
)
from vllm.model_executor.layers.fused_moe.router.base_router import BaseRouter
from vllm.transformers_utils.model_arch_config_convertor import (
ModelArchConfigConvertorBase,
)
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
MLAAttentionSpec,
SlidingWindowMLASpec,
UniformTypeKVCacheSpecs,
)
@@ -42,6 +35,7 @@ def _capturer_with_buffer(
num_experts_per_tok: int = 2,
dp_rank: int = 0,
tp_size: int = 1,
dtype: torch.dtype = torch.int32,
) -> RoutedExpertsCapturer:
# Bypass __init__ so the test can use a CPU buffer and skip the
# VllmConfig dependency. The CUDA device-tensor allocation in the
@@ -49,10 +43,11 @@ def _capturer_with_buffer(
c = RoutedExpertsCapturer.__new__(RoutedExpertsCapturer)
c.dp_rank = dp_rank
c.tp_size = tp_size
c.output_dtype = torch.uint8
c.device_buffer = torch.full(
(max_tokens, num_layers, num_experts_per_tok),
-1,
dtype=torch.int32,
dtype=dtype,
)
return c
@@ -88,74 +83,36 @@ def _make_modular_routed_experts():
)
def _make_model_config(hf_config):
num_experts_per_token = ModelArchConfigConvertorBase(
hf_config, hf_config
).get_num_experts_per_token()
model_config = SimpleNamespace(
hf_text_config=hf_config,
model_arch_config=SimpleNamespace(
num_experts_per_token=num_experts_per_token,
def _full_attention_kv_group(
spec_type: type[FullAttentionSpec] = FullAttentionSpec,
) -> KVCacheGroupSpec:
full_attention = spec_type(
block_size=16,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
)
return KVCacheGroupSpec(
["layer"],
UniformTypeKVCacheSpecs(
block_size=16,
kv_cache_specs={"layer": full_attention},
),
)
model_config.get_num_experts = lambda: hf_config.num_experts
model_config.get_num_experts_per_tok = lambda: (
ModelConfig.get_num_experts_per_tok(model_config)
)
model_config.get_total_num_hidden_layers = lambda: hf_config.num_hidden_layers
return model_config
def test_routed_experts_manager_uses_gemma4_top_k_experts():
hf_config = SimpleNamespace(
num_experts=8,
top_k_experts=2,
num_hidden_layers=3,
)
vllm_config = SimpleNamespace(model_config=_make_model_config(hf_config))
kv_cache_spec = FullAttentionSpec(
block_size=4,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
)
kv_cache_config = KVCacheConfig(
num_blocks=2,
kv_cache_tensors=[],
kv_cache_groups=[KVCacheGroupSpec(["layer"], kv_cache_spec)],
)
@pytest.mark.parametrize("eplb_enabled", [False, True])
def test_base_router_capture_pre_eplb_mapping(eplb_enabled):
eplb_state = None
if eplb_enabled:
eplb_state = EplbLayerState()
eplb_state.expert_load_view = torch.zeros(32, dtype=torch.int64)
eplb_state.logical_to_physical_map = torch.arange(32).view(32, 1)
eplb_state.logical_replica_count = torch.ones(32, dtype=torch.int64)
eplb_state.should_record_tensor = torch.ones((), dtype=torch.bool)
eplb_state.num_unpadded_tokens_tensors = [torch.tensor(0, dtype=torch.int32)]
router = _make_router(eplb_state)
manager = RoutedExpertsManager(vllm_config, kv_cache_config)
assert manager.routed_experts_by_slot.shape == (8, 3, 2)
def test_routed_experts_manager_uses_kimi_k3_experts_per_token():
hf_config = SimpleNamespace(
num_experts=8,
num_experts_per_token=2,
num_hidden_layers=3,
)
vllm_config = SimpleNamespace(model_config=_make_model_config(hf_config))
kv_cache_spec = FullAttentionSpec(
block_size=4,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
)
kv_cache_config = KVCacheConfig(
num_blocks=2,
kv_cache_tensors=[],
kv_cache_groups=[KVCacheGroupSpec(["layer"], kv_cache_spec)],
)
manager = RoutedExpertsManager(vllm_config, kv_cache_config)
assert manager.routed_experts_by_slot.shape == (8, 3, 2)
def test_base_router_capture_pre_eplb_mapping():
router = _make_router()
captured = []
def capture_fn(ids):
@@ -173,34 +130,7 @@ def test_base_router_capture_pre_eplb_mapping():
assert torch.equal(topk_ids, torch.tensor([[11, 12], [13, 14]]))
def test_base_router_capture_with_eplb_enabled():
eplb_state = EplbLayerState()
eplb_state.expert_load_view = torch.zeros(32, dtype=torch.int64)
eplb_state.logical_to_physical_map = torch.arange(32).view(32, 1)
eplb_state.logical_replica_count = torch.ones(32, dtype=torch.int64)
eplb_state.should_record_tensor = torch.ones((), dtype=torch.bool)
eplb_state.num_unpadded_tokens_tensors = [torch.tensor(0, dtype=torch.int32)]
router = _make_router(eplb_state=eplb_state)
captured = []
def capture_fn(ids):
captured.append(ids.clone())
router.set_capture_fn(capture_fn)
_, topk_ids = router.select_experts(
hidden_states=torch.empty(1),
router_logits=torch.empty(1),
)
assert len(captured) == 1
# Capture should see logical ids pre-EPLB mapping.
assert torch.equal(captured[0], torch.tensor([[1, 2], [3, 4]]))
# Our DummyRouter mapping adds +10.
assert torch.equal(topk_ids, torch.tensor([[11, 12], [13, 14]]))
def test_public_binding_only_visits_target_model(monkeypatch):
def test_public_binding_binds_target_model_router(monkeypatch):
class DummyFusedMoE:
def __init__(self, layer_id):
self.layer_id = layer_id
@@ -208,7 +138,6 @@ def test_public_binding_only_visits_target_model(monkeypatch):
self._quant_method = _make_modular_routed_experts().quant_method
target_module = DummyFusedMoE(layer_id=7)
draft_module = DummyFusedMoE(layer_id=0)
import vllm.model_executor.layers.fused_moe.layer as fused_moe_layer
@@ -221,12 +150,30 @@ def test_public_binding_only_visits_target_model(monkeypatch):
)
assert target_module.router.capture_fn is not None
assert draft_module.router.capture_fn is None
topk_ids = torch.tensor([[5, 6]])
target_module.router.capture_fn(topk_ids)
assert calls == [(7, topk_ids)]
def test_public_binding_supports_direct_capture_source():
class DirectCaptureSource(torch.nn.Module):
def __init__(self):
super().__init__()
self.layer_id = 3
self.capture_fn = None
source = DirectCaptureSource()
calls = []
capturer = types.SimpleNamespace(capture=lambda *args: calls.append(args))
bind_routed_experts_capturer(source, capturer)
assert source.capture_fn is not None
topk_ids = torch.tensor([[1, 2]])
source.capture_fn(topk_ids)
assert calls == [(3, topk_ids)]
def test_public_binding_rejects_monolithic_without_replay_support(monkeypatch):
class DummyFusedMoE:
def __init__(self):
@@ -277,6 +224,51 @@ def test_routed_experts_capturer_single_dp_no_metadata():
assert capturer.device_buffer[3, 0, 0].item() == -1
@pytest.mark.parametrize(
("num_experts", "dtype_name", "torch_dtype"),
[(256, "uint8", torch.uint8), (257, "uint16", torch.uint16)],
)
def test_routed_experts_capturer_exposes_output_profile(
monkeypatch, num_experts, dtype_name, torch_dtype
):
import vllm.model_executor.layers.fused_moe.routed_experts_capturer as module
monkeypatch.setattr(module, "current_platform", SimpleNamespace(device_type="cpu"))
config = SimpleNamespace(
model_config=SimpleNamespace(
get_total_num_hidden_layers=lambda: 3,
get_num_experts=lambda: num_experts,
get_num_experts_per_tok=lambda: 2,
hf_text_config=SimpleNamespace(model_type="test"),
),
parallel_config=SimpleNamespace(data_parallel_rank=0, tensor_parallel_size=1),
)
capturer = RoutedExpertsCapturer(8, config)
assert capturer.shape_per_token == (3, 2)
assert capturer.output_dtype_name == dtype_name
assert capturer.output_dtype == torch_dtype
assert capturer.device_buffer.shape == (8, 3, 2)
@pytest.mark.parametrize("output_dtype", [torch.uint8, torch.uint16])
def test_routed_experts_capturer_narrows_snapshot(output_dtype):
capturer = _capturer_with_buffer(dtype=torch.int32)
capturer.output_dtype = output_dtype
topk = torch.tensor([[1, 2], [254, 255]], dtype=torch.int64)
ctx = SimpleNamespace(dp_metadata=None)
with patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx):
capturer.capture(layer_id=0, topk_ids=topk)
output = capturer.snapshot_routing_data(2)
assert capturer.device_buffer.dtype == torch.int32
assert capturer.device_buffer[:2, 0, :].tolist() == topk.tolist()
assert output.dtype == output_dtype
assert output[:, 0, :].tolist() == topk.tolist()
assert output.data_ptr() != capturer.device_buffer.data_ptr()
def test_routed_experts_capturer_dp_naive_concatenated_all_ranks():
"""N == sum(num_tokens_dp): slice this rank's segment from concatenated topk."""
capturer = _capturer_with_buffer(dp_rank=1)
@@ -307,6 +299,47 @@ def test_routed_experts_capturer_dp_modular_local_tokens():
assert torch.equal(capturer.device_buffer[:3, 0, :], topk)
def test_routed_experts_capturer_dp_ep_gathered_shards():
capturer = _capturer_with_buffer(dp_rank=1, tp_size=2)
num_tokens_dp = torch.tensor([3, 2], dtype=torch.int32)
ctx = SimpleNamespace(
dp_metadata=SimpleNamespace(
num_tokens_across_dp_cpu=num_tokens_dp,
local_sizes=[2, 2, 1, 1],
)
)
topk = torch.tensor(
[[0, 1], [2, 3], [4, 5], [-1, -1], [10, 11], [12, 13]],
dtype=torch.int32,
)
with patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx):
capturer.capture(layer_id=0, topk_ids=topk)
assert torch.equal(capturer.device_buffer[:2, 0], topk[4:])
def test_routed_experts_capturer_sp_modular_gathers_tp_shards():
capturer = _capturer_with_buffer(dp_rank=1, tp_size=2)
num_tokens_dp = torch.tensor([2, 3], dtype=torch.int32)
ctx = SimpleNamespace(
dp_metadata=SimpleNamespace(num_tokens_across_dp_cpu=num_tokens_dp)
)
local_shard = torch.tensor([[10, 11], [12, 13]], dtype=torch.int32)
gathered = torch.tensor([[10, 11], [12, 13], [14, 15], [-1, -1]], dtype=torch.int32)
tp_group = Mock()
tp_group.all_gather.return_value = gathered
with (
patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx),
patch(f"{_REC_MODULE}.get_tp_group", return_value=tp_group),
):
capturer.capture(layer_id=0, topk_ids=local_shard)
tp_group.all_gather.assert_called_once_with(local_shard, dim=0)
assert torch.equal(capturer.device_buffer[:3, 0], gathered[:3])
def test_routed_experts_capturer_dp_unexpected_batch_raises():
"""Mismatch between topk batch dim and DP layout: fail fast."""
capturer = _capturer_with_buffer(dp_rank=0)
@@ -324,128 +357,119 @@ def test_routed_experts_capturer_dp_unexpected_batch_raises():
assert capturer.device_buffer[0, 0, 0].item() == -1
def test_routed_experts_attention_group_is_shared_and_fail_closed():
"""Both sides key routing data by this gid, so it must skip non-full-attention
groups rather than defaulting to 0, and fail closed when none exists."""
common = dict(num_kv_heads=1, head_size=1, dtype=torch.float32)
def test_get_aux_output_connector_passes_config(monkeypatch):
import vllm.distributed.aux_output_connector.worker as aux_output_worker
connector = Mock()
constructor = Mock(return_value=connector)
monkeypatch.setattr(aux_output_worker, "AuxOutputWorkerConnector", constructor)
config = SimpleNamespace(
kv_cache_groups=[
KVCacheGroupSpec(
["swa_layer"],
SlidingWindowMLASpec(block_size=4, sliding_window=8, **common),
),
KVCacheGroupSpec(["full_layer"], FullAttentionSpec(block_size=4, **common)),
]
scheduler_config=SimpleNamespace(max_num_batched_tokens=32)
)
assert get_routed_experts_attn_gid(config) == 1
model = Mock()
kv_cache_config = Mock()
with pytest.raises(ValueError, match="requires a full-attention KV cache group"):
get_routed_experts_attn_gid(SimpleNamespace(kv_cache_groups=[]))
result = aux_output_worker.get_aux_output_connector(model, config, kv_cache_config)
def test_routed_experts_attention_group_unwraps_uniform_type_specs():
"""DeepSeek-V4-shaped groups wrap their specs in ``UniformTypeKVCacheSpecs``.
The wrapper is not a ``FullAttentionSpec``, so a bare isinstance check finds
no group and fails closed on every worker. Unwrap semantics themselves are
covered by ``test_is_full_attention_spec_*`` in tests/v1/core.
"""
common = dict(num_kv_heads=1, head_size=1, dtype=torch.float32)
swa_spec = SlidingWindowMLASpec(block_size=4, sliding_window=8, **common)
mla_spec = MLAAttentionSpec(block_size=4, **common)
config = SimpleNamespace(
kv_cache_groups=[
KVCacheGroupSpec(
["swa_layer"],
UniformTypeKVCacheSpecs(
block_size=4, kv_cache_specs={"swa_layer": swa_spec}
),
),
KVCacheGroupSpec(
["mla_layer"],
UniformTypeKVCacheSpecs(
block_size=4, kv_cache_specs={"mla_layer": mla_spec}
),
),
]
constructor.assert_called_once_with(
model=model,
kv_cache_config=kv_cache_config,
vllm_config=config,
)
assert get_routed_experts_attn_gid(config) == 1
swa_only = SimpleNamespace(kv_cache_groups=[config.kv_cache_groups[0]])
with pytest.raises(ValueError, match="requires a full-attention KV cache group"):
get_routed_experts_attn_gid(swa_only)
assert result is connector
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
def test_mrv2_async_output_returns_existing_routed_experts_field():
from vllm.v1.outputs import ModelRunnerOutput, RoutedExpertsTensors
from vllm.v1.worker.gpu.async_utils import AsyncOutput
from vllm.v1.worker.gpu.sample.output import SamplerOutput
routed_experts = RoutedExpertsTensors(
routing_data=torch.arange(6, dtype=torch.int32, device="cuda").reshape(3, 1, 2),
slot_mapping=torch.tensor([11, 12, 13], device="cuda"),
)
num_sampled = torch.tensor([1], dtype=torch.int32, device="cuda")
sampler_output = SamplerOutput(
sampled_token_ids=torch.tensor([[1]], device="cuda"),
logprobs_tensors=None,
num_nans=None,
num_sampled=num_sampled,
num_rejected=torch.tensor([0], dtype=torch.int32, device="cuda"),
)
output = AsyncOutput(
model_runner_output=ModelRunnerOutput(req_ids=["req"], req_id_to_index={}),
sampler_output=sampler_output,
num_sampled_tokens=num_sampled,
main_stream=torch.cuda.current_stream(),
copy_stream=torch.cuda.Stream(),
routed_experts=routed_experts,
).get_output()
assert output.routed_experts is not None
assert output.routed_experts.routing_data[:, 0, 0].tolist() == [0, 2, 4]
assert output.routed_experts.slot_mapping.tolist() == [11, 12, 13]
@pytest.mark.parametrize("rank", [0, 1])
def test_all_tp_ranks_initialize_capture(monkeypatch, rank):
pytest.importorskip("vllm.vllm_flash_attn", exc_type=ImportError)
import vllm.v1.worker.gpu.model_runner as model_runner
def test_aux_output_worker_connector_binds_capture_on_non_output_rank(monkeypatch):
import vllm.distributed.aux_output_connector.worker as aux_output_worker
capturer = Mock()
constructor = Mock(return_value=capturer)
bind = Mock()
monkeypatch.setattr(model_runner, "RoutedExpertsCapturer", constructor)
monkeypatch.setattr(model_runner, "bind_routed_experts_capturer", bind)
monkeypatch.setattr(aux_output_worker, "RoutedExpertsCapturer", constructor)
monkeypatch.setattr(aux_output_worker, "bind_routed_experts_capturer", bind)
monkeypatch.setattr(
aux_output_worker,
"get_tp_group",
lambda: SimpleNamespace(is_first_rank=False, world_size=1),
)
runner = model_runner.GPUModelRunner.__new__(model_runner.GPUModelRunner)
runner.max_num_tokens = 32
runner.vllm_config = SimpleNamespace(parallel_config=SimpleNamespace(rank=rank))
runner.kv_cache_config = SimpleNamespace()
runner.model = Mock()
runner.init_routed_experts_capturer()
config = SimpleNamespace(
aux_output_config=SimpleNamespace(enable_return_routed_experts=True),
kv_transfer_config=None,
max_concurrent_batches=2,
scheduler_config=SimpleNamespace(max_num_batched_tokens=32),
)
model = Mock()
connector = aux_output_worker.AuxOutputWorkerConnector(
vllm_config=config,
model=model,
kv_cache_config=SimpleNamespace(kv_cache_groups=[_full_attention_kv_group()]),
)
constructor.assert_called_once_with(
max_num_batched_tokens=32,
vllm_config=runner.vllm_config,
kv_cache_config=runner.kv_cache_config,
vllm_config=config,
)
bind.assert_called_once_with(runner.model, capturer)
assert runner.routed_experts_capturer is capturer
bind.assert_called_once_with(model, capturer)
connector.begin_step(Mock())
assert connector.prepare_output(Mock()) is None
capturer.snapshot_routing_data.assert_not_called()
def test_aux_output_worker_connector_default_capacity(monkeypatch):
import vllm.distributed.aux_output_connector.worker as aux_output_worker
tp_group = SimpleNamespace(is_first_rank=True, world_size=1)
store_constructor = Mock()
background_store_constructor = Mock(side_effect=lambda store, **_: store)
capturer = SimpleNamespace(shape_per_token=(2,), output_dtype_name="int32")
monkeypatch.setattr(aux_output_worker, "get_tp_group", lambda: tp_group)
monkeypatch.setattr(
aux_output_worker, "RoutedExpertsCapturer", Mock(return_value=capturer)
)
monkeypatch.setattr(aux_output_worker, "bind_routed_experts_capturer", Mock())
monkeypatch.setattr(
aux_output_worker,
"resolve_kv_cache_block_sizes",
lambda *_: (32, 16),
)
monkeypatch.setattr(aux_output_worker, "BlockObjectStore", store_constructor)
monkeypatch.setattr(
aux_output_worker, "BackgroundBlockObjectStore", background_store_constructor
)
monkeypatch.setattr(aux_output_worker, "RoutedExpertsBuffer", Mock())
config = SimpleNamespace(
aux_output_config=SimpleNamespace(max_bytes=None),
kv_transfer_config=None,
cache_config=SimpleNamespace(enable_prefix_caching=True),
scheduler_config=SimpleNamespace(max_num_seqs=8, max_num_batched_tokens=32),
max_concurrent_batches=2,
)
kwargs = dict(
vllm_config=config,
model=Mock(),
kv_cache_config=SimpleNamespace(
num_blocks=10,
kv_cache_groups=[_full_attention_kv_group(MLAAttentionSpec)],
),
)
aux_output_worker.AuxOutputWorkerConnector(**kwargs)
assert store_constructor.call_args.kwargs["max_bytes"] == 2560
assert store_constructor.call_args.kwargs["object_nbytes"] == 128
assert background_store_constructor.call_args.kwargs["max_pending_batches"] == 16
def test_v2_model_runner_accepts_routed_experts(monkeypatch):
monkeypatch.setattr("importlib.metadata.entry_points", lambda **_: ())
config = SimpleNamespace(
model_config=SimpleNamespace(
enable_return_routed_experts=True,
use_mla=False,
logits_processors=None,
enable_prompt_embeds=False,
),
aux_output_config=SimpleNamespace(enable_return_routed_experts=True),
speculative_config=None,
parallel_config=SimpleNamespace(
prefill_context_parallel_size=1,
+1 -2
View File
@@ -354,8 +354,7 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
scheduler.finished_req_ids_dict = None
scheduler.grammar_compile_error_reqs = set()
scheduler.vllm_config = Mock()
scheduler.vllm_config.model_config.enable_return_routed_experts = False
scheduler.enable_return_routed_experts = False
scheduler.aux_output_connector = None
scheduler.return_sampling_mask = False
scheduler.recompute_kv_load_failures = False
scheduler.defer_block_free = False
+71 -23
View File
@@ -19,6 +19,10 @@ from vllm.config import (
SpeculativeConfig,
VllmConfig,
)
from vllm.distributed.aux_output_connector.connector import (
AuxOutputSchedulerConnector,
AuxRequestOutput,
)
from vllm.distributed.ec_transfer.ec_connector.metrics import ECConnectorStats
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import (
HiSparseConnector,
@@ -55,7 +59,6 @@ from vllm.v1.outputs import (
ECConnectorOutput,
KVConnectorOutput,
ModelRunnerOutput,
RoutedExpertsLists,
SamplingMaskLists,
make_empty_encoder_model_runner_output,
)
@@ -136,21 +139,18 @@ def test_add_requests():
assert len(scheduler.waiting) == i + 1
def test_routed_experts_prompt_start_at_prompt_end():
@pytest.mark.parametrize("max_tokens", [1, 2])
def test_routed_experts_prompt_start_at_prompt_end(max_tokens):
"""A prompt-end offset should return empty routing data, not crash."""
scheduler = create_scheduler()
(request,) = create_requests(num_requests=1)
(request,) = create_requests(num_requests=1, max_tokens=max_tokens)
request.sampling_params.routed_experts_prompt_start = request.num_prompt_tokens
scheduler.aux_output_connector = AuxOutputSchedulerConnector()
scheduler.add_request(request)
scheduler_output = scheduler.schedule()
num_tokens = scheduler_output.num_scheduled_tokens[request.request_id]
manager = Mock()
manager.routed_experts_by_slot = np.empty((0, 1, 1), dtype=np.uint8)
manager.get.return_value = np.empty((0, 1, 1), dtype=np.uint8)
scheduler.enable_return_routed_experts = True
scheduler.routed_experts_mgr = manager
scheduler._re_block_ids = {request.request_id: []}
assert scheduler_output.aux_output_connector_metadata.requests == {
request.request_id: request.num_prompt_tokens
}
outputs = scheduler.update_from_output(
scheduler_output,
@@ -161,21 +161,23 @@ def test_routed_experts_prompt_start_at_prompt_end():
logprobs=None,
prompt_logprobs_dict={},
pooler_output=[],
routed_experts=RoutedExpertsLists(
routing_data=np.zeros((num_tokens, 1, 1), dtype=np.uint8),
slot_mapping=np.arange(num_tokens),
),
aux_output_connector_output={
request.request_id: AuxRequestOutput(
request.num_prompt_tokens,
np.empty((0, 1, 1), dtype=np.uint8),
)
},
),
)
manager.get.assert_called_once_with(
[], request.num_prompt_tokens, token_start=request.num_prompt_tokens
)
assert outputs[request.client_index].outputs[0].routed_experts.shape == (0, 1, 1)
output = outputs[request.client_index].outputs[0]
assert output.new_token_ids == [0]
assert output.routed_experts.shape == (0, 1, 1)
def test_finish_request():
scheduler = create_scheduler()
scheduler.aux_output_connector = Mock()
requests = create_requests(num_requests=10)
for request in requests:
scheduler.add_request(request)
@@ -184,6 +186,7 @@ def test_finish_request():
scheduler.finish_requests(request.request_id, RequestStatus.FINISHED_ABORTED)
assert request.request_id not in scheduler.requests
assert len(scheduler.waiting) == 9 - i
scheduler.aux_output_connector.request_finished.assert_called_with(request)
def test_get_num_unfinished_requests():
@@ -1373,6 +1376,31 @@ def test_preemption_re_records_prefix_cache_query():
assert stats.preempted_requests == 1
def test_preemption_processes_stale_aux_output():
scheduler = create_scheduler(enable_prefix_caching=True)
request = create_requests(num_requests=1)[0]
scheduler.add_request(request)
scheduler_output = scheduler.schedule()
scheduler.running.remove(request)
scheduler.aux_output_connector = Mock()
scheduler._preempt_request(request, 0.0)
scheduler.aux_output_connector.request_finished.assert_called_once_with(request)
scheduler.update_from_output(
scheduler_output,
ModelRunnerOutput(
req_ids=[request.request_id],
req_id_to_index={request.request_id: 0},
sampled_token_ids=[[1000]],
logprobs=None,
prompt_logprobs_dict={},
pooler_output=[],
),
)
scheduler.aux_output_connector.take_output.assert_called_once_with(request, None)
def test_prefix_cache_stats_not_recorded_when_caching_disabled():
"""With prefix caching off there is no local lookup, so admitting a request
records no phantom miss."""
@@ -1461,6 +1489,7 @@ def test_prefix_cache_stats_counted_once_for_retried_then_scheduled_request():
def test_scheduler_reset_prefix_cache():
scheduler = create_scheduler(enable_prefix_caching=True)
scheduler.aux_output_connector = Mock()
requests = create_requests(num_requests=10)
for request in requests:
scheduler.add_request(request)
@@ -1477,10 +1506,20 @@ def test_scheduler_reset_prefix_cache():
# Reset prefix cache should fail since there are still running requests
# and they are taking KV cache
assert not scheduler.reset_prefix_cache()
scheduler.aux_output_connector.reset.assert_not_called()
with pytest.raises(RuntimeError, match=r"pause\(mode='keep'\)"):
scheduler.reset_prefix_cache(reset_running_requests=True)
# pause(mode="keep") also waits for scheduled model outputs to drain.
scheduler.set_pause_state(PauseState.PAUSED_ALL)
with pytest.raises(RuntimeError, match="model output is in flight"):
scheduler.reset_prefix_cache(reset_running_requests=True)
for request in requests:
request.num_in_flight_tokens = 0
# Reset prefix cache with reset_running_requests=True. All running requests
# Should be pushed back to the waiting queue and kv cache should be freed
assert scheduler.reset_prefix_cache(reset_running_requests=True)
scheduler.aux_output_connector.reset.assert_called_once_with()
# Verify requests moved from running to waiting
assert len(scheduler.waiting) == len(requests)
@@ -1490,6 +1529,16 @@ def test_scheduler_reset_prefix_cache():
assert scheduler.waiting[i] == request
@pytest.mark.parametrize("reset_successful", [False, True])
def test_aux_output_reset_follows_kv_reset_result(reset_successful: bool):
scheduler = create_scheduler(enable_prefix_caching=True)
scheduler.aux_output_connector = Mock()
scheduler.kv_cache_manager.reset_prefix_cache = Mock(return_value=reset_successful)
assert scheduler.reset_prefix_cache() is reset_successful
assert scheduler.aux_output_connector.reset.call_count == int(reset_successful)
def test_kv_cache_release_after_keep_pause_preserves_requests():
"""A completed keep pause permits release and recomputation of live requests."""
scheduler = create_scheduler(max_num_seqs=1, enable_prefix_caching=True)
@@ -3791,10 +3840,9 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
scheduler.kv_event_publisher = Mock()
scheduler.finished_req_ids = set()
scheduler.finished_req_ids_dict = None
scheduler.aux_output_connector = None
scheduler.grammar_compile_error_reqs = set()
scheduler.vllm_config = Mock()
scheduler.vllm_config.model_config.enable_return_routed_experts = False
scheduler.enable_return_routed_experts = False
scheduler.return_sampling_mask = False
scheduler.recompute_kv_load_failures = False
scheduler.defer_block_free = False
@@ -1122,6 +1122,7 @@ def _run_mamba_prefix_cache_mrv2(
logprobs_tensors=None,
num_nans=None,
num_sampled=num_sampled,
num_rejected=num_rejected,
)
return sampler_output, num_sampled, num_rejected
+47 -1
View File
@@ -3,8 +3,9 @@
import math
import time
from unittest.mock import MagicMock
from unittest.mock import MagicMock, Mock
import numpy as np
import pytest
from tests.v1.engine.utils import (
@@ -1307,6 +1308,7 @@ def test_lora_request_tracking(log_stats: bool, dummy_test_vectors):
async def test_request_output_collector():
NUM_REQS = 3
TEXT = "a"
routed_experts = np.arange(12, dtype=np.uint8).reshape(2, 3, 2)
def make_outputs() -> list[RequestOutput]:
return [
@@ -1322,6 +1324,9 @@ async def test_request_output_collector():
token_ids=[idx],
cumulative_logprob=(idx + 1 * 1.0),
logprobs=[{"a": idx, "b": idx}],
routed_experts=(
routed_experts if idx == NUM_REQS - 1 else None
),
finish_reason="length" if (idx == NUM_REQS - 1) else None,
)
],
@@ -1374,6 +1379,7 @@ async def test_request_output_collector():
assert output.finished
assert output.outputs[0].finish_reason == "length"
np.testing.assert_array_equal(output.outputs[0].routed_experts, routed_experts)
# Text, token_ids, and logprobs should get merged.
assert output.outputs[0].text == TEXT * num_to_put
for tok_0, tok_1 in zip(output.outputs[0].token_ids, list(range(num_to_put))):
@@ -1385,6 +1391,46 @@ async def test_request_output_collector():
assert output.outputs[0].cumulative_logprob == cumulative_logprob_expected
def test_routed_experts_are_accumulated_until_finish():
state = RequestState(
request_id="request-int",
external_req_id="request",
parent_req=None,
request_index=0,
lora_request=None,
output_kind=RequestOutputKind.DELTA,
prompt="prompt",
prompt_token_ids=[1, 2, 3, 4],
prompt_embeds=None,
logprobs_processor=Mock(
logprobs=None,
cumulative_logprob=0.0,
pop_prompt_logprobs=Mock(return_value=None),
),
detokenizer=Mock(get_next_output_text=Mock(return_value="")),
max_tokens_param=2,
arrival_time=0.0,
queue=None,
log_stats=False,
stream_interval=1,
)
prompt_chunk = np.arange(24, dtype=np.uint8).reshape(4, 3, 2)
decode_chunk = np.arange(12, dtype=np.uint8).reshape(2, 3, 2) + 24
state.routed_experts_chunks.append(prompt_chunk)
partial = state.make_request_output([5], None, None, None)
assert partial is not None
assert partial.outputs[0].routed_experts is None
state.routed_experts_chunks.append(decode_chunk)
finished = state.make_request_output([6], None, FinishReason.LENGTH, None)
assert finished is not None
np.testing.assert_array_equal(
finished.outputs[0].routed_experts,
np.concatenate((prompt_chunk, decode_chunk)),
)
@pytest.mark.asyncio
async def test_cumulative_output_collector_n():
"""Test collector correctly handles multiple outputs by index."""
+12 -17
View File
@@ -3,12 +3,14 @@
import numpy as np
from vllm.distributed.aux_output_connector.connector import (
AuxRequestOutput,
)
from vllm.v1.executor.ray_utils import detach_zero_copy_from_model_runner_output
from vllm.v1.outputs import (
LogprobsLists,
LogprobsTensors,
ModelRunnerOutput,
RoutedExpertsLists,
)
@@ -59,26 +61,19 @@ def test_detach_zero_copy_from_model_runner_output_copies_only_numpy_views():
assert output.prompt_logprobs_dict["req-0"] is prompt_logprobs
def test_detach_zero_copy_routed_experts_without_logprobs():
def test_detach_zero_copy_aux_output_without_logprobs():
rows = _make_readonly(np.arange(12, dtype=np.uint8).reshape(2, 3, 2))
output = ModelRunnerOutput(
req_ids=["req-0"],
req_id_to_index={"req-0": 0},
routed_experts=RoutedExpertsLists(
routing_data=_make_readonly(np.arange(12, dtype=np.int32).reshape(2, 3, 2)),
slot_mapping=_make_readonly(np.array([7, 8], dtype=np.int64)),
),
aux_output_connector_output={"req-0": AuxRequestOutput(0, rows)},
)
original = output.routed_experts
assert output.logprobs is None
detach_zero_copy_from_model_runner_output(output)
detached = output.routed_experts
assert detached is not None
assert detached is not original
assert detached.routing_data is not original.routing_data
assert detached.slot_mapping is not original.slot_mapping
assert detached.routing_data.flags.writeable
assert detached.slot_mapping.flags.writeable
np.testing.assert_array_equal(detached.routing_data, original.routing_data)
np.testing.assert_array_equal(detached.slot_mapping, original.slot_mapping)
aux_output = output.aux_output_connector_output
assert aux_output is not None
detached = aux_output["req-0"].rows
assert detached is not rows
assert detached.flags.writeable
np.testing.assert_array_equal(detached, rows)
+3
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.config.attention import AttentionConfig, HiSparseConfig
from vllm.config.aux_output import AuxOutputConfig
from vllm.config.cache import CacheConfig
from vllm.config.compilation import (
CompilationConfig,
@@ -66,6 +67,8 @@ from vllm.config.weight_transfer import WeightTransferConfig
# __all__ should only contain classes and functions.
# Types and globals should be imported from their respective modules.
__all__ = [
# From vllm.config.aux_output
"AuxOutputConfig",
# From vllm.config.attention
"AttentionConfig",
"HiSparseConfig",
+31
View File
@@ -0,0 +1,31 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Configuration for execution auxiliary outputs."""
from pydantic import Field
from vllm.config.utils import config
@config
class AuxOutputConfig:
"""Configuration for auxiliary-output delivery."""
enable_return_routed_experts: bool = False
"""Capture and return routed-experts auxiliary outputs."""
max_bytes: int | None = Field(default=None, gt=0)
"""LRU capacity, or ``None`` to derive it from the KV cache capacity."""
@property
def enabled(self) -> bool:
"""Whether any execution auxiliary output is enabled."""
return self.enable_return_routed_experts
def compute_hash(self) -> str:
"""Hash AuxOutput settings that alter the model forward graph."""
from vllm.config.utils import hash_factors
return hash_factors(
{"enable_return_routed_experts": self.enable_return_routed_experts}
)
-2
View File
@@ -247,8 +247,6 @@ class ModelConfig:
NOTE: This disables both `torch.compile` and CUDA graphs, and is
equivalent to setting `-cc.mode=none -cc.cudagraph_mode=none`."""
enable_return_routed_experts: bool = False
"""Whether to return routed experts."""
return_sampling_mask: bool = False
"""Whether to return the post-processing token support for each sample."""
max_logprobs: int = Field(default=20, ge=-1)
+53 -33
View File
@@ -28,6 +28,7 @@ from vllm.utils import random_uuid
from vllm.utils.hashing import safe_hash
from .attention import AttentionConfig, HiSparseConfig
from .aux_output import AuxOutputConfig
from .cache import CacheConfig
from .compilation import CompilationConfig, CompilationMode, CUDAGraphMode
from .device import DeviceConfig
@@ -376,6 +377,8 @@ class VllmConfig:
"""Model weight offloading configuration."""
attention_config: AttentionConfig = Field(default_factory=AttentionConfig)
"""Attention configuration."""
aux_output_config: AuxOutputConfig = Field(default_factory=AuxOutputConfig)
"""Execution auxiliary output configuration."""
engram_config: EngramConfig | None = None
"""N-gram embedding storage and sharding settings."""
mamba_config: MambaConfig = Field(default_factory=MambaConfig)
@@ -554,6 +557,7 @@ class VllmConfig:
vllm_factors.append(self.ec_transfer_config.compute_hash())
else:
vllm_factors.append("None")
vllm_factors.append(self.aux_output_config.compute_hash())
if self.additional_config:
if isinstance(additional_config := self.additional_config, dict):
additional_config_hash = safe_hash(
@@ -1084,6 +1088,53 @@ class VllmConfig:
# This is the same for all backends
self.kv_transfer_config.kv_role = "kv_both"
def _verify_aux_output_compatibility(self) -> None:
"""Reject configurations unsupported by enabled auxiliary outputs."""
if not self.aux_output_config.enabled:
return
if not self.use_v2_model_runner:
raise ValueError(
"AuxOutput Connector requires Model Runner V2; set "
"VLLM_USE_V2_MODEL_RUNNER=1."
)
if self.model_config.runner_type != "generate":
raise ValueError("AuxOutput Connector only supports generate runners.")
if not self.model_config.is_moe:
raise ValueError("AuxOutput Connector only supports MoE models.")
if not self.cache_config.enable_prefix_caching:
raise ValueError("AuxOutput Connector requires prefix caching.")
if (
self.speculative_config is not None
and self.speculative_config.enable_adaptive_verification
):
raise ValueError(
"--enable-return-routed-experts is incompatible with "
"adaptive speculative verification."
)
if self.parallel_config.pipeline_parallel_size > 1:
raise ValueError(
"--enable-return-routed-experts is incompatible with "
"pipeline parallelism (PP > 1)."
)
if (
self.parallel_config.decode_context_parallel_size > 1
or self.parallel_config.prefill_context_parallel_size > 1
):
raise ValueError(
"--enable-return-routed-experts is incompatible with "
"context parallelism (DCP/PCP > 1)."
)
kv_transfer_config = self.kv_transfer_config
if (
kv_transfer_config is not None
and kv_transfer_config.is_kv_transfer_instance
):
raise ValueError(
"--enable-return-routed-experts is incompatible with KV "
"connectors (PD disaggregation and KV cache offload)."
)
def _verify_kv_transfer_compat(self) -> None:
"""Reject configurations that silently corrupt KV transfers."""
if (
@@ -1294,38 +1345,6 @@ class VllmConfig:
self.parallel_config.is_moe_model = self.model_config.is_moe
if (
self.model_config is not None
and self.model_config.enable_return_routed_experts
):
if self.parallel_config.pipeline_parallel_size > 1:
raise ValueError(
"--enable-return-routed-experts is incompatible with "
"pipeline parallelism (PP > 1)."
)
if (
self.parallel_config.decode_context_parallel_size > 1
or self.parallel_config.prefill_context_parallel_size > 1
):
raise ValueError(
"--enable-return-routed-experts is incompatible with context "
"parallelism (DCP > 1 or PCP > 1)."
)
# Incompatible with any KV connector — covers both PD disaggregation
# (kv_producer/kv_consumer: routing captured on P can't reach D) and
# single-instance KV offload/sharing (kv_both: slot_mapping semantics
# change when KV blocks live outside local GPU memory, breaking the
# slot-indexed routed_experts buffer).
if (
self.kv_transfer_config is not None
and self.kv_transfer_config.is_kv_transfer_instance
):
raise ValueError(
"--enable-return-routed-experts is incompatible with KV "
"connectors (PD disaggregation, KV cache offload)."
)
if (
self.model_config is not None
and self.model_config.multimodal_config is not None
@@ -2032,6 +2051,7 @@ class VllmConfig:
# Resolve kv_offloading-derived connector name into kv_transfer_config
# before the HMA check below, which inspects the connector class.
self._post_init_kv_transfer_config()
self._verify_aux_output_compatibility()
if self.is_mm_encoder_only and self.cache_config.enable_prefix_caching:
# Such an instance publishes encoder embeddings and runs no language
@@ -2689,7 +2709,7 @@ class VllmConfig:
f"quantization={self.model_config.quantization}, "
f"quantization_config={self.model_config.quantization_config}, " # noqa
f"enforce_eager={self.model_config.enforce_eager}, "
f"enable_return_routed_experts={self.model_config.enable_return_routed_experts}, " # noqa
f"aux_output_config={self.aux_output_config!r}, "
f"kv_cache_dtype={self.cache_config.cache_dtype}, "
f"device_config={self.device_config.device}, "
f"structured_outputs_config={self.structured_outputs_config!r}, "
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,157 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Scheduler-side control plane for execution auxiliary outputs."""
from __future__ import annotations
from collections.abc import Iterator, Sequence
from dataclasses import dataclass
from typing import TYPE_CHECKING
import numpy as np
if TYPE_CHECKING:
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.request import Request
@dataclass
class PackedBlockHashes:
"""Contiguous, self-contained block hashes for scheduler-worker IPC."""
data: bytes
item_size: int
def __iter__(self) -> Iterator[bytes]:
for start in range(0, len(self.data), self.item_size):
yield self.data[start : start + self.item_size]
@dataclass
class AuxOutputConnectorMetadata:
generation: int
requests: dict[str, int]
block_hashes: dict[str, PackedBlockHashes]
finished_requests: tuple[str, ...]
@dataclass
class AuxRequestOutput:
token_start: int
rows: np.ndarray
class AuxOutputSchedulerConnector:
"""Build worker metadata without owning auxiliary output payloads or stores."""
def __init__(self) -> None:
# Number of hashes already sent to the worker for each active request.
self._sent_hash_counts: dict[str, int] = {}
# Terminal events are delivered with the next connector metadata.
self._finished_requests: dict[str, PackedBlockHashes | None] = {}
self._generation = 0
def build_connector_meta(
self,
scheduler_output: SchedulerOutput,
requests: dict[str, Request],
) -> AuxOutputConnectorMetadata:
"""Build one step's incremental worker metadata."""
scheduled_requests: dict[str, int] = {}
block_hashes_by_request: dict[str, PackedBlockHashes] = {}
for request_id in scheduler_output.num_scheduled_tokens:
num_sent = self._sent_hash_counts.setdefault(request_id, 0)
request = requests[request_id]
packed = self._pack_new_hashes(request.block_hashes, num_sent)
if packed is not None:
block_hashes_by_request[request_id] = packed
self._sent_hash_counts[request_id] = len(request.block_hashes)
assert request.sampling_params is not None
scheduled_requests[request_id] = max(
request.sampling_params.routed_experts_prompt_start,
0 if request.num_output_tokens == 0 else request.num_tokens - 1,
)
# A settled token can complete a hash block after the next async schedule
# was built. Send a hash-only update if the request was not rescheduled.
if len(self._sent_hash_counts) != len(scheduled_requests):
for request_id, num_sent in self._sent_hash_counts.items():
if request_id in scheduled_requests:
continue
request = requests[request_id]
packed = self._pack_new_hashes(request.block_hashes, num_sent)
if packed is None:
continue
block_hashes_by_request[request_id] = packed
self._sent_hash_counts[request_id] = len(request.block_hashes)
# Sending transfers ownership of these one-shot events.
finished_requests = tuple(self._finished_requests)
block_hashes_by_request.update(
(request_id, block_hashes)
for request_id, block_hashes in self._finished_requests.items()
if block_hashes is not None
)
self._finished_requests = {}
return AuxOutputConnectorMetadata(
self._generation,
scheduled_requests,
block_hashes_by_request,
finished_requests,
)
def take_output(
self,
request: Request,
output: dict[str, AuxRequestOutput] | None,
) -> np.ndarray | None:
"""Return the accepted R3 rows for one scheduled request."""
request_id = request.request_id
assert output is not None and request_id in output, (
f"auxiliary output worker output is missing {request_id}"
)
request_output = output[request_id]
token_end = request.num_tokens - 1
local_end = token_end - request_output.token_start
if local_end < 0:
assert not request.is_finished(), (
"finished auxiliary output output has no accepted token range: "
f"request={request_id}, token_end={token_end}, "
f"output_start={request_output.token_start}, "
"output_end="
f"{request_output.token_start + len(request_output.rows)}"
)
return None
assert local_end <= len(request_output.rows), (
"auxiliary output worker output has an invalid token range: "
f"request={request_id}, token_end={token_end}, "
f"output_start={request_output.token_start}, "
f"output_end={request_output.token_start + len(request_output.rows)}"
)
return request_output.rows[:local_end]
def request_finished(self, request: Request) -> None:
"""Queue a request's terminal event and final block hashes."""
request_id = request.request_id
num_sent = self._sent_hash_counts.pop(request_id, None)
if num_sent is None:
return
# The next metadata delivers this terminal event to the worker.
self._finished_requests[request_id] = self._pack_new_hashes(
request.block_hashes, num_sent
)
@staticmethod
def _pack_new_hashes(
block_hashes: Sequence[bytes], num_sent: int
) -> PackedBlockHashes | None:
assert num_sent <= len(block_hashes), "KV block-hash history shrank"
new_hashes = block_hashes[num_sent:]
if not new_hashes:
return None
return PackedBlockHashes(b"".join(new_hashes), len(new_hashes[0]))
def reset(self) -> None:
"""Start a new auxiliary output namespace after a prefix-cache reset."""
# The worker drops temporary state on generation changes; resend hashes.
self._sent_hash_counts.clear()
self._finished_requests.clear()
self._generation += 1
@@ -0,0 +1,219 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Routed-experts auxiliary output keys, publication, and materialization."""
from __future__ import annotations
from collections.abc import Hashable, Iterable, Sequence
from dataclasses import dataclass
from typing import Any
import numpy as np
from vllm.distributed.aux_output_connector.store import (
BackgroundBlockObjectStore,
BlockObject,
BlockObjectStore,
)
@dataclass
class _RequestTail:
slot: int
block_start: int
length: int = 0
class RoutedExpertsBuffer:
"""Own incomplete and unkeyed full routed-experts blocks."""
def __init__(
self,
dtype: np.dtype[Any],
shape_per_token: tuple[int, ...],
block_size: int,
max_num_seqs: int,
max_num_batched_tokens: int,
max_concurrent_batches: int,
) -> None:
self.dtype = dtype
self.shape_per_token = shape_per_token
self.block_size = block_size
max_step_blocks = (
max_num_batched_tokens + block_size - 1
) // block_size + max_num_seqs
# In-flight full blocks plus one incomplete tail per active request.
max_blocks = max_concurrent_batches * max_step_blocks + max_num_seqs
self._rows: np.ndarray = np.empty(
(max_blocks, block_size, *shape_per_token), dtype=dtype
)
self._free_slots = list(range(len(self._rows) - 1, -1, -1))
self._owned_slots: dict[int, int] = {}
self._requests: dict[Hashable, _RequestTail] = {}
def _tail(self, request_id: Hashable, block_start: int) -> _RequestTail:
tail = self._requests.get(request_id)
if tail is not None:
assert tail.block_start == block_start, (
"auxiliary output capture skipped an incomplete block: "
f"request={request_id}, expected={tail.block_start}, "
f"actual={block_start}"
)
return tail
assert self._free_slots, "auxiliary output block pool is exhausted"
tail = _RequestTail(self._free_slots.pop(), block_start)
self._requests[request_id] = tail
return tail
def capture(
self, request_id: Hashable, token_start: int, rows: np.ndarray
) -> list[tuple[int, np.ndarray]]:
"""Stage rows and return completed blocks without retaining them."""
rows = np.asarray(rows)
assert rows.shape[1:] == self.shape_per_token and rows.dtype == self.dtype, (
"routed-experts capture profile changed"
)
if token_start < 0:
raise ValueError("auxiliary output token start must be non-negative")
completed: list[tuple[int, np.ndarray]] = []
offset = 0
while offset < len(rows):
position = token_start + offset
block_start = position // self.block_size * self.block_size
local_start = position - block_start
# Full aligned input blocks do not need tail staging.
if (
request_id not in self._requests
and local_start == 0
and len(rows) - offset >= self.block_size
):
completed.append((block_start, rows[offset : offset + self.block_size]))
offset += self.block_size
continue
tail = self._tail(request_id, block_start)
assert local_start == tail.length, (
"auxiliary output capture is not contiguous: "
f"request={request_id}, expected={block_start + tail.length}, "
f"actual={position}"
)
count = min(self.block_size - local_start, len(rows) - offset)
if count == 1:
self._rows[tail.slot, local_start] = rows[offset]
else:
self._rows[tail.slot, local_start : local_start + count] = rows[
offset : offset + count
]
tail.length += count
offset += count
if tail.length == self.block_size:
block = self._rows[tail.slot]
del self._requests[request_id]
self._owned_slots[id(block)] = tail.slot
completed.append((block_start, block))
return completed
def read(
self, request_id: Hashable, token_start: int, token_end: int
) -> np.ndarray:
tail = self._requests.get(request_id)
assert tail is not None, (
f"auxiliary output buffer is missing request {request_id}"
)
local_start = token_start - tail.block_start
local_end = token_end - tail.block_start
assert 0 <= local_start < local_end <= tail.length, (
"auxiliary output range is unavailable: "
f"request={request_id}, range=[{token_start}, {token_end}), "
f"available=[{tail.block_start}, {tail.block_start + tail.length})"
)
return self._rows[tail.slot, local_start:local_end].copy()
def retain_block(self, rows: np.ndarray) -> np.ndarray:
"""Retain one unkeyed block after the current capture call."""
if id(rows) in self._owned_slots:
return rows
assert self._free_slots, "auxiliary output block pool is exhausted"
slot = self._free_slots.pop()
retained = self._rows[slot]
retained[...] = rows
self._owned_slots[id(retained)] = slot
return retained
def release_block(self, rows: np.ndarray) -> None:
slot = self._owned_slots.pop(id(rows), None)
if slot is not None:
self._free_slots.append(slot)
def discard(self, request_id: Hashable) -> None:
tail = self._requests.pop(request_id, None)
if tail is not None:
self._free_slots.append(tail.slot)
def reset(self) -> None:
self._requests.clear()
self._owned_slots.clear()
self._free_slots[:] = range(len(self._rows) - 1, -1, -1)
def routed_experts_keys(
block_hashes: Iterable[bytes], aux_output_namespace: str
) -> list[str]:
prefix = f"vllm-artifact/{aux_output_namespace}/"
return [prefix + block_hash.hex() for block_hash in block_hashes]
def materialize_routed_experts(
store: BackgroundBlockObjectStore | BlockObjectStore,
aux_output_keys: list[str],
*,
shape_per_token: tuple[int, ...],
dtype: np.dtype[Any],
) -> np.ndarray:
if not aux_output_keys:
raise ValueError("routed-experts auxiliary output key list must not be empty")
payload = store.get_concatenated(aux_output_keys)
return np.frombuffer(payload, dtype=dtype).reshape((-1, *shape_per_token))
def publish_routed_experts(
store: BackgroundBlockObjectStore | BlockObjectStore,
*,
batches: Sequence[tuple[Sequence[str], list[tuple[int, np.ndarray]]]],
block_size: int,
retain_keys: Sequence[str] = (),
release_keys: Sequence[str] = (),
) -> None:
"""Publish immutable full R3 blocks."""
objects = []
for aux_output_keys, blocks in batches:
for block_start, array in blocks:
if block_start < 0 or block_start % block_size:
raise ValueError(
"auxiliary output block start is not hash-block aligned"
)
block_index = block_start // block_size
if block_index >= len(aux_output_keys):
raise ValueError(
"auxiliary output block has no corresponding key: "
f"start={block_start}, index={block_index}, "
f"keys={len(aux_output_keys)}, block_size={block_size}"
)
if len(array) != block_size:
raise ValueError(
"auxiliary output block length does not match hash block size"
)
objects.append(
BlockObject(
key=aux_output_keys[block_index],
payload=array.tobytes(order="C"),
)
)
store.put(
objects,
retain_keys=retain_keys,
release_keys=release_keys,
)
@@ -0,0 +1,234 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""In-process auxiliary-output storage."""
from __future__ import annotations
import mmap
import queue
import threading
from collections import OrderedDict
from collections.abc import Iterable
from dataclasses import dataclass
@dataclass(frozen=True)
class BlockObject:
"""One immutable auxiliary output object."""
key: str
payload: bytes
class BlockObjectStoreError(RuntimeError):
"""AuxOutput storage or retrieval failed."""
class BackgroundBlockObjectStore:
"""Serialize store mutations on a background thread."""
def __init__(self, store: BlockObjectStore, *, max_pending_batches: int) -> None:
self._store = store
self._queue: queue.Queue[
tuple[list[BlockObject], tuple[str, ...], tuple[str, ...]] | None
] = queue.Queue(maxsize=max_pending_batches)
self._error: BaseException | None = None
self._closed = False
self._thread = threading.Thread(
target=self._run,
daemon=True,
name="vllm-aux-output-writer",
)
self._thread.start()
def _run(self) -> None:
while True:
update = self._queue.get()
try:
if update is None:
return
if self._error is None:
objects, retain_keys, release_keys = update
self._store.put(
objects,
retain_keys=retain_keys,
release_keys=release_keys,
)
except BaseException as error:
self._error = error
finally:
self._queue.task_done()
def _raise_if_failed(self) -> None:
if self._error is not None:
raise BlockObjectStoreError(
"auxiliary output publication failed"
) from self._error
def put(
self,
objects: list[BlockObject],
*,
retain_keys: Iterable[str] = (),
release_keys: Iterable[str] = (),
) -> None:
retains = tuple(retain_keys)
releases = tuple(release_keys)
if not objects and not retains and not releases:
return
if self._closed:
raise RuntimeError("auxiliary output store is closed")
self._queue.put((objects, retains, releases))
self._raise_if_failed()
def get_concatenated(self, keys: list[str]) -> bytes:
self._queue.join()
self._raise_if_failed()
return self._store.get_concatenated(keys)
def close(self) -> None:
if self._closed:
return
self._closed = True
self._queue.join()
self._queue.put(None)
self._thread.join()
self._store.close()
self._raise_if_failed()
class BlockObjectStore:
"""Single-owner bounded store that fails closed after an eviction."""
_UNALLOCATED_SLOT = -1
def __init__(
self,
*,
max_bytes: int,
object_nbytes: int,
) -> None:
if object_nbytes <= 0:
raise ValueError("auxiliary output object size must be positive")
if max_bytes < object_nbytes:
raise ValueError("auxiliary output store must fit at least one object")
self.object_nbytes = object_nbytes
self.num_slots = max_bytes // object_nbytes
self._lru: OrderedDict[str, int] = OrderedDict()
self._references: dict[str, int] = {}
self._free_slots: list[int] = []
self._next_slot = 0
self._arena: mmap.mmap | None = mmap.mmap(
-1, self.num_slots * self.object_nbytes
)
def _evict_to_fit(self, protected: set[str]) -> None:
excess = len(self._lru) - self.num_slots
if excess <= 0:
return
victims = []
for key in self._lru:
if key not in self._references and key not in protected:
victims.append(key)
if len(victims) == excess:
break
if len(victims) != excess:
raise BlockObjectStoreError(
"auxiliary output store cannot retain the requested batch: "
f"limit={self.num_slots} objects"
)
for victim in victims:
slot = self._lru.pop(victim)
if slot != self._UNALLOCATED_SLOT:
self._free_slots.append(slot)
def _allocate_slot(self) -> int:
if self._free_slots:
return self._free_slots.pop()
slot = self._next_slot
self._next_slot += 1
return slot
def put(
self,
objects: list[BlockObject],
*,
retain_keys: Iterable[str] = (),
release_keys: Iterable[str] = (),
) -> None:
arena = self._arena
if arena is None:
raise RuntimeError("auxiliary output store is closed")
unique = {obj.key: obj for obj in objects}
self._retain(retain_keys)
terminal_order = self._release(release_keys)
for key in unique:
self._lru.setdefault(key, self._UNALLOCATED_SLOT)
self._lru.move_to_end(key)
for key in terminal_order:
if key in self._lru:
self._lru.move_to_end(key)
try:
self._evict_to_fit(set(unique) - set(terminal_order))
except BlockObjectStoreError:
for key in unique:
if self._lru.get(key) == self._UNALLOCATED_SLOT:
del self._lru[key]
raise
for object_id, obj in unique.items():
slot = self._lru.get(object_id)
if slot is None:
continue
if slot != self._UNALLOCATED_SLOT:
continue
slot = self._allocate_slot()
offset = slot * self.object_nbytes
arena[offset : offset + self.object_nbytes] = obj.payload
self._lru[object_id] = slot
def _retain(self, keys: Iterable[str]) -> None:
for key in keys:
references = self._references.get(key, 0)
self._references[key] = references + 1
def _release(self, keys: Iterable[str]) -> list[str]:
terminal_order = []
for key in keys:
references = self._references[key] - 1
if references:
self._references[key] = references
continue
del self._references[key]
terminal_order.append(key)
return terminal_order
def get_concatenated(self, keys: list[str]) -> bytes:
arena_obj = self._arena
if arena_obj is None:
raise RuntimeError("auxiliary output store is closed")
try:
entries = [self._lru[key] for key in keys]
except KeyError as error:
raise BlockObjectStoreError(
"auxiliary output object does not exist; the object may have been "
f"evicted (used={len(self._lru)}, "
f"limit={self.num_slots} objects). Increase "
"aux_output_config.max_bytes when a KV cache hit requires it."
) from error
arena = memoryview(arena_obj)
try:
payload = b"".join(
arena[slot * self.object_nbytes : (slot + 1) * self.object_nbytes]
for slot in entries
)
finally:
arena.release()
for key in keys:
self._lru.move_to_end(key)
return payload
def close(self) -> None:
arena = self._arena
if arena is not None:
self._arena = None
arena.close()
@@ -0,0 +1,412 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Worker-side auxiliary-output data plane."""
from __future__ import annotations
from collections.abc import Iterable, Sequence
from dataclasses import dataclass, field
from threading import Lock
from typing import TYPE_CHECKING, Any
import numpy as np
import torch
from vllm.config import VllmConfig
from vllm.distributed.aux_output_connector.connector import (
AuxOutputConnectorMetadata,
AuxRequestOutput,
)
from vllm.distributed.aux_output_connector.routed_experts import (
RoutedExpertsBuffer,
materialize_routed_experts,
publish_routed_experts,
routed_experts_keys,
)
from vllm.distributed.aux_output_connector.store import (
BackgroundBlockObjectStore,
BlockObjectStore,
)
from vllm.distributed.parallel_state import get_tp_group
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
RoutedExpertsCapturer,
bind_routed_experts_capturer,
)
from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes
if TYPE_CHECKING:
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.gpu.input_batch import InputBatch
@dataclass
class _WorkerRequestState:
aux_output_keys: list[str] = field(default_factory=list)
pending_blocks: list[tuple[int, np.ndarray]] = field(default_factory=list)
capture_cursor: int | None = None
scheduled_cursor: int = 0
emit_cursor: int = 0
pending_outputs: int = 0
# Terminal event received; teardown waits for in-flight step outputs.
finished: bool = False
@dataclass
class PendingAuxOutput:
"""Own one step's R3 snapshot until its asynchronous copy is consumed."""
connector: AuxOutputWorkerConnector
request_ids: list[str]
token_starts: np.ndarray
query_start_loc: np.ndarray
# GPU snapshot taken on the main stream.
routed_experts_gpu: torch.Tensor
# Filled by enqueue_cpu_copy on the output copy stream.
routed_experts: np.ndarray | None = None
num_sampled: np.ndarray | None = None
num_rejected: np.ndarray | None = None
def enqueue_cpu_copy(
self, num_sampled: np.ndarray, num_rejected: np.ndarray
) -> None:
"""Enqueue asynchronous D2H copies; call on the output copy stream."""
self.num_sampled = num_sampled
self.num_rejected = num_rejected
self.routed_experts = self.routed_experts_gpu.to(
"cpu", non_blocking=True
).numpy()
def process_output(self) -> dict[str, AuxRequestOutput]:
return self.connector.process_output(self)
class AuxOutputWorkerConnector:
"""Own capture, request tails, and backend resources on the output worker."""
def __init__(
self,
*,
vllm_config: VllmConfig,
model: torch.nn.Module,
kv_cache_config: KVCacheConfig,
) -> None:
max_num_batched_tokens = vllm_config.scheduler_config.max_num_batched_tokens
capturer = RoutedExpertsCapturer(
max_num_batched_tokens=max_num_batched_tokens, vllm_config=vllm_config
)
bind_routed_experts_capturer(model, capturer)
self._capturer = capturer
self._store: BackgroundBlockObjectStore | None = None
self._buffer: RoutedExpertsBuffer | None = None
self._requests: dict[str, _WorkerRequestState] = {}
self._generation = 0
self._step_metadata: AuxOutputConnectorMetadata | None = None
# Steps whose asynchronous copy has not been consumed yet. The engine
# consumes step outputs in order, so at most max_concurrent_batches
# are outstanding; guarded by _lock against the output thread.
self._pending_outputs: list[PendingAuxOutput] = []
self._lock = Lock()
self._max_concurrent_batches = vllm_config.max_concurrent_batches
# Every TP rank participates in capture collectives, but only the
# executor output rank owns the auxiliary output data plane.
if not get_tp_group().is_first_rank:
return
shape_per_token = self._capturer.shape_per_token
dtype: np.dtype[Any] = np.dtype(self._capturer.output_dtype_name)
scheduler_block_size, hash_block_size = resolve_kv_cache_block_sizes(
kv_cache_config, vllm_config
)
hashes_per_kv_block = scheduler_block_size // hash_block_size
block_nbytes = hash_block_size * int(np.prod(shape_per_token)) * dtype.itemsize
max_bytes = vllm_config.aux_output_config.max_bytes
if max_bytes is None:
max_bytes = kv_cache_config.num_blocks * hashes_per_kv_block * block_nbytes
self._store = BackgroundBlockObjectStore(
BlockObjectStore(max_bytes=max_bytes, object_nbytes=block_nbytes),
max_pending_batches=2 * vllm_config.scheduler_config.max_num_seqs,
)
self._buffer = RoutedExpertsBuffer(
dtype,
shape_per_token,
hash_block_size,
vllm_config.scheduler_config.max_num_seqs,
max_num_batched_tokens,
vllm_config.max_concurrent_batches,
)
def prepare_output(self, input_batch: InputBatch) -> PendingAuxOutput | None:
"""Snapshot one step's R3 tensor for asynchronous CPU transfer."""
if self._buffer is None or self._step_metadata is None:
return None
request_ids = list(input_batch.req_ids)
query_start_loc = input_batch.query_start_loc_np[: len(request_ids) + 1]
num_rows = int(query_start_loc[-1])
pending_output = PendingAuxOutput(
connector=self,
request_ids=request_ids,
token_starts=input_batch.num_computed_tokens_np,
query_start_loc=query_start_loc,
routed_experts_gpu=self._capturer.snapshot_routing_data(num_rows),
)
with self._lock:
assert len(self._pending_outputs) < self._max_concurrent_batches, (
"auxiliary output step outputs are not consumed in order"
)
self._pending_outputs.append(pending_output)
for request_id in pending_output.request_ids:
self._requests[request_id].pending_outputs += 1
return pending_output
def process_output(self, pending: PendingAuxOutput) -> dict[str, AuxRequestOutput]:
"""Commit one consumed R3 snapshot and build request outputs.
Request teardown deferred by begin_step is completed here once no
in-flight step covers the request.
"""
with self._lock:
teardown = []
release_keys: list[str] = []
try:
outputs = self._commit_output(pending)
finally:
self._pending_outputs.remove(pending)
for request_id in pending.request_ids:
state = self._requests[request_id]
state.pending_outputs -= 1
if state.pending_outputs == 0 and state.finished:
teardown.append(request_id)
release_keys.extend(reversed(state.aux_output_keys))
if teardown:
self._publish_blocks([], release_keys=release_keys)
self._teardown(teardown)
return outputs
def _commit_output(self, pending: PendingAuxOutput) -> dict[str, AuxRequestOutput]:
buffer = self._buffer
store = self._store
assert buffer is not None and store is not None
block_size = buffer.block_size
routed_experts = pending.routed_experts
num_sampled = pending.num_sampled
num_rejected = pending.num_rejected
assert (
routed_experts is not None
and num_sampled is not None
and num_rejected is not None
), "auxiliary output CPU copy was not enqueued"
# Publish the whole batch before materializing any consumer output.
materialize_outputs: list[tuple[str, int, int]] = []
block_batches = []
outputs: dict[str, AuxRequestOutput] = {}
# Use the ModelRunner's actual batch boundaries rather than rebuilding them.
for request_id, token_start, start, end, sampled, rejected in zip(
pending.request_ids,
pending.token_starts,
pending.query_start_loc[:-1],
pending.query_start_loc[1:],
num_sampled,
num_rejected,
strict=True,
):
request_num_tokens = end - start
assert request_num_tokens > 0, (
"auxiliary output request token count must be positive"
)
state = self._requests[request_id]
# Capture precedes speculative acceptance, so discard the rejected
# suffix. Batch boundaries still span the full executed range.
rejected = int(rejected)
assert 0 <= rejected <= request_num_tokens, (
"auxiliary output rejected-token count is invalid"
)
rows = routed_experts[start : end - rejected]
capture_start = token_start
capture_cursor = state.capture_cursor
if capture_cursor is None:
capture_cursor = capture_start
assert capture_start >= capture_cursor, (
"auxiliary output capture moved backwards"
)
if capture_start > capture_cursor:
# Reattach after an optimistically scheduled suffix was rejected.
assert capture_cursor < state.scheduled_cursor, (
"auxiliary output capture has an unbacked token gap"
)
capture_start = capture_cursor
emit_start = state.emit_cursor
# Complete blocks without keys remain pending until a hash update.
completed = buffer.capture(request_id, capture_start, rows)
token_end = capture_start + len(rows)
state.capture_cursor = token_end
state.scheduled_cursor = token_start + request_num_tokens
block_batches.append((state, completed))
if sampled > 0 and emit_start <= token_end:
if emit_start >= capture_start:
outputs[request_id] = AuxRequestOutput(
emit_start, rows[emit_start - capture_start :]
)
state.emit_cursor = token_end
else:
materialize_outputs.append((request_id, emit_start, token_end))
# A consumer may reuse a block produced earlier in the same batch.
self._publish_blocks(block_batches)
for request_id, emit_start, token_end in materialize_outputs:
state = self._requests[request_id]
stored_end = (
min(token_end // block_size, len(state.aux_output_keys)) * block_size
)
if emit_start < stored_end:
first_block = emit_start // block_size
stored = materialize_routed_experts(
store,
state.aux_output_keys[first_block : stored_end // block_size],
shape_per_token=buffer.shape_per_token,
dtype=buffer.dtype,
)
local_start = emit_start % block_size
rows = stored[local_start : local_start + stored_end - emit_start]
if stored_end < token_end:
rows = np.concatenate(
(rows, buffer.read(request_id, stored_end, token_end))
)
else:
rows = buffer.read(request_id, emit_start, token_end)
outputs[request_id] = AuxRequestOutput(emit_start, rows)
state.emit_cursor = token_end
return outputs
def _publish_blocks(
self,
batches: list[tuple[_WorkerRequestState, list[tuple[int, np.ndarray]]]],
retain_keys: Sequence[str] = (),
release_keys: Sequence[str] = (),
) -> None:
store = self._store
buffer = self._buffer
assert store is not None and buffer is not None
ready_batches = []
for state, completed in batches:
blocks = state.pending_blocks + completed
keyed_end = len(state.aux_output_keys) * buffer.block_size
ready = [(start, rows) for start, rows in blocks if start < keyed_end]
state.pending_blocks = [
(start, buffer.retain_block(rows))
for start, rows in blocks
if start >= keyed_end
]
if ready:
ready_batches.append((state.aux_output_keys, ready))
if ready_batches or retain_keys or release_keys:
publish_routed_experts(
store,
batches=ready_batches,
block_size=buffer.block_size,
retain_keys=retain_keys,
release_keys=release_keys,
)
for _, blocks in ready_batches:
for _, rows in blocks:
buffer.release_block(rows)
def _teardown(self, request_ids: Iterable[str]) -> None:
"""Drop finished requests' temporary state after publishing key releases."""
buffer = self._buffer
assert buffer is not None
for request_id in request_ids:
state = self._requests.pop(request_id)
for _, rows in state.pending_blocks:
buffer.release_block(rows)
buffer.discard(request_id)
def begin_step(self, metadata: AuxOutputConnectorMetadata | None) -> None:
"""Apply one scheduler step's request and block-hash updates.
While an unconsumed step covers a finished request, its teardown is
deferred to that step's process_output.
"""
self._step_metadata = metadata
if self._buffer is None or metadata is None:
return
assert not metadata.requests.keys() & metadata.finished_requests, (
"auxiliary output request cannot run and finish in one step"
)
assert metadata.generation >= self._generation, (
"auxiliary output metadata generation moved backwards"
)
with self._lock:
release_keys: list[str] = []
if metadata.generation > self._generation:
# A generation change follows a prefix-cache reset, which the
# scheduler only performs once all model output is consumed.
assert not self._pending_outputs, (
"auxiliary output generation changed with output in flight"
)
release_keys.extend(
key
for state in self._requests.values()
for key in reversed(state.aux_output_keys)
)
self._buffer.reset()
self._requests.clear()
self._generation = metadata.generation
for request_id, emit_start in metadata.requests.items():
state = self._requests.setdefault(
request_id, _WorkerRequestState(emit_cursor=emit_start)
)
if state.pending_outputs == 0:
# In-flight steps leave the worker cursor behind the
# scheduler's optimistic view; only check settled requests.
assert emit_start <= state.emit_cursor, (
"auxiliary output Scheduler emit cursor moved ahead"
)
block_batches: list[
tuple[_WorkerRequestState, list[tuple[int, np.ndarray]]]
] = []
retained_keys: list[str] = []
for request_id, block_hashes in metadata.block_hashes.items():
state = self._requests[request_id]
keys = routed_experts_keys(block_hashes, str(self._generation))
state.aux_output_keys.extend(keys)
retained_keys.extend(keys)
block_batches.append((state, []))
finished_now: list[str] = []
for request_id in metadata.finished_requests:
state = self._requests[request_id]
state.finished = True
if state.pending_outputs:
continue
finished_now.append(request_id)
release_keys.extend(reversed(state.aux_output_keys))
self._publish_blocks(block_batches, retained_keys, release_keys)
self._teardown(finished_now)
def close(self) -> None:
with self._lock:
self._pending_outputs.clear()
if self._store is not None:
try:
self._store.close()
finally:
self._store = None
def get_aux_output_connector(
model: torch.nn.Module, vllm_config: VllmConfig, kv_cache_config: KVCacheConfig
) -> AuxOutputWorkerConnector:
return AuxOutputWorkerConnector(
model=model,
kv_cache_config=kv_cache_config,
vllm_config=vllm_config,
)
+15 -3
View File
@@ -35,6 +35,7 @@ from typing_extensions import TypeIs
import vllm.envs as envs
from vllm.config import (
AttentionConfig,
AuxOutputConfig,
CacheConfig,
CompilationConfig,
ConfigType,
@@ -447,7 +448,9 @@ class EngineArgs:
"""Arguments for vLLM engine."""
model: str = ModelConfig.model
enable_return_routed_experts: bool = ModelConfig.enable_return_routed_experts
# Public compatibility argument. Canonical runtime state lives in
# AuxOutputConfig.
enable_return_routed_experts: bool = False
return_sampling_mask: bool = ModelConfig.return_sampling_mask
model_weights: str = ModelConfig.model_weights
served_model_name: str | list[str] | None = ModelConfig.served_model_name
@@ -674,6 +677,7 @@ class EngineArgs:
structured_outputs_config: StructuredOutputsConfig = get_field(
VllmConfig, "structured_outputs_config"
)
aux_output_config: AuxOutputConfig = get_field(VllmConfig, "aux_output_config")
reasoning_parser: str = StructuredOutputsConfig.reasoning_parser
reasoning_parser_plugin: str | None = None
@@ -816,6 +820,10 @@ class EngineArgs:
self.compilation_config = CompilationConfig(**self.compilation_config)
if isinstance(self.attention_config, dict):
self.attention_config = AttentionConfig(**self.attention_config)
if isinstance(self.aux_output_config, dict):
self.aux_output_config = AuxOutputConfig(**self.aux_output_config)
if self.enable_return_routed_experts:
self.aux_output_config.enable_return_routed_experts = True
if isinstance(self.engram_config, dict):
self.engram_config = EngramConfig(**self.engram_config)
if isinstance(self.mamba_config, dict):
@@ -884,6 +892,7 @@ class EngineArgs:
"""Shared CLI arguments for vLLM engine."""
# Model arguments
model_kwargs = get_kwargs(ModelConfig)
aux_output_kwargs = get_kwargs(AuxOutputConfig)
model_group = parser.add_argument_group(
title="ModelConfig",
description=ModelConfig.__doc__,
@@ -923,7 +932,7 @@ class EngineArgs:
model_group.add_argument("--enforce-eager", **model_kwargs["enforce_eager"])
model_group.add_argument(
"--enable-return-routed-experts",
**model_kwargs["enable_return_routed_experts"],
**aux_output_kwargs["enable_return_routed_experts"],
)
model_group.add_argument(
"--return-sampling-mask",
@@ -1749,6 +1758,9 @@ class EngineArgs:
vllm_group.add_argument(
"--structured-outputs-config", **vllm_kwargs["structured_outputs_config"]
)
vllm_group.add_argument(
"--aux-output-config", **vllm_kwargs["aux_output_config"]
)
vllm_group.add_argument("--profiler-config", **vllm_kwargs["profiler_config"])
vllm_group.add_argument(
"--optimization-level", **vllm_kwargs["optimization_level"]
@@ -1856,7 +1868,6 @@ class EngineArgs:
quantization_config=self.quantization_config,
allow_deprecated_quantization=self.allow_deprecated_quantization,
enforce_eager=self.enforce_eager,
enable_return_routed_experts=self.enable_return_routed_experts,
return_sampling_mask=self.return_sampling_mask,
max_logprobs=self.max_logprobs,
logprobs_mode=self.logprobs_mode,
@@ -2669,6 +2680,7 @@ class EngineArgs:
load_config=load_config,
offload_config=offload_config,
attention_config=attention_config,
aux_output_config=self.aux_output_config,
engram_config=self.engram_config,
mamba_config=mamba_config,
kernel_config=kernel_config,
@@ -10,15 +10,12 @@ from collections.abc import Callable
from functools import partial
from typing import Protocol, runtime_checkable
import numpy as np
import torch
from vllm.config import VllmConfig
from vllm.distributed.parallel_state import get_tp_group
from vllm.forward_context import get_forward_context
from vllm.platforms import current_platform
from vllm.v1.kv_cache_interface import KVCacheConfig, is_full_attention_spec
from vllm.v1.outputs import RoutedExpertsTensors
logger = logging.getLogger(__name__)
@@ -46,37 +43,29 @@ def _get_routed_experts_shape(vllm_config: VllmConfig) -> tuple[int, int, int]:
class RoutedExpertsCapturer:
"""Worker-side capturer for routed experts, lives on GPU.
Layer-level hooks call :meth:`capture` from inside the forward pass
with the per-layer ``topk_ids`` tensor. The tensor is sliced to the
tokens owned by this DP rank and written into a preallocated device
buffer. At the end of the step, :class:`GPUModelRunner` reads the
device buffer, issues a D2H copy into a pinned CPU buffer, and hands
the result to the scheduler via :class:`RoutedExpertsLists`.
Layer-level hooks call :meth:`capture` inside the forward pass. Routing
rows owned by this DP rank are written into a preallocated device buffer.
The device / pinned-CPU transit buffers use ``torch.int32`` (not a
narrow ``uint8``/``uint16`` sized by ``num_experts``). This keeps the
SP all-gather path free of dtype casts, matches the router's native
``topk_ids`` indices dtype more closely, and costs only a few MB per
worker (``max_num_batched_tokens * num_layers * top_k * 4`` bytes).
The scheduler-side slot buffer
(``RoutedExpertsManager.routed_experts_by_slot``) still uses the
narrow dtype -- numpy fancy-index assignment in ``store_batch``
narrows the data on the way in.
The device buffer uses ``int32``. Stable snapshots use the narrowest dtype
that can represent every logical expert ID.
Invariants:
- One instance per worker; shape is fixed at init and covers the
worst-case step (``max_num_batched_tokens`` tokens).
- Every routed layer overwrites the current step's token rows.
- ``device_buffer.dtype`` is ``torch.int32``.
"""
def __init__(
self,
max_num_batched_tokens: int,
vllm_config: VllmConfig,
kv_cache_config: KVCacheConfig,
) -> None:
num_layers, _, num_experts_per_tok = _get_routed_experts_shape(vllm_config)
num_layers, num_experts, num_experts_per_tok = _get_routed_experts_shape(
vllm_config
)
self.shape_per_token = (num_layers, num_experts_per_tok)
self.output_dtype_name = "uint8" if num_experts <= 256 else "uint16"
self.output_dtype = getattr(torch, self.output_dtype_name)
logger.info(
"RoutedExpertsCapturer: allocating buffer with "
"max_tokens=%d, num_layers=%d, num_experts_per_tok=%d "
@@ -87,23 +76,12 @@ class RoutedExpertsCapturer:
vllm_config.model_config.hf_text_config.model_type,
)
self.device_buffer = torch.zeros(
(
max_num_batched_tokens,
num_layers,
num_experts_per_tok,
),
# Use int32 for the device / host transit buffers: it
# matches the router's native topk_ids dtype, is universally
# supported by NCCL (uint8/uint16 are version-dependent),
# and the extra bytes are small (few MB per worker). The
# big scheduler-side slot buffer stays narrow.
(max_num_batched_tokens, *self.shape_per_token),
dtype=torch.int32,
device=current_platform.device_type,
)
self.dp_rank = vllm_config.parallel_config.data_parallel_rank
self.tp_size = vllm_config.parallel_config.tensor_parallel_size
# KV cache group whose slot layout the routing data is keyed by.
self.attn_gid = get_routed_experts_attn_gid(kv_cache_config)
def capture(self, layer_id: int, topk_ids: torch.Tensor) -> None:
"""Capture expert routing decisions for a specific layer.
@@ -139,31 +117,21 @@ class RoutedExpertsCapturer:
"""
ctx = get_forward_context()
if ctx.dp_metadata is None: # single dp
start_loc = 0
end_loc = topk_ids.shape[0]
token_num_per_dp = topk_ids.shape[0]
else: # multi dp
if ctx.dp_metadata is None:
local_topk_ids = topk_ids
else:
num_tokens_dp = ctx.dp_metadata.num_tokens_across_dp_cpu
token_num_per_dp = int(num_tokens_dp[self.dp_rank].item())
num_local_tokens = int(num_tokens_dp[self.dp_rank].item())
total = int(num_tokens_dp.sum().item())
n = topk_ids.shape[0]
shard_sizes = getattr(ctx.dp_metadata, "local_sizes", None)
gathered_size = sum(shard_sizes) if shard_sizes is not None else -1
if n == total:
# Naive dispatch: all DP ranks' tokens concatenated
# before routing. This rank owns tokens
# [end_loc - token_num_per_dp, end_loc).
cumsum = torch.cumsum(num_tokens_dp, dim=0)
end_loc = int(cumsum[self.dp_rank].item())
start_loc = end_loc - token_num_per_dp
elif n == token_num_per_dp:
# Modular-kernel path: DP combine happens inside
# quant_method.apply; select_experts only sees this
# rank's tokens, take the whole tensor.
start_loc = 0
end_loc = token_num_per_dp
end = int(num_tokens_dp[: self.dp_rank + 1].sum().item())
local_topk_ids = topk_ids[end - num_local_tokens : end]
elif n == num_local_tokens:
local_topk_ids = topk_ids
elif shard_sizes is not None and n == gathered_size:
# Naive DP+EP dispatch gathers the sequence-parallel shards in
# flattened DP-then-TP order. ``local_sizes`` is the exact
@@ -173,12 +141,11 @@ class RoutedExpertsCapturer:
assert len(shard_sizes) % num_dp_ranks == 0
shards_per_dp_rank = len(shard_sizes) // num_dp_ranks
first_shard = self.dp_rank * shards_per_dp_rank
start_loc = sum(shard_sizes[:first_shard])
end_loc = start_loc + token_num_per_dp
start = sum(shard_sizes[:first_shard])
local_topk_ids = topk_ids[start : start + num_local_tokens]
elif (
self.tp_size > 1
and n != token_num_per_dp
and n == (token_num_per_dp + self.tp_size - 1) // self.tp_size
and n == (num_local_tokens + self.tp_size - 1) // self.tp_size
):
# SP + modular-kernel path. All-gather across the TP
# group along dim=0 to reconstruct the full per-DP-rank
@@ -193,60 +160,30 @@ class RoutedExpertsCapturer:
#
# ``topk_ids`` is already whatever the router produced
# (typically int32/int64, both supported by NCCL); the
# downstream ``device_buffer[...] = topk_ids[...]``
# setitem narrows into int32 automatically.
topk_ids = get_tp_group().all_gather(topk_ids, dim=0)
start_loc = 0
end_loc = token_num_per_dp
# downstream buffer assignment narrows to the capture dtype.
local_topk_ids = get_tp_group().all_gather(topk_ids, dim=0)[
:num_local_tokens
]
else:
sp_expected = (
(token_num_per_dp + self.tp_size - 1) // self.tp_size
if self.tp_size > 0
else -1
)
sp_expected = (num_local_tokens + self.tp_size - 1) // self.tp_size
raise AssertionError(
"RoutedExpertsCapturer: unexpected topk_ids batch "
f"dim {n} (expected {total}, {token_num_per_dp}, "
f"dim {n} (expected {total}, {num_local_tokens}, "
f"{gathered_size}, or {sp_expected} for "
f"dp_rank={self.dp_rank}, tp_size={self.tp_size})"
)
if layer_id >= self.device_buffer.shape[1]:
if not 0 <= layer_id < self.device_buffer.shape[1]:
raise IndexError(
f"routed-experts layer {layer_id} exceeds capture buffer "
f"layer count {self.device_buffer.shape[1]}"
)
self.device_buffer[:token_num_per_dp, layer_id, :] = topk_ids[
start_loc:end_loc, :
]
self.device_buffer[: len(local_topk_ids), layer_id] = local_topk_ids
def get_device_buffer(self) -> torch.Tensor:
"""Return the underlying device buffer so the model runner can
issue the D2H copy. The tensor is shared; callers must either
clone or fully drain it before the next forward pass overwrites it.
"""
return self.device_buffer
def get_routed_experts(
self, slot_mappings: torch.Tensor, num_tokens: int
) -> RoutedExpertsTensors:
"""Snapshot this step's routing data and its attention slot mapping.
Both tensors are cloned since the capture buffer and the shared
``slot_mappings`` are overwritten by the next step, which may race
with an in-flight D2H copy.
Args:
slot_mappings: Per-KV-cache-group slot mappings for this step,
shape ``(num_kv_cache_groups, max_num_batched_tokens)``.
num_tokens: Total number of tokens scheduled in this step.
"""
return RoutedExpertsTensors(
routing_data=self.device_buffer[:num_tokens].clone(),
slot_mapping=slot_mappings[self.attn_gid, :num_tokens].clone(),
)
def snapshot_routing_data(self, num_tokens: int) -> torch.Tensor:
"""Return a stable snapshot of the current routing data."""
return self.device_buffer[:num_tokens].to(self.output_dtype)
def bind_routed_experts_capturer(
@@ -268,20 +205,12 @@ def bind_routed_experts_capturer(
continue
if not isinstance(module, MoERunner):
continue
layer_id = module.layer_id
def capture_fn(
topk_ids: torch.Tensor,
layer_id: int = layer_id,
capturer: RoutedExpertsCapturer = capturer,
) -> None:
capturer.capture(layer_id, topk_ids)
capture_fn = partial(capturer.capture, module.layer_id)
quant_method = module._quant_method
moe_kernel = getattr(quant_method, "moe_kernel", None)
impl = getattr(moe_kernel, "impl", None)
fused_experts = getattr(impl, "fused_experts", None)
if quant_method.is_monolithic:
moe_kernel = getattr(quant_method, "moe_kernel", None)
impl = getattr(moe_kernel, "impl", None)
fused_experts = getattr(impl, "fused_experts", None)
if not (
isinstance(fused_experts, FusedMoEExpertsMonolithic)
and fused_experts.supports_routing_replay_capture()
@@ -303,132 +232,3 @@ def bind_routed_experts_capturer(
if num_bound == 0:
raise ValueError("No supported MoE router found for routed-experts capture.")
def get_routed_experts_attn_gid(kv_cache_config: KVCacheConfig) -> int:
"""Return the full-attention KV cache group used for routed experts."""
for gid, group in enumerate(kv_cache_config.kv_cache_groups):
if is_full_attention_spec(group.kv_cache_spec):
return gid
raise ValueError("Routed-experts capture requires a full-attention KV cache group.")
class RoutedExpertsManager:
"""Scheduler-side slot-indexed buffer for routed experts.
Lives on CPU in the scheduler process. Each slot corresponds to
``block_id * block_size + offset_in_block`` where ``block_id`` is
drawn from the physical KV-cache block pool, so routing data is
tied to physical blocks and naturally survives preemption for
prefix-cached blocks (prefix hits re-expose the same slots).
Data flow per step:
1. Worker D2Hs its device capture buffer into
:class:`RoutedExpertsLists` and returns it via
:class:`ModelRunnerOutput`.
2. Scheduler calls :meth:`store_batch` with that step's
``(routing_data, slot_mapping)`` — a single CPU->CPU
fancy-index assign, ~few MB per step.
3. On request completion / abort / preemption, the scheduler
calls :meth:`get` with the request's block IDs to recover
the full per-token routing.
Memory: ``routed_experts_by_slot`` is sized for the whole block
pool (``num_blocks * block_size`` slots). For large block pools
this can reach multiple GB; see the init log for the exact size.
"""
def __init__(
self,
vllm_config: VllmConfig,
kv_cache_config: KVCacheConfig,
) -> None:
# Hybrid models also have KV groups whose slot layout differs.
self.attn_gid = get_routed_experts_attn_gid(kv_cache_config)
attn_group = kv_cache_config.kv_cache_groups[self.attn_gid]
self.block_size = attn_group.kv_cache_spec.block_size
# All kv_cache_groups share the same physical block pool, so
# block IDs span [0, num_blocks) regardless of how many groups
# exist. Sizing to the full pool avoids index-out-of-range
# when different groups happen to land on the same block.
num_layers, num_experts, num_experts_per_tok = _get_routed_experts_shape(
vllm_config
)
max_num_slots = kv_cache_config.num_blocks * self.block_size
# Expert IDs are 0..num_experts-1; uint8 fits 256 distinct
# values so the boundary is ``<= 256`` (NOT ``< 256``). Keeping
# this narrow matters because the slot buffer is sized for the
# whole block pool and can reach multiple GB.
expert_id_dtype = np.uint8 if num_experts <= 256 else np.uint16
self.routed_experts_by_slot = np.zeros(
(
max_num_slots,
num_layers,
num_experts_per_tok,
),
dtype=expert_id_dtype,
)
logger.info(
"RoutedExpertsManager CPU buffer: %.2f GB "
"(slots=%d, layers=%d, top_k=%d, dtype=%s)",
self.routed_experts_by_slot.nbytes / 1e9,
max_num_slots,
num_layers,
num_experts_per_tok,
self.routed_experts_by_slot.dtype.name,
)
def store_batch(self, data: np.ndarray, slot_mapping: np.ndarray) -> None:
"""Persist one step's routed experts into the slot buffer.
Equivalent to ``slot_buffer[slot_mapping] = data``; numpy fancy
indexing handles repeated / out-of-order indices. Called once
per scheduler step in ``update_from_output``.
"""
self.routed_experts_by_slot[slot_mapping] = data
def get(
self,
block_ids: list[int],
num_tokens: int,
token_start: int = 0,
) -> np.ndarray:
"""Read routed experts data for a completed / preempted request.
Reconstructs a per-token slot_mapping from the request's block
IDs and returns the routing slice. Because numpy fancy indexing
returns a **copy** (not a view), the returned ndarray is safe
to hold across subsequent :meth:`store_batch` calls — do not
replace the fancy index with a slice without re-verifying.
Args:
block_ids: Block IDs from the attention KV-cache group.
num_tokens: Number of tokens that have gone through a forward
pass and therefore have routing data written to their
slots (typically ``request.num_tokens - 1``; the last
sampled token has not been forwarded yet). Slots beyond
``request.num_computed_tokens`` are zero-initialized.
token_start: Skip the first ``token_start`` tokens from the
result. The slot_mapping is sliced before the fancy-index
read, so only the requested slots are fetched — no large
intermediate array is allocated. Clamped to
``[0, num_tokens]`` automatically.
Returns:
Array of shape (num_tokens - token_start, num_layers,
num_experts_per_tok).
"""
bs = self.block_size
block_ids_array = np.array(block_ids, dtype=np.int32)
block_offsets = np.arange(bs)
# slot = block_id * block_size + offset_in_block; flatten the
# (num_blocks, block_size) grid and trim to num_tokens, then
# skip the first token_start entries so only the requested
# range is fetched in a single fancy-index read.
slot_mapping = (
block_ids_array.reshape(-1, 1) * bs + block_offsets.reshape(1, -1)
).flatten()[:num_tokens]
slot_mapping = slot_mapping[token_start:]
return self.routed_experts_by_slot[slot_mapping]
+4
View File
@@ -194,6 +194,10 @@ class RequestOutput:
completion.cumulative_logprob = (
next_completion.cumulative_logprob
)
# R3 is returned on the terminal output and must survive
# aggregation with earlier chunks that have no R3.
if next_completion.routed_experts is not None:
completion.routed_experts = next_completion.routed_experts
completion.finish_reason = next_completion.finish_reason
completion.stop_reason = next_completion.stop_reason
else:
+7
View File
@@ -14,6 +14,9 @@ if TYPE_CHECKING:
import numpy.typing as npt
import torch
from vllm.distributed.aux_output_connector.connector import (
AuxOutputConnectorMetadata,
)
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorMetadata
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata
from vllm.lora.request import LoRARequest
@@ -23,6 +26,7 @@ if TYPE_CHECKING:
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
from vllm.v1.request import Request
else:
AuxOutputConnectorMetadata = object
ECConnectorMetadata = object
KVConnectorMetadata = object
KVCacheBlockCopy = object
@@ -285,6 +289,9 @@ class SchedulerOutput:
# synchronously during this step (load_async=False).
has_sync_kv_loads: bool = False
# Execution-auxiliary output control metadata consumed by the worker connector.
aux_output_connector_metadata: AuxOutputConnectorMetadata | None = None
# EC Cache Connector metadata
ec_connector_metadata: ECConnectorMetadata | None = None
# EC Cache Manager metadata
+33 -98
View File
@@ -9,6 +9,7 @@ from typing import Any
from vllm.compilation.cuda_graph import CUDAGraphStat
from vllm.config import KVEventsConfig, VllmConfig
from vllm.distributed.aux_output_connector.connector import AuxOutputSchedulerConnector
from vllm.distributed.ec_transfer.ec_connector.base import (
ECConnectorBase,
ECConnectorMetadata,
@@ -26,9 +27,6 @@ from vllm.distributed.kv_transfer.kv_connector.v1 import (
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
RoutedExpertsManager,
)
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
from vllm.multimodal.encoder_budget import MultiModalBudget
from vllm.multimodal.utils import get_mm_features_in_window
@@ -393,26 +391,13 @@ class Scheduler(SchedulerInterface):
if self.log_stats and vllm_config.observability_config.enable_mfu_metrics:
self.perf_metrics = ModelMetrics(vllm_config)
self.enable_return_routed_experts = (
vllm_config.model_config.enable_return_routed_experts
self.aux_output_connector = (
AuxOutputSchedulerConnector()
if vllm_config.aux_output_config.enabled
else None
)
self.return_sampling_mask = vllm_config.model_config.return_sampling_mask
if self.enable_return_routed_experts:
assert self.dcp_world_size == 1 and self.pcp_world_size == 1, (
"enable_return_routed_experts does not support context parallelism "
"(dcp_world_size > 1 or pcp_world_size > 1)"
)
self.routed_experts_mgr = RoutedExpertsManager(
vllm_config=vllm_config,
kv_cache_config=kv_cache_config,
)
# Block-ID snapshot taken at schedule time (before forward),
# so update_from_output can read slot data even if a later
# schedule() frees the blocks (async scheduling race).
self._re_block_ids: dict[str, list[int]] = {}
self._pause_state: PauseState = PauseState.UNPAUSED
# In-flight requests still prefilling (prefill chunks + in-progress
@@ -1487,6 +1472,13 @@ class Scheduler(SchedulerInterface):
meta = self._build_kv_connector_meta(self.connector, scheduler_output)
scheduler_output.kv_connector_metadata = meta
if self.aux_output_connector is not None:
scheduler_output.aux_output_connector_metadata = (
self.aux_output_connector.build_connector_meta(
scheduler_output, self.requests
)
)
# Build the connector meta for ECConnector
if self.ec_connector is not None:
ec_meta: ECConnectorMetadata = self.ec_connector.build_connector_meta(
@@ -1541,6 +1533,8 @@ class Scheduler(SchedulerInterface):
assert request.status == RequestStatus.RUNNING, (
"Only running requests can be preempted"
)
if self.aux_output_connector is not None:
self.aux_output_connector.request_finished(request)
self._free_request_blocks(request)
self.encoder_cache_manager.free(request)
self._inflight_prefills.discard(request)
@@ -1596,22 +1590,6 @@ class Scheduler(SchedulerInterface):
if not request.is_prefill_chunk:
self._inflight_prefills.discard(request)
# Snapshot block IDs for routed experts before forward starts.
# A concurrent schedule() may preempt requests and free blocks
# before update_from_output runs; the snapshot survives that.
# Use update() to preserve entries from the previous step that
# have not yet been consumed by update_from_output (async
# scheduling may call _update_after_schedule again before the
# prior update_from_output runs).
if self.enable_return_routed_experts:
gid = self.routed_experts_mgr.attn_gid
self._re_block_ids.update(
{
rid: self.kv_cache_manager.get_blocks(rid).get_block_ids()[gid]
for rid in num_scheduled_tokens
}
)
# Clear the finished and preempted request IDs.
# NOTE: We shouldn't just clear() here because it will also affect
# the scheduler output.
@@ -1962,28 +1940,6 @@ class Scheduler(SchedulerInterface):
num_scheduled_tokens,
)
)
# Persist per-step routed experts into the scheduler-side slot
# buffer (CPU->CPU fancy-index assign; ~few MB per step).
# MUST precede the per-request routing reads below: stopped
# requests may terminate on tokens generated in this very step,
# whose routing was just D2H'd into model_runner_output.
routing_data = None
routing_offsets: dict[str, int] = {}
if model_runner_output.routed_experts is not None:
re = model_runner_output.routed_experts
self.routed_experts_mgr.store_batch(re.routing_data, re.slot_mapping)
routing_data = re.routing_data.astype(
self.routed_experts_mgr.routed_experts_by_slot.dtype,
copy=False,
)
# Build offset map using model runner's request order
# (input_batch ordering), NOT scheduler dict order.
offset = 0
for rid in model_runner_output.req_ids:
routing_offsets[rid] = offset
offset += num_scheduled_tokens[rid]
# NOTE(woosuk): As len(num_scheduled_tokens) can be up to 1K or more,
# the below loop can be a performance bottleneck. We should do our best
# to avoid expensive operations inside the loop.
@@ -2076,7 +2032,6 @@ class Scheduler(SchedulerInterface):
ec_transfer_params = None
prefill_stats = None
status_before_stop = request.status
num_output_tokens_before = len(request._output_token_ids)
# Check for stop and update request status.
if new_token_ids:
@@ -2113,48 +2068,13 @@ class Scheduler(SchedulerInterface):
stopped = True
routed_experts = None
if (
self.enable_return_routed_experts
and routing_data is not None
and new_token_ids
):
req_offset = routing_offsets[req_id]
end = req_offset + num_tokens_scheduled
block_ids = self._re_block_ids.pop(req_id, [])
if num_output_tokens_before == 0:
# Prefill completed: read full prompt routing from
# slot buffer using the block-ID snapshot taken at
# schedule time (immune to async preemption).
if (
request.sampling_params is not None
and request.sampling_params.routed_experts_prompt_start
is not None
):
prompt_start = (
request.sampling_params.routed_experts_prompt_start
)
assert prompt_start <= request.num_prompt_tokens
else:
prompt_start = 0
routed_experts = self.routed_experts_mgr.get(
block_ids,
request.num_prompt_tokens,
token_start=prompt_start,
)
else:
if scheduled_spec_token_ids:
# Spec decode: accepted tokens at the START of
# the scheduled range, rejected at the end.
routed_experts = routing_data[
req_offset : req_offset + len(new_token_ids)
]
else:
# Normal decode / re-prefill: token(s) at the END.
routed_experts = routing_data[end - len(new_token_ids) : end]
should_emit_output = bool(
new_token_ids or pooler_output is not None or stopped
)
if self.aux_output_connector is not None and should_emit_output:
routed_experts = self.aux_output_connector.take_output(
request, model_runner_output.aux_output_connector_output
)
if should_emit_output:
prefill_stats = request.take_prefill_stats()
if prefill_stats is not None:
@@ -2641,6 +2561,8 @@ class Scheduler(SchedulerInterface):
) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
assert request.is_finished()
if self.aux_output_connector is not None:
self.aux_output_connector.request_finished(request)
self._inflight_prefills.discard(request)
connector_delay_free_blocks, kv_xfer_params = self._connector_finished(request)
@@ -2773,6 +2695,16 @@ class Scheduler(SchedulerInterface):
Otherwise, this method will only reset the KV prefix cache when there
is no running requests taking KV cache.
"""
if reset_running_requests and self.aux_output_connector is not None:
if self._pause_state != PauseState.PAUSED_ALL:
raise RuntimeError(
"AuxOutput Connector only supports resetting running requests "
"after pause(mode='keep')."
)
if any(request.num_in_flight_tokens for request in self.requests.values()):
raise RuntimeError(
"AuxOutput Connector cannot reset while model output is in flight."
)
if reset_running_requests:
# For logging.
timestamp = time.monotonic()
@@ -2803,6 +2735,9 @@ class Scheduler(SchedulerInterface):
if reset_connector:
reset_successful = self.reset_connector_cache() and reset_successful
if reset_successful and self.aux_output_connector is not None:
self.aux_output_connector.reset()
return reset_successful
def reset_connector_cache(self) -> bool:
+4 -8
View File
@@ -233,14 +233,10 @@ def detach_zero_copy_from_model_runner_output(output: "ModelRunnerOutput") -> No
token_ids_c, logprobs_c, ranks_c, cu_num_generated_tokens
)
if output.routed_experts is not None:
routing_data, slot_mapping = output.routed_experts
routing_data_c = _copy_if_readonly(routing_data)
slot_mapping_c = _copy_if_readonly(slot_mapping)
if routing_data_c is not routing_data or slot_mapping_c is not slot_mapping:
output.routed_experts = type(output.routed_experts)(
routing_data_c, slot_mapping_c
)
aux_output = output.aux_output_connector_output
if aux_output is not None:
for request_output in aux_output.values():
request_output.rows = _copy_if_readonly(request_output.rows)
class FutureWrapper(Future):
+3 -77
View File
@@ -15,6 +15,7 @@ from vllm.utils.torch_utils import PIN_MEMORY
from vllm.v1.core.sched.output import SchedulerOutput
if TYPE_CHECKING:
from vllm.distributed.aux_output_connector.connector import AuxRequestOutput
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorWorkerMetadata
from vllm.distributed.ec_transfer.ec_connector.metrics import ECConnectorStats
from vllm.distributed.kv_events import KVConnectorKVEvents
@@ -23,6 +24,7 @@ if TYPE_CHECKING:
)
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats
else:
AuxRequestOutput = object
KVConnectorStats = object
KVConnectorWorkerMetadata = object
KVConnectorKVEvents = object
@@ -188,73 +190,6 @@ class LogprobsTensors(NamedTuple):
)
class RoutedExpertsTensors(NamedTuple):
"""Device-side snapshot of routed experts data, pending async D2H.
Produced by :class:`GPUModelRunner` at the end of each async-scheduled
step. The copy stream waits on the default stream, then issues
non-blocking D2H via :meth:`to_cpu_nonblocking` into a pinned CPU
buffer; :class:`AsyncGPUModelRunnerOutput.get_output` synchronizes
the copy before the scheduler reads it.
Sliced to ``total_num_scheduled_tokens`` (step-level, across all
requests — NOT per-request). Both ``routing_data`` and
``slot_mapping`` must be private clones when sourced from shared
capturer / prepare-input buffers, so the next forward pass /
``_prepare_inputs`` on the default stream does not race with a
D2H still pending on the copy stream.
"""
# (num_scheduled_tokens, num_layers, num_experts_per_tok)
routing_data: torch.Tensor
# (num_scheduled_tokens,)
slot_mapping: torch.Tensor
def to_cpu_nonblocking(self) -> "RoutedExpertsTensors":
"""Issue non-blocking D2H on the current stream.
NOTE: ``non_blocking=True`` only delivers true overlap when the
CPU target is pinned. The current fallback here allocates a
new pageable CPU tensor per call, which silently degrades to a
synchronous copy; acceptable because the sync happens on the
dedicated copy stream, not the default stream.
"""
if self.routing_data.device.type == "cpu":
return self
return RoutedExpertsTensors(
self.routing_data.to("cpu", non_blocking=True),
self.slot_mapping.to("cpu", non_blocking=True),
)
def tolists(self) -> "RoutedExpertsLists":
"""Convert to the numpy-backed form consumed by the scheduler.
``.cpu()`` is a no-op when the tensor is already on CPU, so this
is cheap for the post-D2H case; for raw device tensors it will
synchronously block, which is only reached in tests.
"""
return RoutedExpertsLists(
self.routing_data.cpu().numpy(),
self.slot_mapping.cpu().numpy(),
)
class RoutedExpertsLists(NamedTuple):
"""CPU-side routed experts, the form :meth:`RoutedExpertsManager.store_batch`
consumes.
Batched per scheduler step: the leading dim is the number of tokens
scheduled across all requests in this step (``total_num_scheduled_tokens``),
not per-request tokens. ``slot_mapping[i]`` tells the scheduler which
physical KV-cache slot row ``i`` of ``routing_data`` belongs to.
"""
# (num_scheduled_tokens, num_layers, num_experts_per_tok)
routing_data: np.ndarray
# (num_scheduled_tokens,)
slot_mapping: np.ndarray
# [num_reqs, <dynamic>]
# The shape of each element depends on the pooler used
PoolerOutput: TypeAlias = torch.Tensor | list[torch.Tensor] | list[torch.Tensor | None]
@@ -361,16 +296,7 @@ class ModelRunnerOutput:
# information related to cudagraph execution
cudagraph_stats: CUDAGraphStat | None = None
# Per-step routed experts data captured by the worker.
# ``routing_data`` shape: (num_scheduled_tokens, num_layers,
# num_experts_per_tok); expert IDs as uint8/uint16.
# ``slot_mapping`` shape: (num_scheduled_tokens,); physical KV-cache
# slot for each row of routing_data.
# ``num_scheduled_tokens`` is step-level (total across all requests
# in this step), not per-request. The scheduler persists this into
# its slot buffer via ``slot_buffer[slot_mapping] = routing_data``.
# ``None`` when ``enable_return_routed_experts`` is off.
routed_experts: RoutedExpertsLists | None = None
aux_output_connector_output: dict[str, AuxRequestOutput] | None = None
# ``None`` when ``return_sampling_mask`` is off.
sampling_masks: SamplingMaskLists | None = None
+13 -9
View File
@@ -15,12 +15,12 @@ from vllm.v1.outputs import (
LogprobsTensors,
ModelRunnerOutput,
PoolerOutput,
RoutedExpertsTensors,
)
from vllm.v1.worker.gpu.sample.output import SamplerOutput, SamplingMaskTensors
from vllm.v1.worker.utils import raise_if_nan_logits
if TYPE_CHECKING:
from vllm.distributed.aux_output_connector.worker import PendingAuxOutput
from vllm.v1.worker.gpu.input_batch import InputBatch
@@ -120,8 +120,8 @@ class AsyncOutput(AsyncModelRunnerOutput):
num_sampled_tokens: torch.Tensor,
main_stream: torch.cuda.Stream,
copy_stream: torch.cuda.Stream,
check_ep_fault: bool = False,
routed_experts: RoutedExpertsTensors | None = None,
check_ep_fault: bool,
pending_aux_output: "PendingAuxOutput | None",
):
# NOTE(woosuk): We must retain references to the GPU tensors,
# as the copy operations are performed on a different CUDA stream than
@@ -129,7 +129,7 @@ class AsyncOutput(AsyncModelRunnerOutput):
self.model_runner_output = model_runner_output
self.sampler_output = sampler_output
self.num_sampled_tokens = num_sampled_tokens
self.routed_experts = routed_experts
self.pending_aux_output = pending_aux_output
# Blocking (sleep) event to avoid busy-polling the CUDA driver lock.
self.copy_event = torch.cuda.Event(blocking=True)
self._has_fault: torch.Tensor | None = None
@@ -152,13 +152,15 @@ class AsyncOutput(AsyncModelRunnerOutput):
self.sampling_mask_tensors = (
sampler_output.sampling_mask_tensors.to_cpu_nonblocking()
)
self.routed_experts_cpu: RoutedExpertsTensors | None = None
if routed_experts is not None:
self.routed_experts_cpu = routed_experts.to_cpu_nonblocking()
self.prompt_logprobs_dict = {
k: v.to_cpu_nonblocking() if v is not None else None
for k, v in self.model_runner_output.prompt_logprobs_dict.items()
}
if self.pending_aux_output is not None:
self.pending_aux_output.enqueue_cpu_copy(
num_sampled=self.num_sampled_tokens_np,
num_rejected=async_copy_to_np(sampler_output.num_rejected),
)
if check_ep_fault:
has_fault = get_ep_all2all_manager().query_fault()
self._has_fault = has_fault.to("cpu", non_blocking=True)
@@ -192,8 +194,10 @@ class AsyncOutput(AsyncModelRunnerOutput):
if self.logprobs_tensors is not None:
self.model_runner_output.logprobs = self.logprobs_tensors.tolists()
self.model_runner_output.prompt_logprobs_dict = self.prompt_logprobs_dict
if self.routed_experts_cpu is not None:
self.model_runner_output.routed_experts = self.routed_experts_cpu.tolists()
if self.pending_aux_output is not None:
self.model_runner_output.aux_output_connector_output = (
self.pending_aux_output.process_output()
)
if self._has_fault is not None and self._has_fault.item():
mask = get_ep_all2all_manager().query_active_mask()
+24 -24
View File
@@ -32,6 +32,10 @@ from vllm.compilation.counter import compilation_counter
from vllm.compilation.cuda_graph import CUDAGraphStat
from vllm.config import VllmConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.distributed.aux_output_connector.worker import (
AuxOutputWorkerConnector,
get_aux_output_connector,
)
from vllm.distributed.parallel_state import (
get_dcp_group,
get_pp_group,
@@ -39,10 +43,6 @@ from vllm.distributed.parallel_state import (
from vllm.forward_context import BatchDescriptor, set_forward_context
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.all2all_utils import get_ep_all2all_manager
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
RoutedExpertsCapturer,
bind_routed_experts_capturer,
)
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
initialize_mamba_ssu_backend,
)
@@ -73,7 +73,6 @@ from vllm.v1.outputs import (
DraftTokenIds,
ECConnectorOutput,
ModelRunnerOutput,
RoutedExpertsTensors,
)
from vllm.v1.watermarking import create_watermarker
from vllm.v1.watermarking.gpu_sampler import GPUWatermarkSampler
@@ -348,7 +347,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# Expert parallelism load balancer.
self.eplb = EPLBController(self.parallel_config, self.device)
self.routed_experts_capturer: RoutedExpertsCapturer | None = None
# The AuxOutput Connector owns R3 capture, copying, and storage.
self.aux_output_connector: AuxOutputWorkerConnector | None = None
set_offloader(create_offloader(self.vllm_config.offload_config))
@@ -356,15 +356,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.max_model_len = max_model_len
self.req_states.max_model_len = max_model_len
def init_routed_experts_capturer(self) -> None:
"""Initialize target-model capture on every participating worker."""
self.routed_experts_capturer = RoutedExpertsCapturer(
max_num_batched_tokens=self.max_num_tokens,
vllm_config=self.vllm_config,
kv_cache_config=self.kv_cache_config,
)
bind_routed_experts_capturer(self.model, self.routed_experts_capturer)
def get_supported_tasks(self) -> tuple[SupportedTask, ...]:
tasks: list[SupportedTask] = []
if self.model_config.runner_type == "generate":
@@ -756,6 +747,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
else:
self.kv_connector = get_kv_connector(self.vllm_config, kv_caches_dict)
# AuxOutput connector requires resolved kv_cache_config.
if self.vllm_config.aux_output_config.enabled:
self.aux_output_connector = get_aux_output_connector(
self.model, self.vllm_config, kv_cache_config
)
def _init_kv_zero_meta(self) -> None:
"""Build KV-block zeroing metadata; invoked from gpu_worker."""
self.kv_block_zeroer = KVBlockZeroer(
@@ -1648,6 +1645,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.add_requests(scheduler_output)
self.update_requests(scheduler_output)
self.block_tables.apply_staged_writes()
if self.aux_output_connector is not None:
# Register this step before the GPU forward.
self.aux_output_connector.begin_step(
scheduler_output.aux_output_connector_metadata
)
if scheduler_output.total_num_scheduled_tokens == 0:
# No need to run the model.
empty_output = self.kv_connector.no_forward(scheduler_output)
@@ -1969,11 +1971,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
aux_hidden_states = None
output_intermediate_tensors = model_output
routed_experts = None
if not dummy_run and (capturer := self.routed_experts_capturer) is not None:
assert slot_mappings is not None
routed_experts = capturer.get_routed_experts(slot_mappings, num_toks)
finished_req_ids = scheduler_output.finished_req_ids
self.execute_model_state = ExecuteModelState(
input_batch=input_batch,
@@ -1984,7 +1981,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
dp_sync=dp_sync,
finished_req_ids=finished_req_ids,
ec_connector_output=ec_connector_output,
routed_experts=routed_experts,
cudagraph_stats=cudagraph_stats,
)
@@ -2014,7 +2010,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
dp_sync = self.execute_model_state.dp_sync
finished_req_ids = self.execute_model_state.finished_req_ids
ec_connector_output = self.execute_model_state.ec_connector_output
routed_experts = self.execute_model_state.routed_experts
cudagraph_stats = self.execute_model_state.cudagraph_stats
self.execute_model_state = None
@@ -2083,6 +2078,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
prompt_logprobs_dict=prompt_logprobs_dict, # type: ignore[arg-type]
cudagraph_stats=cudagraph_stats,
)
pending_aux_output = None
if self.aux_output_connector is not None:
pending_aux_output = self.aux_output_connector.prepare_output(input_batch)
# Start async output copy here so that it can overlap with speculator proposal.
async_output = AsyncOutput(
model_runner_output=model_runner_output,
@@ -2091,7 +2090,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
main_stream=self.main_stream,
copy_stream=self.output_copy_stream,
check_ep_fault=self.check_ep_fault,
routed_experts=routed_experts,
pending_aux_output=pending_aux_output,
)
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None
@@ -2244,6 +2243,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
"""Release GPU tensors (model weights, KV caches, workspace) so that
memory is reclaimable when running in the same process."""
torch.accelerator.synchronize()
if self.aux_output_connector is not None:
self.aux_output_connector.close()
self.cudagraph_manager = None
self.fast_prefill = None
if hasattr(self, "kv_caches"):
@@ -2316,7 +2317,6 @@ class ExecuteModelState(NamedTuple):
dp_sync: DPSyncState | None
finished_req_ids: set[str]
ec_connector_output: ECConnectorOutput | None
routed_experts: RoutedExpertsTensors | None
cudagraph_stats: CUDAGraphStat | None
-2
View File
@@ -617,8 +617,6 @@ def gather_sampler_output(
device=device,
)
if local_output is not None:
assert local_output.num_sampled is not None
assert local_output.num_rejected is not None
assert not gather_num_nans or local_output.num_nans is not None
num_src_cols = min(
local_output.sampled_token_ids.shape[1], max_num_logits_per_req
+2 -2
View File
@@ -17,8 +17,8 @@ class SamplerOutput:
sampled_token_ids: torch.Tensor
logprobs_tensors: LogprobsTensors | None
num_nans: torch.Tensor | None
num_sampled: torch.Tensor | None
num_rejected: torch.Tensor | None = None
num_sampled: torch.Tensor
num_rejected: torch.Tensor
sampling_mask_tensors: SamplingMaskTensors | None = None
-127
View File
@@ -61,10 +61,6 @@ from vllm.lora.layers import BaseLayerWithLoRA, LoRAMapping, LoRAMappingType
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.fused_moe.all2all_utils import get_ep_all2all_manager
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
RoutedExpertsCapturer,
bind_routed_experts_capturer,
)
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFuncsByType
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
initialize_mamba_ssu_backend,
@@ -176,8 +172,6 @@ from vllm.v1.outputs import (
LogprobsTensors,
ModelRunnerOutput,
PoolerOutput,
RoutedExpertsLists,
RoutedExpertsTensors,
SamplerOutput,
make_empty_encoder_model_runner_output,
)
@@ -293,7 +287,6 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput):
invalid_req_indices: list[int],
async_output_copy_stream: torch.cuda.Stream,
vocab_size: int,
routed_experts: RoutedExpertsTensors | None = None,
check_ep_fault: bool = False,
num_nans: torch.Tensor | None = None,
):
@@ -309,7 +302,6 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput):
self._sampled_token_ids = sampled_token_ids
self.vocab_size = vocab_size
self._logprobs_tensors = logprobs_tensors
self._routed_experts = routed_experts
self._num_nans = num_nans
self._has_fault: torch.Tensor | None = None
@@ -325,11 +317,6 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput):
if self._logprobs_tensors
else None
)
self._routed_experts_cpu = (
self._routed_experts.to_cpu_nonblocking()
if self._routed_experts is not None
else None
)
self._num_nans_cpu = (
self._num_nans.to("cpu", non_blocking=True)
if self._num_nans is not None
@@ -370,10 +357,6 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput):
output.sampled_token_ids = valid_sampled_token_ids
output.logprobs = logprobs_lists
if self._routed_experts_cpu is not None:
output.routed_experts = self._routed_experts_cpu.tolists()
del self._routed_experts
if self._num_nans_cpu is not None:
output.num_nans_in_logits = nans_to_dict(
self._num_nans_cpu.tolist(), output.req_id_to_index
@@ -537,9 +520,6 @@ class GPUModelRunner(
# These will be overridden in load_model()
self.is_multimodal_pruning_enabled = False
self.requires_sequential_video_encoding = False
# Set to True after init_routed_experts_capturer() completes.
# Prevents routed experts code from running during profiling/dummy run.
self.routed_experts_initialized = False
self.max_model_len = model_config.max_model_len
# Always set to false after the first forward pass
@@ -2360,18 +2340,6 @@ class GPUModelRunner(
block_table_gid_0 = _get_block_table(0)
slot_mapping_gid_0 = slot_mappings[0]
if self.routed_experts_initialized:
# Copy this step's attention slot_mapping into our private
# device buffer. The shared ``slot_mappings[attn_gid]`` is
# owned by the attention block table and will be overwritten
# by the next ``_prepare_inputs``; we need a stable snapshot
# because the async D2H may still be in flight on the copy
# stream when the next step runs.
slot_mapping_attn = slot_mappings[self.routed_experts_capturer.attn_gid]
self.routed_experts_slot_mapping_device[:num_tokens].copy_(
slot_mapping_attn[:num_tokens]
)
num_computed_tokens_cpu = self.input_batch.num_computed_tokens_cpu_tensor[
:num_reqs_padded
]
@@ -3745,20 +3713,6 @@ class GPUModelRunner(
invalid_req_indices = []
logprobs_lists = None
if not self.use_async_scheduling:
# Sync scheduling: issue routed experts D2H into the pinned
# CPU buffer BEFORE ``_to_list`` below. ``_to_list`` does
# ``event.synchronize()`` on the async copy stream which
# waits for every D2H queued on the default stream since
# the last sync, so this enqueue is naturally covered
# without requiring its own synchronize.
if self.routed_experts_initialized:
buf = self.routed_experts_capturer.get_device_buffer()
total = scheduler_output.total_num_scheduled_tokens
self.routed_experts_cpu[:total].copy_(buf[:total], non_blocking=True)
self.routed_experts_slot_mapping_cpu[:total].copy_(
self.routed_experts_slot_mapping_device[:total],
non_blocking=True,
)
with gpu_sync_allowed():
# Get the valid generated tokens.
max_gen_len = sampled_token_ids.shape[-1]
@@ -4802,39 +4756,14 @@ class GPUModelRunner(
else None,
num_nans_in_logits=num_nans_in_logits,
cudagraph_stats=cudagraph_stats,
routed_experts=None,
)
if not self.use_async_scheduling:
if self.routed_experts_initialized:
# Sync path: D2H was issued in ``_bookkeeping_sync`` and
# synchronized by ``_to_list``'s event.synchronize(), so
# the pinned buffers are ready to be wrapped as numpy.
total = scheduler_output.total_num_scheduled_tokens
output.routed_experts = RoutedExpertsLists(
routing_data=self.routed_experts_cpu[:total].numpy(),
slot_mapping=self.routed_experts_slot_mapping_cpu[:total].numpy(),
)
return output
with record_function_or_nullcontext(
"gpu_model_runner: AsyncGPUModelRunnerOutput"
):
# Async path: produce a device-side snapshot that the async
# copy stream can D2H later. Both tensors must be private
# clones because:
# - ``routing_data`` source is the shared capturer buffer,
# which the next forward overwrites on the default stream.
# - ``slot_mapping`` source is our own
# ``routed_experts_slot_mapping_device``, which the
# next ``_prepare_inputs`` overwrites on the default
# stream while the D2H is still pending on the copy
# stream.
# Without clones, the copy stream would read torn data.
routed_experts_snapshot = self.get_routed_experts(
scheduler_output.total_num_scheduled_tokens
)
async_output = AsyncGPUModelRunnerOutput(
model_runner_output=output,
sampled_token_ids=sampler_output.sampled_token_ids,
@@ -4842,7 +4771,6 @@ class GPUModelRunner(
invalid_req_indices=invalid_req_indices,
async_output_copy_stream=self._get_or_create_async_output_copy_stream(),
vocab_size=self.input_batch.vocab_size,
routed_experts=routed_experts_snapshot,
check_ep_fault=self.check_ep_fault,
num_nans=num_nans_device,
)
@@ -7468,61 +7396,6 @@ class GPUModelRunner(
kv_transfer_group.register_kv_caches(kv_caches)
kv_transfer_group.set_host_xfer_buffer_ops(copy_kv_blocks)
def get_routed_experts(
self,
num_tokens: int,
) -> RoutedExpertsTensors | None:
if not self.routed_experts_initialized:
return None
device_buffer = self.routed_experts_capturer.get_device_buffer()
return RoutedExpertsTensors(
routing_data=device_buffer[:num_tokens].clone(),
slot_mapping=self.routed_experts_slot_mapping_device[:num_tokens].clone(),
)
def init_routed_experts_capturer(self):
logger.info(
"Initializing routed experts capturer, enable_return_routed_experts: %s",
self.model_config.enable_return_routed_experts,
)
self.routed_experts_capturer = RoutedExpertsCapturer(
max_num_batched_tokens=self.scheduler_config.max_num_batched_tokens,
vllm_config=self.vllm_config,
kv_cache_config=self.kv_cache_config,
)
bind_routed_experts_capturer(self.model, self.routed_experts_capturer)
# Pinned CPU buffer for non-blocking D2H of ``routing_data`` on
# the sync scheduling path. Shape / dtype mirror the device
# capturer exactly so ``copy_`` is a straight memcpy.
self.routed_experts_cpu = torch.empty(
self.routed_experts_capturer.device_buffer.shape,
dtype=self.routed_experts_capturer.device_buffer.dtype,
device="cpu",
pin_memory=PIN_MEMORY,
)
# ``slot_mapping`` dtype is fixed to int64 by
# ``block_table.slot_mapping``; we mirror that here.
max_tokens = self.scheduler_config.max_num_batched_tokens
self.routed_experts_slot_mapping_cpu = torch.empty(
(max_tokens,),
dtype=torch.int64,
device="cpu",
pin_memory=PIN_MEMORY,
)
# Private device buffer so the shared ``block_table.slot_mapping``
# can be overwritten by the next ``_prepare_inputs`` while the
# D2H is still pending on the copy stream. Written in
# ``_prepare_inputs``, read in ``_bookkeeping_sync`` (sync path)
# or cloned into a snapshot (async path).
self.routed_experts_slot_mapping_device = torch.empty(
(max_tokens,),
dtype=torch.int64,
device=self.device,
)
self.routed_experts_initialized = True
def may_add_encoder_only_layers_to_kv_cache_config(self) -> None:
"""Add encoder-only layers to the KV cache config."""
block_size = self.vllm_config.cache_config.block_size
-3
View File
@@ -761,9 +761,6 @@ class Worker(WorkerBase):
),
)
if self.model_config.enable_return_routed_experts:
self.model_runner.init_routed_experts_capturer()
# Build KV-zero metadata outside the CuMem pool so the bookkeeping
# GPU tensors (seg_addrs, block-id buffers) use the standard PyTorch
# allocator and are not discarded during sleep/wake cycles.