[AuxOutput] Add block-keyed storage for routed-expert outputs (#45635)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
aoshen02
OpenAI Codex
Roger Wang
Nick Hill
Codex
parent
0aee727ff6
commit
0b7f11a1ee
@@ -0,0 +1,119 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config import AuxOutputConfig, VllmConfig
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
def _config(
|
||||
*,
|
||||
use_v2: bool = True,
|
||||
pp: int = 1,
|
||||
dcp: int = 1,
|
||||
pcp: int = 1,
|
||||
connector: str | None = None,
|
||||
runner_type: str = "generate",
|
||||
is_moe: bool = True,
|
||||
sliding_window: int | None = None,
|
||||
attention_chunk_size: int | None = None,
|
||||
enable_prefix_caching: bool = True,
|
||||
adaptive_verification: bool = False,
|
||||
):
|
||||
return SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
runner_type=runner_type,
|
||||
is_moe=is_moe,
|
||||
get_sliding_window=lambda: sliding_window,
|
||||
attention_chunk_size=attention_chunk_size,
|
||||
),
|
||||
use_v2_model_runner=use_v2,
|
||||
parallel_config=SimpleNamespace(
|
||||
pipeline_parallel_size=pp,
|
||||
decode_context_parallel_size=dcp,
|
||||
prefill_context_parallel_size=pcp,
|
||||
),
|
||||
aux_output_config=AuxOutputConfig(enable_return_routed_experts=True),
|
||||
cache_config=SimpleNamespace(
|
||||
enable_prefix_caching=enable_prefix_caching,
|
||||
),
|
||||
speculative_config=(
|
||||
SimpleNamespace(enable_adaptive_verification=True)
|
||||
if adaptive_verification
|
||||
else None
|
||||
),
|
||||
kv_transfer_config=(
|
||||
None
|
||||
if connector is None
|
||||
else SimpleNamespace(
|
||||
is_kv_transfer_instance=True,
|
||||
kv_connector=connector,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_aux_output_config_defaults():
|
||||
config = AuxOutputConfig()
|
||||
|
||||
assert not config.enabled
|
||||
assert not config.enable_return_routed_experts
|
||||
assert config.max_bytes is None
|
||||
|
||||
|
||||
def test_aux_output_capture_changes_compilation_hash():
|
||||
disabled = AuxOutputConfig()
|
||||
enabled = AuxOutputConfig(enable_return_routed_experts=True)
|
||||
|
||||
assert disabled.compute_hash() != enabled.compute_hash()
|
||||
|
||||
|
||||
def test_legacy_routed_experts_flag_updates_aux_output_config():
|
||||
args = EngineArgs(enable_return_routed_experts=True)
|
||||
|
||||
assert args.aux_output_config.enabled
|
||||
assert args.aux_output_config.enable_return_routed_experts
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kwargs", "error"),
|
||||
[
|
||||
({"use_v2": False}, "requires Model Runner V2"),
|
||||
({"runner_type": "pooling"}, "only supports generate runners"),
|
||||
({"is_moe": False}, "only supports MoE models"),
|
||||
({"enable_prefix_caching": False}, "requires prefix caching"),
|
||||
(
|
||||
{"adaptive_verification": True},
|
||||
"adaptive speculative verification",
|
||||
),
|
||||
({"pp": 2}, "pipeline parallelism"),
|
||||
({"dcp": 2}, "context parallelism"),
|
||||
({"pcp": 2}, "context parallelism"),
|
||||
({"connector": "MooncakeConnector"}, "incompatible with KV connectors"),
|
||||
],
|
||||
)
|
||||
def test_aux_output_connector_rejects_unsupported_configuration(kwargs, error):
|
||||
with pytest.raises(ValueError, match=error):
|
||||
VllmConfig._verify_aux_output_compatibility(_config(**kwargs))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"kwargs",
|
||||
[{"sliding_window": 4096}, {"attention_chunk_size": 4096}],
|
||||
)
|
||||
def test_aux_output_config_defers_attention_layout_to_kv_config(kwargs):
|
||||
VllmConfig._verify_aux_output_compatibility(_config(**kwargs))
|
||||
|
||||
|
||||
def test_aux_output_guards_are_inactive_when_capture_is_disabled():
|
||||
config: Any = VllmConfig.__new__(VllmConfig)
|
||||
config.model_config = SimpleNamespace()
|
||||
config.aux_output_config = AuxOutputConfig()
|
||||
|
||||
config._verify_aux_output_compatibility()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -12,15 +12,13 @@ from ...utils import RemoteOpenAIServer
|
||||
MODEL_NAME = "TitanML/tiny-mixtral"
|
||||
|
||||
# tiny-mixtral config: 8 local experts, top-2 routing, 2 hidden layers.
|
||||
# The published config has sliding_window=4096, which produces
|
||||
# SlidingWindowSpec kv-cache groups; RoutedExpertsManager requires a
|
||||
# FullAttentionSpec group, so we override sliding_window=null below.
|
||||
# Its published sliding window is incompatible with AuxOutput retention.
|
||||
NUM_LOCAL_EXPERTS = 8
|
||||
NUM_EXPERTS_PER_TOK = 2
|
||||
NUM_HIDDEN_LAYERS = 2
|
||||
|
||||
|
||||
def assert_valid_routed_experts(encoded: str | None) -> None:
|
||||
def decode_routed_experts(encoded: str | None) -> np.ndarray:
|
||||
assert encoded is not None
|
||||
routed_experts = np.load(io.BytesIO(base64.b64decode(encoded)))
|
||||
assert routed_experts.ndim == 3
|
||||
@@ -30,6 +28,7 @@ def assert_valid_routed_experts(encoded: str | None) -> None:
|
||||
assert topk == NUM_EXPERTS_PER_TOK
|
||||
assert (routed_experts >= 0).all()
|
||||
assert (routed_experts < NUM_LOCAL_EXPERTS).all()
|
||||
return routed_experts
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
@@ -44,7 +43,11 @@ def server():
|
||||
"--hf-overrides",
|
||||
'{"sliding_window": null}',
|
||||
]
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME,
|
||||
args,
|
||||
env_dict={"VLLM_USE_V2_MODEL_RUNNER": "1"},
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@@ -63,4 +66,8 @@ async def test_routed_experts(server):
|
||||
choice = result.model_dump()["choices"][0]
|
||||
|
||||
assert choice["token_ids"] is not None
|
||||
assert_valid_routed_experts(choice["routed_experts"])
|
||||
assert result.usage is not None
|
||||
routed_experts = decode_routed_experts(choice["routed_experts"])
|
||||
assert len(routed_experts) == (
|
||||
result.usage.prompt_tokens + len(choice["token_ids"]) - 1
|
||||
)
|
||||
|
||||
@@ -15,9 +15,7 @@ MODEL_NAME = "TitanML/tiny-mixtral"
|
||||
GEN_ENDPOINT = "/inference/v1/generate"
|
||||
|
||||
# tiny-mixtral config: 8 local experts, top-2 routing, 2 hidden layers.
|
||||
# The published config has sliding_window=4096, which produces
|
||||
# SlidingWindowSpec kv-cache groups; RoutedExpertsManager requires a
|
||||
# FullAttentionSpec group, so we override sliding_window=null below.
|
||||
# Its published sliding window is incompatible with AuxOutput retention.
|
||||
NUM_LOCAL_EXPERTS = 8
|
||||
NUM_EXPERTS_PER_TOK = 2
|
||||
NUM_HIDDEN_LAYERS = 2
|
||||
@@ -36,7 +34,13 @@ def server():
|
||||
'{"sliding_window": null}',
|
||||
"--enable-scale-out",
|
||||
]
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME,
|
||||
args,
|
||||
env_dict={
|
||||
"VLLM_USE_V2_MODEL_RUNNER": "1",
|
||||
},
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@@ -56,9 +60,10 @@ async def client(server: RemoteOpenAIServer):
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_routed_experts(client):
|
||||
"""Test that /inference/v1/generate returns routed_experts when enabled."""
|
||||
prompt_token_ids = [1, 2, 3]
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": [1, 2, 3],
|
||||
"token_ids": prompt_token_ids,
|
||||
"sampling_params": {"max_tokens": 10, "temperature": 0.0},
|
||||
"stream": False,
|
||||
}
|
||||
@@ -80,3 +85,4 @@ async def test_generate_routed_experts(client):
|
||||
assert topk == NUM_EXPERTS_PER_TOK
|
||||
assert (routed_experts >= 0).all()
|
||||
assert (routed_experts < NUM_LOCAL_EXPERTS).all()
|
||||
assert num_tokens == len(prompt_token_ids) + len(choice["token_ids"]) - 1
|
||||
|
||||
@@ -7,26 +7,19 @@ from unittest.mock import Mock, patch
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CompilationMode
|
||||
from vllm.distributed.eplb.eplb_state import EplbLayerState
|
||||
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
RoutedExpertsCapturer,
|
||||
RoutedExpertsManager,
|
||||
bind_routed_experts_capturer,
|
||||
get_routed_experts_attn_gid,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.base_router import BaseRouter
|
||||
from vllm.transformers_utils.model_arch_config_convertor import (
|
||||
ModelArchConfigConvertorBase,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowMLASpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
|
||||
@@ -42,6 +35,7 @@ def _capturer_with_buffer(
|
||||
num_experts_per_tok: int = 2,
|
||||
dp_rank: int = 0,
|
||||
tp_size: int = 1,
|
||||
dtype: torch.dtype = torch.int32,
|
||||
) -> RoutedExpertsCapturer:
|
||||
# Bypass __init__ so the test can use a CPU buffer and skip the
|
||||
# VllmConfig dependency. The CUDA device-tensor allocation in the
|
||||
@@ -49,10 +43,11 @@ def _capturer_with_buffer(
|
||||
c = RoutedExpertsCapturer.__new__(RoutedExpertsCapturer)
|
||||
c.dp_rank = dp_rank
|
||||
c.tp_size = tp_size
|
||||
c.output_dtype = torch.uint8
|
||||
c.device_buffer = torch.full(
|
||||
(max_tokens, num_layers, num_experts_per_tok),
|
||||
-1,
|
||||
dtype=torch.int32,
|
||||
dtype=dtype,
|
||||
)
|
||||
return c
|
||||
|
||||
@@ -88,74 +83,36 @@ def _make_modular_routed_experts():
|
||||
)
|
||||
|
||||
|
||||
def _make_model_config(hf_config):
|
||||
num_experts_per_token = ModelArchConfigConvertorBase(
|
||||
hf_config, hf_config
|
||||
).get_num_experts_per_token()
|
||||
model_config = SimpleNamespace(
|
||||
hf_text_config=hf_config,
|
||||
model_arch_config=SimpleNamespace(
|
||||
num_experts_per_token=num_experts_per_token,
|
||||
def _full_attention_kv_group(
|
||||
spec_type: type[FullAttentionSpec] = FullAttentionSpec,
|
||||
) -> KVCacheGroupSpec:
|
||||
full_attention = spec_type(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
return KVCacheGroupSpec(
|
||||
["layer"],
|
||||
UniformTypeKVCacheSpecs(
|
||||
block_size=16,
|
||||
kv_cache_specs={"layer": full_attention},
|
||||
),
|
||||
)
|
||||
model_config.get_num_experts = lambda: hf_config.num_experts
|
||||
model_config.get_num_experts_per_tok = lambda: (
|
||||
ModelConfig.get_num_experts_per_tok(model_config)
|
||||
)
|
||||
model_config.get_total_num_hidden_layers = lambda: hf_config.num_hidden_layers
|
||||
return model_config
|
||||
|
||||
|
||||
def test_routed_experts_manager_uses_gemma4_top_k_experts():
|
||||
hf_config = SimpleNamespace(
|
||||
num_experts=8,
|
||||
top_k_experts=2,
|
||||
num_hidden_layers=3,
|
||||
)
|
||||
vllm_config = SimpleNamespace(model_config=_make_model_config(hf_config))
|
||||
kv_cache_spec = FullAttentionSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=2,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[KVCacheGroupSpec(["layer"], kv_cache_spec)],
|
||||
)
|
||||
@pytest.mark.parametrize("eplb_enabled", [False, True])
|
||||
def test_base_router_capture_pre_eplb_mapping(eplb_enabled):
|
||||
eplb_state = None
|
||||
if eplb_enabled:
|
||||
eplb_state = EplbLayerState()
|
||||
eplb_state.expert_load_view = torch.zeros(32, dtype=torch.int64)
|
||||
eplb_state.logical_to_physical_map = torch.arange(32).view(32, 1)
|
||||
eplb_state.logical_replica_count = torch.ones(32, dtype=torch.int64)
|
||||
eplb_state.should_record_tensor = torch.ones((), dtype=torch.bool)
|
||||
eplb_state.num_unpadded_tokens_tensors = [torch.tensor(0, dtype=torch.int32)]
|
||||
router = _make_router(eplb_state)
|
||||
|
||||
manager = RoutedExpertsManager(vllm_config, kv_cache_config)
|
||||
|
||||
assert manager.routed_experts_by_slot.shape == (8, 3, 2)
|
||||
|
||||
|
||||
def test_routed_experts_manager_uses_kimi_k3_experts_per_token():
|
||||
hf_config = SimpleNamespace(
|
||||
num_experts=8,
|
||||
num_experts_per_token=2,
|
||||
num_hidden_layers=3,
|
||||
)
|
||||
vllm_config = SimpleNamespace(model_config=_make_model_config(hf_config))
|
||||
kv_cache_spec = FullAttentionSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=2,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[KVCacheGroupSpec(["layer"], kv_cache_spec)],
|
||||
)
|
||||
|
||||
manager = RoutedExpertsManager(vllm_config, kv_cache_config)
|
||||
|
||||
assert manager.routed_experts_by_slot.shape == (8, 3, 2)
|
||||
|
||||
|
||||
def test_base_router_capture_pre_eplb_mapping():
|
||||
router = _make_router()
|
||||
captured = []
|
||||
|
||||
def capture_fn(ids):
|
||||
@@ -173,34 +130,7 @@ def test_base_router_capture_pre_eplb_mapping():
|
||||
assert torch.equal(topk_ids, torch.tensor([[11, 12], [13, 14]]))
|
||||
|
||||
|
||||
def test_base_router_capture_with_eplb_enabled():
|
||||
eplb_state = EplbLayerState()
|
||||
eplb_state.expert_load_view = torch.zeros(32, dtype=torch.int64)
|
||||
eplb_state.logical_to_physical_map = torch.arange(32).view(32, 1)
|
||||
eplb_state.logical_replica_count = torch.ones(32, dtype=torch.int64)
|
||||
eplb_state.should_record_tensor = torch.ones((), dtype=torch.bool)
|
||||
eplb_state.num_unpadded_tokens_tensors = [torch.tensor(0, dtype=torch.int32)]
|
||||
router = _make_router(eplb_state=eplb_state)
|
||||
|
||||
captured = []
|
||||
|
||||
def capture_fn(ids):
|
||||
captured.append(ids.clone())
|
||||
|
||||
router.set_capture_fn(capture_fn)
|
||||
_, topk_ids = router.select_experts(
|
||||
hidden_states=torch.empty(1),
|
||||
router_logits=torch.empty(1),
|
||||
)
|
||||
|
||||
assert len(captured) == 1
|
||||
# Capture should see logical ids pre-EPLB mapping.
|
||||
assert torch.equal(captured[0], torch.tensor([[1, 2], [3, 4]]))
|
||||
# Our DummyRouter mapping adds +10.
|
||||
assert torch.equal(topk_ids, torch.tensor([[11, 12], [13, 14]]))
|
||||
|
||||
|
||||
def test_public_binding_only_visits_target_model(monkeypatch):
|
||||
def test_public_binding_binds_target_model_router(monkeypatch):
|
||||
class DummyFusedMoE:
|
||||
def __init__(self, layer_id):
|
||||
self.layer_id = layer_id
|
||||
@@ -208,7 +138,6 @@ def test_public_binding_only_visits_target_model(monkeypatch):
|
||||
self._quant_method = _make_modular_routed_experts().quant_method
|
||||
|
||||
target_module = DummyFusedMoE(layer_id=7)
|
||||
draft_module = DummyFusedMoE(layer_id=0)
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.layer as fused_moe_layer
|
||||
|
||||
@@ -221,12 +150,30 @@ def test_public_binding_only_visits_target_model(monkeypatch):
|
||||
)
|
||||
|
||||
assert target_module.router.capture_fn is not None
|
||||
assert draft_module.router.capture_fn is None
|
||||
topk_ids = torch.tensor([[5, 6]])
|
||||
target_module.router.capture_fn(topk_ids)
|
||||
assert calls == [(7, topk_ids)]
|
||||
|
||||
|
||||
def test_public_binding_supports_direct_capture_source():
|
||||
class DirectCaptureSource(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.layer_id = 3
|
||||
self.capture_fn = None
|
||||
|
||||
source = DirectCaptureSource()
|
||||
calls = []
|
||||
capturer = types.SimpleNamespace(capture=lambda *args: calls.append(args))
|
||||
|
||||
bind_routed_experts_capturer(source, capturer)
|
||||
|
||||
assert source.capture_fn is not None
|
||||
topk_ids = torch.tensor([[1, 2]])
|
||||
source.capture_fn(topk_ids)
|
||||
assert calls == [(3, topk_ids)]
|
||||
|
||||
|
||||
def test_public_binding_rejects_monolithic_without_replay_support(monkeypatch):
|
||||
class DummyFusedMoE:
|
||||
def __init__(self):
|
||||
@@ -277,6 +224,51 @@ def test_routed_experts_capturer_single_dp_no_metadata():
|
||||
assert capturer.device_buffer[3, 0, 0].item() == -1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_experts", "dtype_name", "torch_dtype"),
|
||||
[(256, "uint8", torch.uint8), (257, "uint16", torch.uint16)],
|
||||
)
|
||||
def test_routed_experts_capturer_exposes_output_profile(
|
||||
monkeypatch, num_experts, dtype_name, torch_dtype
|
||||
):
|
||||
import vllm.model_executor.layers.fused_moe.routed_experts_capturer as module
|
||||
|
||||
monkeypatch.setattr(module, "current_platform", SimpleNamespace(device_type="cpu"))
|
||||
config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
get_total_num_hidden_layers=lambda: 3,
|
||||
get_num_experts=lambda: num_experts,
|
||||
get_num_experts_per_tok=lambda: 2,
|
||||
hf_text_config=SimpleNamespace(model_type="test"),
|
||||
),
|
||||
parallel_config=SimpleNamespace(data_parallel_rank=0, tensor_parallel_size=1),
|
||||
)
|
||||
|
||||
capturer = RoutedExpertsCapturer(8, config)
|
||||
|
||||
assert capturer.shape_per_token == (3, 2)
|
||||
assert capturer.output_dtype_name == dtype_name
|
||||
assert capturer.output_dtype == torch_dtype
|
||||
assert capturer.device_buffer.shape == (8, 3, 2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("output_dtype", [torch.uint8, torch.uint16])
|
||||
def test_routed_experts_capturer_narrows_snapshot(output_dtype):
|
||||
capturer = _capturer_with_buffer(dtype=torch.int32)
|
||||
capturer.output_dtype = output_dtype
|
||||
topk = torch.tensor([[1, 2], [254, 255]], dtype=torch.int64)
|
||||
ctx = SimpleNamespace(dp_metadata=None)
|
||||
with patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx):
|
||||
capturer.capture(layer_id=0, topk_ids=topk)
|
||||
|
||||
output = capturer.snapshot_routing_data(2)
|
||||
assert capturer.device_buffer.dtype == torch.int32
|
||||
assert capturer.device_buffer[:2, 0, :].tolist() == topk.tolist()
|
||||
assert output.dtype == output_dtype
|
||||
assert output[:, 0, :].tolist() == topk.tolist()
|
||||
assert output.data_ptr() != capturer.device_buffer.data_ptr()
|
||||
|
||||
|
||||
def test_routed_experts_capturer_dp_naive_concatenated_all_ranks():
|
||||
"""N == sum(num_tokens_dp): slice this rank's segment from concatenated topk."""
|
||||
capturer = _capturer_with_buffer(dp_rank=1)
|
||||
@@ -307,6 +299,47 @@ def test_routed_experts_capturer_dp_modular_local_tokens():
|
||||
assert torch.equal(capturer.device_buffer[:3, 0, :], topk)
|
||||
|
||||
|
||||
def test_routed_experts_capturer_dp_ep_gathered_shards():
|
||||
capturer = _capturer_with_buffer(dp_rank=1, tp_size=2)
|
||||
num_tokens_dp = torch.tensor([3, 2], dtype=torch.int32)
|
||||
ctx = SimpleNamespace(
|
||||
dp_metadata=SimpleNamespace(
|
||||
num_tokens_across_dp_cpu=num_tokens_dp,
|
||||
local_sizes=[2, 2, 1, 1],
|
||||
)
|
||||
)
|
||||
topk = torch.tensor(
|
||||
[[0, 1], [2, 3], [4, 5], [-1, -1], [10, 11], [12, 13]],
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
with patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx):
|
||||
capturer.capture(layer_id=0, topk_ids=topk)
|
||||
|
||||
assert torch.equal(capturer.device_buffer[:2, 0], topk[4:])
|
||||
|
||||
|
||||
def test_routed_experts_capturer_sp_modular_gathers_tp_shards():
|
||||
capturer = _capturer_with_buffer(dp_rank=1, tp_size=2)
|
||||
num_tokens_dp = torch.tensor([2, 3], dtype=torch.int32)
|
||||
ctx = SimpleNamespace(
|
||||
dp_metadata=SimpleNamespace(num_tokens_across_dp_cpu=num_tokens_dp)
|
||||
)
|
||||
local_shard = torch.tensor([[10, 11], [12, 13]], dtype=torch.int32)
|
||||
gathered = torch.tensor([[10, 11], [12, 13], [14, 15], [-1, -1]], dtype=torch.int32)
|
||||
tp_group = Mock()
|
||||
tp_group.all_gather.return_value = gathered
|
||||
|
||||
with (
|
||||
patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx),
|
||||
patch(f"{_REC_MODULE}.get_tp_group", return_value=tp_group),
|
||||
):
|
||||
capturer.capture(layer_id=0, topk_ids=local_shard)
|
||||
|
||||
tp_group.all_gather.assert_called_once_with(local_shard, dim=0)
|
||||
assert torch.equal(capturer.device_buffer[:3, 0], gathered[:3])
|
||||
|
||||
|
||||
def test_routed_experts_capturer_dp_unexpected_batch_raises():
|
||||
"""Mismatch between topk batch dim and DP layout: fail fast."""
|
||||
capturer = _capturer_with_buffer(dp_rank=0)
|
||||
@@ -324,128 +357,119 @@ def test_routed_experts_capturer_dp_unexpected_batch_raises():
|
||||
assert capturer.device_buffer[0, 0, 0].item() == -1
|
||||
|
||||
|
||||
def test_routed_experts_attention_group_is_shared_and_fail_closed():
|
||||
"""Both sides key routing data by this gid, so it must skip non-full-attention
|
||||
groups rather than defaulting to 0, and fail closed when none exists."""
|
||||
common = dict(num_kv_heads=1, head_size=1, dtype=torch.float32)
|
||||
def test_get_aux_output_connector_passes_config(monkeypatch):
|
||||
import vllm.distributed.aux_output_connector.worker as aux_output_worker
|
||||
|
||||
connector = Mock()
|
||||
constructor = Mock(return_value=connector)
|
||||
monkeypatch.setattr(aux_output_worker, "AuxOutputWorkerConnector", constructor)
|
||||
config = SimpleNamespace(
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["swa_layer"],
|
||||
SlidingWindowMLASpec(block_size=4, sliding_window=8, **common),
|
||||
),
|
||||
KVCacheGroupSpec(["full_layer"], FullAttentionSpec(block_size=4, **common)),
|
||||
]
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=32)
|
||||
)
|
||||
assert get_routed_experts_attn_gid(config) == 1
|
||||
model = Mock()
|
||||
kv_cache_config = Mock()
|
||||
|
||||
with pytest.raises(ValueError, match="requires a full-attention KV cache group"):
|
||||
get_routed_experts_attn_gid(SimpleNamespace(kv_cache_groups=[]))
|
||||
result = aux_output_worker.get_aux_output_connector(model, config, kv_cache_config)
|
||||
|
||||
|
||||
def test_routed_experts_attention_group_unwraps_uniform_type_specs():
|
||||
"""DeepSeek-V4-shaped groups wrap their specs in ``UniformTypeKVCacheSpecs``.
|
||||
|
||||
The wrapper is not a ``FullAttentionSpec``, so a bare isinstance check finds
|
||||
no group and fails closed on every worker. Unwrap semantics themselves are
|
||||
covered by ``test_is_full_attention_spec_*`` in tests/v1/core.
|
||||
"""
|
||||
common = dict(num_kv_heads=1, head_size=1, dtype=torch.float32)
|
||||
swa_spec = SlidingWindowMLASpec(block_size=4, sliding_window=8, **common)
|
||||
mla_spec = MLAAttentionSpec(block_size=4, **common)
|
||||
config = SimpleNamespace(
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["swa_layer"],
|
||||
UniformTypeKVCacheSpecs(
|
||||
block_size=4, kv_cache_specs={"swa_layer": swa_spec}
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mla_layer"],
|
||||
UniformTypeKVCacheSpecs(
|
||||
block_size=4, kv_cache_specs={"mla_layer": mla_spec}
|
||||
),
|
||||
),
|
||||
]
|
||||
constructor.assert_called_once_with(
|
||||
model=model,
|
||||
kv_cache_config=kv_cache_config,
|
||||
vllm_config=config,
|
||||
)
|
||||
|
||||
assert get_routed_experts_attn_gid(config) == 1
|
||||
|
||||
swa_only = SimpleNamespace(kv_cache_groups=[config.kv_cache_groups[0]])
|
||||
with pytest.raises(ValueError, match="requires a full-attention KV cache group"):
|
||||
get_routed_experts_attn_gid(swa_only)
|
||||
assert result is connector
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
def test_mrv2_async_output_returns_existing_routed_experts_field():
|
||||
from vllm.v1.outputs import ModelRunnerOutput, RoutedExpertsTensors
|
||||
from vllm.v1.worker.gpu.async_utils import AsyncOutput
|
||||
from vllm.v1.worker.gpu.sample.output import SamplerOutput
|
||||
|
||||
routed_experts = RoutedExpertsTensors(
|
||||
routing_data=torch.arange(6, dtype=torch.int32, device="cuda").reshape(3, 1, 2),
|
||||
slot_mapping=torch.tensor([11, 12, 13], device="cuda"),
|
||||
)
|
||||
num_sampled = torch.tensor([1], dtype=torch.int32, device="cuda")
|
||||
sampler_output = SamplerOutput(
|
||||
sampled_token_ids=torch.tensor([[1]], device="cuda"),
|
||||
logprobs_tensors=None,
|
||||
num_nans=None,
|
||||
num_sampled=num_sampled,
|
||||
num_rejected=torch.tensor([0], dtype=torch.int32, device="cuda"),
|
||||
)
|
||||
output = AsyncOutput(
|
||||
model_runner_output=ModelRunnerOutput(req_ids=["req"], req_id_to_index={}),
|
||||
sampler_output=sampler_output,
|
||||
num_sampled_tokens=num_sampled,
|
||||
main_stream=torch.cuda.current_stream(),
|
||||
copy_stream=torch.cuda.Stream(),
|
||||
routed_experts=routed_experts,
|
||||
).get_output()
|
||||
|
||||
assert output.routed_experts is not None
|
||||
assert output.routed_experts.routing_data[:, 0, 0].tolist() == [0, 2, 4]
|
||||
assert output.routed_experts.slot_mapping.tolist() == [11, 12, 13]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rank", [0, 1])
|
||||
def test_all_tp_ranks_initialize_capture(monkeypatch, rank):
|
||||
pytest.importorskip("vllm.vllm_flash_attn", exc_type=ImportError)
|
||||
import vllm.v1.worker.gpu.model_runner as model_runner
|
||||
def test_aux_output_worker_connector_binds_capture_on_non_output_rank(monkeypatch):
|
||||
import vllm.distributed.aux_output_connector.worker as aux_output_worker
|
||||
|
||||
capturer = Mock()
|
||||
constructor = Mock(return_value=capturer)
|
||||
bind = Mock()
|
||||
monkeypatch.setattr(model_runner, "RoutedExpertsCapturer", constructor)
|
||||
monkeypatch.setattr(model_runner, "bind_routed_experts_capturer", bind)
|
||||
monkeypatch.setattr(aux_output_worker, "RoutedExpertsCapturer", constructor)
|
||||
monkeypatch.setattr(aux_output_worker, "bind_routed_experts_capturer", bind)
|
||||
monkeypatch.setattr(
|
||||
aux_output_worker,
|
||||
"get_tp_group",
|
||||
lambda: SimpleNamespace(is_first_rank=False, world_size=1),
|
||||
)
|
||||
|
||||
runner = model_runner.GPUModelRunner.__new__(model_runner.GPUModelRunner)
|
||||
runner.max_num_tokens = 32
|
||||
runner.vllm_config = SimpleNamespace(parallel_config=SimpleNamespace(rank=rank))
|
||||
runner.kv_cache_config = SimpleNamespace()
|
||||
runner.model = Mock()
|
||||
|
||||
runner.init_routed_experts_capturer()
|
||||
config = SimpleNamespace(
|
||||
aux_output_config=SimpleNamespace(enable_return_routed_experts=True),
|
||||
kv_transfer_config=None,
|
||||
max_concurrent_batches=2,
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=32),
|
||||
)
|
||||
model = Mock()
|
||||
connector = aux_output_worker.AuxOutputWorkerConnector(
|
||||
vllm_config=config,
|
||||
model=model,
|
||||
kv_cache_config=SimpleNamespace(kv_cache_groups=[_full_attention_kv_group()]),
|
||||
)
|
||||
|
||||
constructor.assert_called_once_with(
|
||||
max_num_batched_tokens=32,
|
||||
vllm_config=runner.vllm_config,
|
||||
kv_cache_config=runner.kv_cache_config,
|
||||
vllm_config=config,
|
||||
)
|
||||
bind.assert_called_once_with(runner.model, capturer)
|
||||
assert runner.routed_experts_capturer is capturer
|
||||
bind.assert_called_once_with(model, capturer)
|
||||
connector.begin_step(Mock())
|
||||
assert connector.prepare_output(Mock()) is None
|
||||
capturer.snapshot_routing_data.assert_not_called()
|
||||
|
||||
|
||||
def test_aux_output_worker_connector_default_capacity(monkeypatch):
|
||||
import vllm.distributed.aux_output_connector.worker as aux_output_worker
|
||||
|
||||
tp_group = SimpleNamespace(is_first_rank=True, world_size=1)
|
||||
store_constructor = Mock()
|
||||
background_store_constructor = Mock(side_effect=lambda store, **_: store)
|
||||
capturer = SimpleNamespace(shape_per_token=(2,), output_dtype_name="int32")
|
||||
monkeypatch.setattr(aux_output_worker, "get_tp_group", lambda: tp_group)
|
||||
monkeypatch.setattr(
|
||||
aux_output_worker, "RoutedExpertsCapturer", Mock(return_value=capturer)
|
||||
)
|
||||
monkeypatch.setattr(aux_output_worker, "bind_routed_experts_capturer", Mock())
|
||||
monkeypatch.setattr(
|
||||
aux_output_worker,
|
||||
"resolve_kv_cache_block_sizes",
|
||||
lambda *_: (32, 16),
|
||||
)
|
||||
monkeypatch.setattr(aux_output_worker, "BlockObjectStore", store_constructor)
|
||||
monkeypatch.setattr(
|
||||
aux_output_worker, "BackgroundBlockObjectStore", background_store_constructor
|
||||
)
|
||||
monkeypatch.setattr(aux_output_worker, "RoutedExpertsBuffer", Mock())
|
||||
|
||||
config = SimpleNamespace(
|
||||
aux_output_config=SimpleNamespace(max_bytes=None),
|
||||
kv_transfer_config=None,
|
||||
cache_config=SimpleNamespace(enable_prefix_caching=True),
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=8, max_num_batched_tokens=32),
|
||||
max_concurrent_batches=2,
|
||||
)
|
||||
kwargs = dict(
|
||||
vllm_config=config,
|
||||
model=Mock(),
|
||||
kv_cache_config=SimpleNamespace(
|
||||
num_blocks=10,
|
||||
kv_cache_groups=[_full_attention_kv_group(MLAAttentionSpec)],
|
||||
),
|
||||
)
|
||||
|
||||
aux_output_worker.AuxOutputWorkerConnector(**kwargs)
|
||||
assert store_constructor.call_args.kwargs["max_bytes"] == 2560
|
||||
assert store_constructor.call_args.kwargs["object_nbytes"] == 128
|
||||
assert background_store_constructor.call_args.kwargs["max_pending_batches"] == 16
|
||||
|
||||
|
||||
def test_v2_model_runner_accepts_routed_experts(monkeypatch):
|
||||
monkeypatch.setattr("importlib.metadata.entry_points", lambda **_: ())
|
||||
config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
enable_return_routed_experts=True,
|
||||
use_mla=False,
|
||||
logits_processors=None,
|
||||
enable_prompt_embeds=False,
|
||||
),
|
||||
aux_output_config=SimpleNamespace(enable_return_routed_experts=True),
|
||||
speculative_config=None,
|
||||
parallel_config=SimpleNamespace(
|
||||
prefill_context_parallel_size=1,
|
||||
|
||||
@@ -354,8 +354,7 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
|
||||
scheduler.finished_req_ids_dict = None
|
||||
scheduler.grammar_compile_error_reqs = set()
|
||||
scheduler.vllm_config = Mock()
|
||||
scheduler.vllm_config.model_config.enable_return_routed_experts = False
|
||||
scheduler.enable_return_routed_experts = False
|
||||
scheduler.aux_output_connector = None
|
||||
scheduler.return_sampling_mask = False
|
||||
scheduler.recompute_kv_load_failures = False
|
||||
scheduler.defer_block_free = False
|
||||
|
||||
@@ -19,6 +19,10 @@ from vllm.config import (
|
||||
SpeculativeConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.distributed.aux_output_connector.connector import (
|
||||
AuxOutputSchedulerConnector,
|
||||
AuxRequestOutput,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.metrics import ECConnectorStats
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import (
|
||||
HiSparseConnector,
|
||||
@@ -55,7 +59,6 @@ from vllm.v1.outputs import (
|
||||
ECConnectorOutput,
|
||||
KVConnectorOutput,
|
||||
ModelRunnerOutput,
|
||||
RoutedExpertsLists,
|
||||
SamplingMaskLists,
|
||||
make_empty_encoder_model_runner_output,
|
||||
)
|
||||
@@ -136,21 +139,18 @@ def test_add_requests():
|
||||
assert len(scheduler.waiting) == i + 1
|
||||
|
||||
|
||||
def test_routed_experts_prompt_start_at_prompt_end():
|
||||
@pytest.mark.parametrize("max_tokens", [1, 2])
|
||||
def test_routed_experts_prompt_start_at_prompt_end(max_tokens):
|
||||
"""A prompt-end offset should return empty routing data, not crash."""
|
||||
scheduler = create_scheduler()
|
||||
(request,) = create_requests(num_requests=1)
|
||||
(request,) = create_requests(num_requests=1, max_tokens=max_tokens)
|
||||
request.sampling_params.routed_experts_prompt_start = request.num_prompt_tokens
|
||||
scheduler.aux_output_connector = AuxOutputSchedulerConnector()
|
||||
scheduler.add_request(request)
|
||||
scheduler_output = scheduler.schedule()
|
||||
num_tokens = scheduler_output.num_scheduled_tokens[request.request_id]
|
||||
|
||||
manager = Mock()
|
||||
manager.routed_experts_by_slot = np.empty((0, 1, 1), dtype=np.uint8)
|
||||
manager.get.return_value = np.empty((0, 1, 1), dtype=np.uint8)
|
||||
scheduler.enable_return_routed_experts = True
|
||||
scheduler.routed_experts_mgr = manager
|
||||
scheduler._re_block_ids = {request.request_id: []}
|
||||
assert scheduler_output.aux_output_connector_metadata.requests == {
|
||||
request.request_id: request.num_prompt_tokens
|
||||
}
|
||||
|
||||
outputs = scheduler.update_from_output(
|
||||
scheduler_output,
|
||||
@@ -161,21 +161,23 @@ def test_routed_experts_prompt_start_at_prompt_end():
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
routed_experts=RoutedExpertsLists(
|
||||
routing_data=np.zeros((num_tokens, 1, 1), dtype=np.uint8),
|
||||
slot_mapping=np.arange(num_tokens),
|
||||
),
|
||||
aux_output_connector_output={
|
||||
request.request_id: AuxRequestOutput(
|
||||
request.num_prompt_tokens,
|
||||
np.empty((0, 1, 1), dtype=np.uint8),
|
||||
)
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
manager.get.assert_called_once_with(
|
||||
[], request.num_prompt_tokens, token_start=request.num_prompt_tokens
|
||||
)
|
||||
assert outputs[request.client_index].outputs[0].routed_experts.shape == (0, 1, 1)
|
||||
output = outputs[request.client_index].outputs[0]
|
||||
assert output.new_token_ids == [0]
|
||||
assert output.routed_experts.shape == (0, 1, 1)
|
||||
|
||||
|
||||
def test_finish_request():
|
||||
scheduler = create_scheduler()
|
||||
scheduler.aux_output_connector = Mock()
|
||||
requests = create_requests(num_requests=10)
|
||||
for request in requests:
|
||||
scheduler.add_request(request)
|
||||
@@ -184,6 +186,7 @@ def test_finish_request():
|
||||
scheduler.finish_requests(request.request_id, RequestStatus.FINISHED_ABORTED)
|
||||
assert request.request_id not in scheduler.requests
|
||||
assert len(scheduler.waiting) == 9 - i
|
||||
scheduler.aux_output_connector.request_finished.assert_called_with(request)
|
||||
|
||||
|
||||
def test_get_num_unfinished_requests():
|
||||
@@ -1373,6 +1376,31 @@ def test_preemption_re_records_prefix_cache_query():
|
||||
assert stats.preempted_requests == 1
|
||||
|
||||
|
||||
def test_preemption_processes_stale_aux_output():
|
||||
scheduler = create_scheduler(enable_prefix_caching=True)
|
||||
request = create_requests(num_requests=1)[0]
|
||||
scheduler.add_request(request)
|
||||
scheduler_output = scheduler.schedule()
|
||||
scheduler.running.remove(request)
|
||||
scheduler.aux_output_connector = Mock()
|
||||
scheduler._preempt_request(request, 0.0)
|
||||
scheduler.aux_output_connector.request_finished.assert_called_once_with(request)
|
||||
|
||||
scheduler.update_from_output(
|
||||
scheduler_output,
|
||||
ModelRunnerOutput(
|
||||
req_ids=[request.request_id],
|
||||
req_id_to_index={request.request_id: 0},
|
||||
sampled_token_ids=[[1000]],
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
),
|
||||
)
|
||||
|
||||
scheduler.aux_output_connector.take_output.assert_called_once_with(request, None)
|
||||
|
||||
|
||||
def test_prefix_cache_stats_not_recorded_when_caching_disabled():
|
||||
"""With prefix caching off there is no local lookup, so admitting a request
|
||||
records no phantom miss."""
|
||||
@@ -1461,6 +1489,7 @@ def test_prefix_cache_stats_counted_once_for_retried_then_scheduled_request():
|
||||
|
||||
def test_scheduler_reset_prefix_cache():
|
||||
scheduler = create_scheduler(enable_prefix_caching=True)
|
||||
scheduler.aux_output_connector = Mock()
|
||||
requests = create_requests(num_requests=10)
|
||||
for request in requests:
|
||||
scheduler.add_request(request)
|
||||
@@ -1477,10 +1506,20 @@ def test_scheduler_reset_prefix_cache():
|
||||
# Reset prefix cache should fail since there are still running requests
|
||||
# and they are taking KV cache
|
||||
assert not scheduler.reset_prefix_cache()
|
||||
scheduler.aux_output_connector.reset.assert_not_called()
|
||||
|
||||
with pytest.raises(RuntimeError, match=r"pause\(mode='keep'\)"):
|
||||
scheduler.reset_prefix_cache(reset_running_requests=True)
|
||||
|
||||
# pause(mode="keep") also waits for scheduled model outputs to drain.
|
||||
scheduler.set_pause_state(PauseState.PAUSED_ALL)
|
||||
with pytest.raises(RuntimeError, match="model output is in flight"):
|
||||
scheduler.reset_prefix_cache(reset_running_requests=True)
|
||||
for request in requests:
|
||||
request.num_in_flight_tokens = 0
|
||||
|
||||
# Reset prefix cache with reset_running_requests=True. All running requests
|
||||
# Should be pushed back to the waiting queue and kv cache should be freed
|
||||
assert scheduler.reset_prefix_cache(reset_running_requests=True)
|
||||
scheduler.aux_output_connector.reset.assert_called_once_with()
|
||||
|
||||
# Verify requests moved from running to waiting
|
||||
assert len(scheduler.waiting) == len(requests)
|
||||
@@ -1490,6 +1529,16 @@ def test_scheduler_reset_prefix_cache():
|
||||
assert scheduler.waiting[i] == request
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reset_successful", [False, True])
|
||||
def test_aux_output_reset_follows_kv_reset_result(reset_successful: bool):
|
||||
scheduler = create_scheduler(enable_prefix_caching=True)
|
||||
scheduler.aux_output_connector = Mock()
|
||||
scheduler.kv_cache_manager.reset_prefix_cache = Mock(return_value=reset_successful)
|
||||
|
||||
assert scheduler.reset_prefix_cache() is reset_successful
|
||||
assert scheduler.aux_output_connector.reset.call_count == int(reset_successful)
|
||||
|
||||
|
||||
def test_kv_cache_release_after_keep_pause_preserves_requests():
|
||||
"""A completed keep pause permits release and recomputation of live requests."""
|
||||
scheduler = create_scheduler(max_num_seqs=1, enable_prefix_caching=True)
|
||||
@@ -3791,10 +3840,9 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
|
||||
scheduler.kv_event_publisher = Mock()
|
||||
scheduler.finished_req_ids = set()
|
||||
scheduler.finished_req_ids_dict = None
|
||||
scheduler.aux_output_connector = None
|
||||
scheduler.grammar_compile_error_reqs = set()
|
||||
scheduler.vllm_config = Mock()
|
||||
scheduler.vllm_config.model_config.enable_return_routed_experts = False
|
||||
scheduler.enable_return_routed_experts = False
|
||||
scheduler.return_sampling_mask = False
|
||||
scheduler.recompute_kv_load_failures = False
|
||||
scheduler.defer_block_free = False
|
||||
|
||||
@@ -1122,6 +1122,7 @@ def _run_mamba_prefix_cache_mrv2(
|
||||
logprobs_tensors=None,
|
||||
num_nans=None,
|
||||
num_sampled=num_sampled,
|
||||
num_rejected=num_rejected,
|
||||
)
|
||||
return sampler_output, num_sampled, num_rejected
|
||||
|
||||
|
||||
@@ -3,8 +3,9 @@
|
||||
|
||||
import math
|
||||
import time
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import MagicMock, Mock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from tests.v1.engine.utils import (
|
||||
@@ -1307,6 +1308,7 @@ def test_lora_request_tracking(log_stats: bool, dummy_test_vectors):
|
||||
async def test_request_output_collector():
|
||||
NUM_REQS = 3
|
||||
TEXT = "a"
|
||||
routed_experts = np.arange(12, dtype=np.uint8).reshape(2, 3, 2)
|
||||
|
||||
def make_outputs() -> list[RequestOutput]:
|
||||
return [
|
||||
@@ -1322,6 +1324,9 @@ async def test_request_output_collector():
|
||||
token_ids=[idx],
|
||||
cumulative_logprob=(idx + 1 * 1.0),
|
||||
logprobs=[{"a": idx, "b": idx}],
|
||||
routed_experts=(
|
||||
routed_experts if idx == NUM_REQS - 1 else None
|
||||
),
|
||||
finish_reason="length" if (idx == NUM_REQS - 1) else None,
|
||||
)
|
||||
],
|
||||
@@ -1374,6 +1379,7 @@ async def test_request_output_collector():
|
||||
|
||||
assert output.finished
|
||||
assert output.outputs[0].finish_reason == "length"
|
||||
np.testing.assert_array_equal(output.outputs[0].routed_experts, routed_experts)
|
||||
# Text, token_ids, and logprobs should get merged.
|
||||
assert output.outputs[0].text == TEXT * num_to_put
|
||||
for tok_0, tok_1 in zip(output.outputs[0].token_ids, list(range(num_to_put))):
|
||||
@@ -1385,6 +1391,46 @@ async def test_request_output_collector():
|
||||
assert output.outputs[0].cumulative_logprob == cumulative_logprob_expected
|
||||
|
||||
|
||||
def test_routed_experts_are_accumulated_until_finish():
|
||||
state = RequestState(
|
||||
request_id="request-int",
|
||||
external_req_id="request",
|
||||
parent_req=None,
|
||||
request_index=0,
|
||||
lora_request=None,
|
||||
output_kind=RequestOutputKind.DELTA,
|
||||
prompt="prompt",
|
||||
prompt_token_ids=[1, 2, 3, 4],
|
||||
prompt_embeds=None,
|
||||
logprobs_processor=Mock(
|
||||
logprobs=None,
|
||||
cumulative_logprob=0.0,
|
||||
pop_prompt_logprobs=Mock(return_value=None),
|
||||
),
|
||||
detokenizer=Mock(get_next_output_text=Mock(return_value="")),
|
||||
max_tokens_param=2,
|
||||
arrival_time=0.0,
|
||||
queue=None,
|
||||
log_stats=False,
|
||||
stream_interval=1,
|
||||
)
|
||||
prompt_chunk = np.arange(24, dtype=np.uint8).reshape(4, 3, 2)
|
||||
decode_chunk = np.arange(12, dtype=np.uint8).reshape(2, 3, 2) + 24
|
||||
state.routed_experts_chunks.append(prompt_chunk)
|
||||
|
||||
partial = state.make_request_output([5], None, None, None)
|
||||
assert partial is not None
|
||||
assert partial.outputs[0].routed_experts is None
|
||||
|
||||
state.routed_experts_chunks.append(decode_chunk)
|
||||
finished = state.make_request_output([6], None, FinishReason.LENGTH, None)
|
||||
assert finished is not None
|
||||
np.testing.assert_array_equal(
|
||||
finished.outputs[0].routed_experts,
|
||||
np.concatenate((prompt_chunk, decode_chunk)),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cumulative_output_collector_n():
|
||||
"""Test collector correctly handles multiple outputs by index."""
|
||||
|
||||
@@ -3,12 +3,14 @@
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.distributed.aux_output_connector.connector import (
|
||||
AuxRequestOutput,
|
||||
)
|
||||
from vllm.v1.executor.ray_utils import detach_zero_copy_from_model_runner_output
|
||||
from vllm.v1.outputs import (
|
||||
LogprobsLists,
|
||||
LogprobsTensors,
|
||||
ModelRunnerOutput,
|
||||
RoutedExpertsLists,
|
||||
)
|
||||
|
||||
|
||||
@@ -59,26 +61,19 @@ def test_detach_zero_copy_from_model_runner_output_copies_only_numpy_views():
|
||||
assert output.prompt_logprobs_dict["req-0"] is prompt_logprobs
|
||||
|
||||
|
||||
def test_detach_zero_copy_routed_experts_without_logprobs():
|
||||
def test_detach_zero_copy_aux_output_without_logprobs():
|
||||
rows = _make_readonly(np.arange(12, dtype=np.uint8).reshape(2, 3, 2))
|
||||
output = ModelRunnerOutput(
|
||||
req_ids=["req-0"],
|
||||
req_id_to_index={"req-0": 0},
|
||||
routed_experts=RoutedExpertsLists(
|
||||
routing_data=_make_readonly(np.arange(12, dtype=np.int32).reshape(2, 3, 2)),
|
||||
slot_mapping=_make_readonly(np.array([7, 8], dtype=np.int64)),
|
||||
),
|
||||
aux_output_connector_output={"req-0": AuxRequestOutput(0, rows)},
|
||||
)
|
||||
original = output.routed_experts
|
||||
assert output.logprobs is None
|
||||
|
||||
detach_zero_copy_from_model_runner_output(output)
|
||||
|
||||
detached = output.routed_experts
|
||||
assert detached is not None
|
||||
assert detached is not original
|
||||
assert detached.routing_data is not original.routing_data
|
||||
assert detached.slot_mapping is not original.slot_mapping
|
||||
assert detached.routing_data.flags.writeable
|
||||
assert detached.slot_mapping.flags.writeable
|
||||
np.testing.assert_array_equal(detached.routing_data, original.routing_data)
|
||||
np.testing.assert_array_equal(detached.slot_mapping, original.slot_mapping)
|
||||
aux_output = output.aux_output_connector_output
|
||||
assert aux_output is not None
|
||||
detached = aux_output["req-0"].rows
|
||||
assert detached is not rows
|
||||
assert detached.flags.writeable
|
||||
np.testing.assert_array_equal(detached, rows)
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.config.attention import AttentionConfig, HiSparseConfig
|
||||
from vllm.config.aux_output import AuxOutputConfig
|
||||
from vllm.config.cache import CacheConfig
|
||||
from vllm.config.compilation import (
|
||||
CompilationConfig,
|
||||
@@ -66,6 +67,8 @@ from vllm.config.weight_transfer import WeightTransferConfig
|
||||
# __all__ should only contain classes and functions.
|
||||
# Types and globals should be imported from their respective modules.
|
||||
__all__ = [
|
||||
# From vllm.config.aux_output
|
||||
"AuxOutputConfig",
|
||||
# From vllm.config.attention
|
||||
"AttentionConfig",
|
||||
"HiSparseConfig",
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Configuration for execution auxiliary outputs."""
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
from vllm.config.utils import config
|
||||
|
||||
|
||||
@config
|
||||
class AuxOutputConfig:
|
||||
"""Configuration for auxiliary-output delivery."""
|
||||
|
||||
enable_return_routed_experts: bool = False
|
||||
"""Capture and return routed-experts auxiliary outputs."""
|
||||
|
||||
max_bytes: int | None = Field(default=None, gt=0)
|
||||
"""LRU capacity, or ``None`` to derive it from the KV cache capacity."""
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
"""Whether any execution auxiliary output is enabled."""
|
||||
return self.enable_return_routed_experts
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""Hash AuxOutput settings that alter the model forward graph."""
|
||||
from vllm.config.utils import hash_factors
|
||||
|
||||
return hash_factors(
|
||||
{"enable_return_routed_experts": self.enable_return_routed_experts}
|
||||
)
|
||||
@@ -247,8 +247,6 @@ class ModelConfig:
|
||||
|
||||
NOTE: This disables both `torch.compile` and CUDA graphs, and is
|
||||
equivalent to setting `-cc.mode=none -cc.cudagraph_mode=none`."""
|
||||
enable_return_routed_experts: bool = False
|
||||
"""Whether to return routed experts."""
|
||||
return_sampling_mask: bool = False
|
||||
"""Whether to return the post-processing token support for each sample."""
|
||||
max_logprobs: int = Field(default=20, ge=-1)
|
||||
|
||||
+53
-33
@@ -28,6 +28,7 @@ from vllm.utils import random_uuid
|
||||
from vllm.utils.hashing import safe_hash
|
||||
|
||||
from .attention import AttentionConfig, HiSparseConfig
|
||||
from .aux_output import AuxOutputConfig
|
||||
from .cache import CacheConfig
|
||||
from .compilation import CompilationConfig, CompilationMode, CUDAGraphMode
|
||||
from .device import DeviceConfig
|
||||
@@ -376,6 +377,8 @@ class VllmConfig:
|
||||
"""Model weight offloading configuration."""
|
||||
attention_config: AttentionConfig = Field(default_factory=AttentionConfig)
|
||||
"""Attention configuration."""
|
||||
aux_output_config: AuxOutputConfig = Field(default_factory=AuxOutputConfig)
|
||||
"""Execution auxiliary output configuration."""
|
||||
engram_config: EngramConfig | None = None
|
||||
"""N-gram embedding storage and sharding settings."""
|
||||
mamba_config: MambaConfig = Field(default_factory=MambaConfig)
|
||||
@@ -554,6 +557,7 @@ class VllmConfig:
|
||||
vllm_factors.append(self.ec_transfer_config.compute_hash())
|
||||
else:
|
||||
vllm_factors.append("None")
|
||||
vllm_factors.append(self.aux_output_config.compute_hash())
|
||||
if self.additional_config:
|
||||
if isinstance(additional_config := self.additional_config, dict):
|
||||
additional_config_hash = safe_hash(
|
||||
@@ -1084,6 +1088,53 @@ class VllmConfig:
|
||||
# This is the same for all backends
|
||||
self.kv_transfer_config.kv_role = "kv_both"
|
||||
|
||||
def _verify_aux_output_compatibility(self) -> None:
|
||||
"""Reject configurations unsupported by enabled auxiliary outputs."""
|
||||
if not self.aux_output_config.enabled:
|
||||
return
|
||||
if not self.use_v2_model_runner:
|
||||
raise ValueError(
|
||||
"AuxOutput Connector requires Model Runner V2; set "
|
||||
"VLLM_USE_V2_MODEL_RUNNER=1."
|
||||
)
|
||||
if self.model_config.runner_type != "generate":
|
||||
raise ValueError("AuxOutput Connector only supports generate runners.")
|
||||
if not self.model_config.is_moe:
|
||||
raise ValueError("AuxOutput Connector only supports MoE models.")
|
||||
if not self.cache_config.enable_prefix_caching:
|
||||
raise ValueError("AuxOutput Connector requires prefix caching.")
|
||||
if (
|
||||
self.speculative_config is not None
|
||||
and self.speculative_config.enable_adaptive_verification
|
||||
):
|
||||
raise ValueError(
|
||||
"--enable-return-routed-experts is incompatible with "
|
||||
"adaptive speculative verification."
|
||||
)
|
||||
if self.parallel_config.pipeline_parallel_size > 1:
|
||||
raise ValueError(
|
||||
"--enable-return-routed-experts is incompatible with "
|
||||
"pipeline parallelism (PP > 1)."
|
||||
)
|
||||
if (
|
||||
self.parallel_config.decode_context_parallel_size > 1
|
||||
or self.parallel_config.prefill_context_parallel_size > 1
|
||||
):
|
||||
raise ValueError(
|
||||
"--enable-return-routed-experts is incompatible with "
|
||||
"context parallelism (DCP/PCP > 1)."
|
||||
)
|
||||
|
||||
kv_transfer_config = self.kv_transfer_config
|
||||
if (
|
||||
kv_transfer_config is not None
|
||||
and kv_transfer_config.is_kv_transfer_instance
|
||||
):
|
||||
raise ValueError(
|
||||
"--enable-return-routed-experts is incompatible with KV "
|
||||
"connectors (PD disaggregation and KV cache offload)."
|
||||
)
|
||||
|
||||
def _verify_kv_transfer_compat(self) -> None:
|
||||
"""Reject configurations that silently corrupt KV transfers."""
|
||||
if (
|
||||
@@ -1294,38 +1345,6 @@ class VllmConfig:
|
||||
|
||||
self.parallel_config.is_moe_model = self.model_config.is_moe
|
||||
|
||||
if (
|
||||
self.model_config is not None
|
||||
and self.model_config.enable_return_routed_experts
|
||||
):
|
||||
if self.parallel_config.pipeline_parallel_size > 1:
|
||||
raise ValueError(
|
||||
"--enable-return-routed-experts is incompatible with "
|
||||
"pipeline parallelism (PP > 1)."
|
||||
)
|
||||
if (
|
||||
self.parallel_config.decode_context_parallel_size > 1
|
||||
or self.parallel_config.prefill_context_parallel_size > 1
|
||||
):
|
||||
raise ValueError(
|
||||
"--enable-return-routed-experts is incompatible with context "
|
||||
"parallelism (DCP > 1 or PCP > 1)."
|
||||
)
|
||||
|
||||
# Incompatible with any KV connector — covers both PD disaggregation
|
||||
# (kv_producer/kv_consumer: routing captured on P can't reach D) and
|
||||
# single-instance KV offload/sharing (kv_both: slot_mapping semantics
|
||||
# change when KV blocks live outside local GPU memory, breaking the
|
||||
# slot-indexed routed_experts buffer).
|
||||
if (
|
||||
self.kv_transfer_config is not None
|
||||
and self.kv_transfer_config.is_kv_transfer_instance
|
||||
):
|
||||
raise ValueError(
|
||||
"--enable-return-routed-experts is incompatible with KV "
|
||||
"connectors (PD disaggregation, KV cache offload)."
|
||||
)
|
||||
|
||||
if (
|
||||
self.model_config is not None
|
||||
and self.model_config.multimodal_config is not None
|
||||
@@ -2032,6 +2051,7 @@ class VllmConfig:
|
||||
# Resolve kv_offloading-derived connector name into kv_transfer_config
|
||||
# before the HMA check below, which inspects the connector class.
|
||||
self._post_init_kv_transfer_config()
|
||||
self._verify_aux_output_compatibility()
|
||||
|
||||
if self.is_mm_encoder_only and self.cache_config.enable_prefix_caching:
|
||||
# Such an instance publishes encoder embeddings and runs no language
|
||||
@@ -2689,7 +2709,7 @@ class VllmConfig:
|
||||
f"quantization={self.model_config.quantization}, "
|
||||
f"quantization_config={self.model_config.quantization_config}, " # noqa
|
||||
f"enforce_eager={self.model_config.enforce_eager}, "
|
||||
f"enable_return_routed_experts={self.model_config.enable_return_routed_experts}, " # noqa
|
||||
f"aux_output_config={self.aux_output_config!r}, "
|
||||
f"kv_cache_dtype={self.cache_config.cache_dtype}, "
|
||||
f"device_config={self.device_config.device}, "
|
||||
f"structured_outputs_config={self.structured_outputs_config!r}, "
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,157 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Scheduler-side control plane for execution auxiliary outputs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterator, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import numpy as np
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.request import Request
|
||||
|
||||
|
||||
@dataclass
|
||||
class PackedBlockHashes:
|
||||
"""Contiguous, self-contained block hashes for scheduler-worker IPC."""
|
||||
|
||||
data: bytes
|
||||
item_size: int
|
||||
|
||||
def __iter__(self) -> Iterator[bytes]:
|
||||
for start in range(0, len(self.data), self.item_size):
|
||||
yield self.data[start : start + self.item_size]
|
||||
|
||||
|
||||
@dataclass
|
||||
class AuxOutputConnectorMetadata:
|
||||
generation: int
|
||||
requests: dict[str, int]
|
||||
block_hashes: dict[str, PackedBlockHashes]
|
||||
finished_requests: tuple[str, ...]
|
||||
|
||||
|
||||
@dataclass
|
||||
class AuxRequestOutput:
|
||||
token_start: int
|
||||
rows: np.ndarray
|
||||
|
||||
|
||||
class AuxOutputSchedulerConnector:
|
||||
"""Build worker metadata without owning auxiliary output payloads or stores."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
# Number of hashes already sent to the worker for each active request.
|
||||
self._sent_hash_counts: dict[str, int] = {}
|
||||
# Terminal events are delivered with the next connector metadata.
|
||||
self._finished_requests: dict[str, PackedBlockHashes | None] = {}
|
||||
self._generation = 0
|
||||
|
||||
def build_connector_meta(
|
||||
self,
|
||||
scheduler_output: SchedulerOutput,
|
||||
requests: dict[str, Request],
|
||||
) -> AuxOutputConnectorMetadata:
|
||||
"""Build one step's incremental worker metadata."""
|
||||
scheduled_requests: dict[str, int] = {}
|
||||
block_hashes_by_request: dict[str, PackedBlockHashes] = {}
|
||||
for request_id in scheduler_output.num_scheduled_tokens:
|
||||
num_sent = self._sent_hash_counts.setdefault(request_id, 0)
|
||||
request = requests[request_id]
|
||||
packed = self._pack_new_hashes(request.block_hashes, num_sent)
|
||||
if packed is not None:
|
||||
block_hashes_by_request[request_id] = packed
|
||||
self._sent_hash_counts[request_id] = len(request.block_hashes)
|
||||
assert request.sampling_params is not None
|
||||
scheduled_requests[request_id] = max(
|
||||
request.sampling_params.routed_experts_prompt_start,
|
||||
0 if request.num_output_tokens == 0 else request.num_tokens - 1,
|
||||
)
|
||||
# A settled token can complete a hash block after the next async schedule
|
||||
# was built. Send a hash-only update if the request was not rescheduled.
|
||||
if len(self._sent_hash_counts) != len(scheduled_requests):
|
||||
for request_id, num_sent in self._sent_hash_counts.items():
|
||||
if request_id in scheduled_requests:
|
||||
continue
|
||||
request = requests[request_id]
|
||||
packed = self._pack_new_hashes(request.block_hashes, num_sent)
|
||||
if packed is None:
|
||||
continue
|
||||
block_hashes_by_request[request_id] = packed
|
||||
self._sent_hash_counts[request_id] = len(request.block_hashes)
|
||||
# Sending transfers ownership of these one-shot events.
|
||||
finished_requests = tuple(self._finished_requests)
|
||||
block_hashes_by_request.update(
|
||||
(request_id, block_hashes)
|
||||
for request_id, block_hashes in self._finished_requests.items()
|
||||
if block_hashes is not None
|
||||
)
|
||||
self._finished_requests = {}
|
||||
return AuxOutputConnectorMetadata(
|
||||
self._generation,
|
||||
scheduled_requests,
|
||||
block_hashes_by_request,
|
||||
finished_requests,
|
||||
)
|
||||
|
||||
def take_output(
|
||||
self,
|
||||
request: Request,
|
||||
output: dict[str, AuxRequestOutput] | None,
|
||||
) -> np.ndarray | None:
|
||||
"""Return the accepted R3 rows for one scheduled request."""
|
||||
request_id = request.request_id
|
||||
assert output is not None and request_id in output, (
|
||||
f"auxiliary output worker output is missing {request_id}"
|
||||
)
|
||||
request_output = output[request_id]
|
||||
token_end = request.num_tokens - 1
|
||||
local_end = token_end - request_output.token_start
|
||||
if local_end < 0:
|
||||
assert not request.is_finished(), (
|
||||
"finished auxiliary output output has no accepted token range: "
|
||||
f"request={request_id}, token_end={token_end}, "
|
||||
f"output_start={request_output.token_start}, "
|
||||
"output_end="
|
||||
f"{request_output.token_start + len(request_output.rows)}"
|
||||
)
|
||||
return None
|
||||
assert local_end <= len(request_output.rows), (
|
||||
"auxiliary output worker output has an invalid token range: "
|
||||
f"request={request_id}, token_end={token_end}, "
|
||||
f"output_start={request_output.token_start}, "
|
||||
f"output_end={request_output.token_start + len(request_output.rows)}"
|
||||
)
|
||||
return request_output.rows[:local_end]
|
||||
|
||||
def request_finished(self, request: Request) -> None:
|
||||
"""Queue a request's terminal event and final block hashes."""
|
||||
request_id = request.request_id
|
||||
num_sent = self._sent_hash_counts.pop(request_id, None)
|
||||
if num_sent is None:
|
||||
return
|
||||
# The next metadata delivers this terminal event to the worker.
|
||||
self._finished_requests[request_id] = self._pack_new_hashes(
|
||||
request.block_hashes, num_sent
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _pack_new_hashes(
|
||||
block_hashes: Sequence[bytes], num_sent: int
|
||||
) -> PackedBlockHashes | None:
|
||||
assert num_sent <= len(block_hashes), "KV block-hash history shrank"
|
||||
new_hashes = block_hashes[num_sent:]
|
||||
if not new_hashes:
|
||||
return None
|
||||
return PackedBlockHashes(b"".join(new_hashes), len(new_hashes[0]))
|
||||
|
||||
def reset(self) -> None:
|
||||
"""Start a new auxiliary output namespace after a prefix-cache reset."""
|
||||
# The worker drops temporary state on generation changes; resend hashes.
|
||||
self._sent_hash_counts.clear()
|
||||
self._finished_requests.clear()
|
||||
self._generation += 1
|
||||
@@ -0,0 +1,219 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Routed-experts auxiliary output keys, publication, and materialization."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Hashable, Iterable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.distributed.aux_output_connector.store import (
|
||||
BackgroundBlockObjectStore,
|
||||
BlockObject,
|
||||
BlockObjectStore,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _RequestTail:
|
||||
slot: int
|
||||
block_start: int
|
||||
length: int = 0
|
||||
|
||||
|
||||
class RoutedExpertsBuffer:
|
||||
"""Own incomplete and unkeyed full routed-experts blocks."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
dtype: np.dtype[Any],
|
||||
shape_per_token: tuple[int, ...],
|
||||
block_size: int,
|
||||
max_num_seqs: int,
|
||||
max_num_batched_tokens: int,
|
||||
max_concurrent_batches: int,
|
||||
) -> None:
|
||||
self.dtype = dtype
|
||||
self.shape_per_token = shape_per_token
|
||||
self.block_size = block_size
|
||||
max_step_blocks = (
|
||||
max_num_batched_tokens + block_size - 1
|
||||
) // block_size + max_num_seqs
|
||||
# In-flight full blocks plus one incomplete tail per active request.
|
||||
max_blocks = max_concurrent_batches * max_step_blocks + max_num_seqs
|
||||
self._rows: np.ndarray = np.empty(
|
||||
(max_blocks, block_size, *shape_per_token), dtype=dtype
|
||||
)
|
||||
self._free_slots = list(range(len(self._rows) - 1, -1, -1))
|
||||
self._owned_slots: dict[int, int] = {}
|
||||
self._requests: dict[Hashable, _RequestTail] = {}
|
||||
|
||||
def _tail(self, request_id: Hashable, block_start: int) -> _RequestTail:
|
||||
tail = self._requests.get(request_id)
|
||||
if tail is not None:
|
||||
assert tail.block_start == block_start, (
|
||||
"auxiliary output capture skipped an incomplete block: "
|
||||
f"request={request_id}, expected={tail.block_start}, "
|
||||
f"actual={block_start}"
|
||||
)
|
||||
return tail
|
||||
assert self._free_slots, "auxiliary output block pool is exhausted"
|
||||
tail = _RequestTail(self._free_slots.pop(), block_start)
|
||||
self._requests[request_id] = tail
|
||||
return tail
|
||||
|
||||
def capture(
|
||||
self, request_id: Hashable, token_start: int, rows: np.ndarray
|
||||
) -> list[tuple[int, np.ndarray]]:
|
||||
"""Stage rows and return completed blocks without retaining them."""
|
||||
rows = np.asarray(rows)
|
||||
assert rows.shape[1:] == self.shape_per_token and rows.dtype == self.dtype, (
|
||||
"routed-experts capture profile changed"
|
||||
)
|
||||
if token_start < 0:
|
||||
raise ValueError("auxiliary output token start must be non-negative")
|
||||
|
||||
completed: list[tuple[int, np.ndarray]] = []
|
||||
offset = 0
|
||||
while offset < len(rows):
|
||||
position = token_start + offset
|
||||
block_start = position // self.block_size * self.block_size
|
||||
local_start = position - block_start
|
||||
|
||||
# Full aligned input blocks do not need tail staging.
|
||||
if (
|
||||
request_id not in self._requests
|
||||
and local_start == 0
|
||||
and len(rows) - offset >= self.block_size
|
||||
):
|
||||
completed.append((block_start, rows[offset : offset + self.block_size]))
|
||||
offset += self.block_size
|
||||
continue
|
||||
|
||||
tail = self._tail(request_id, block_start)
|
||||
assert local_start == tail.length, (
|
||||
"auxiliary output capture is not contiguous: "
|
||||
f"request={request_id}, expected={block_start + tail.length}, "
|
||||
f"actual={position}"
|
||||
)
|
||||
count = min(self.block_size - local_start, len(rows) - offset)
|
||||
if count == 1:
|
||||
self._rows[tail.slot, local_start] = rows[offset]
|
||||
else:
|
||||
self._rows[tail.slot, local_start : local_start + count] = rows[
|
||||
offset : offset + count
|
||||
]
|
||||
tail.length += count
|
||||
offset += count
|
||||
if tail.length == self.block_size:
|
||||
block = self._rows[tail.slot]
|
||||
del self._requests[request_id]
|
||||
self._owned_slots[id(block)] = tail.slot
|
||||
completed.append((block_start, block))
|
||||
|
||||
return completed
|
||||
|
||||
def read(
|
||||
self, request_id: Hashable, token_start: int, token_end: int
|
||||
) -> np.ndarray:
|
||||
tail = self._requests.get(request_id)
|
||||
assert tail is not None, (
|
||||
f"auxiliary output buffer is missing request {request_id}"
|
||||
)
|
||||
local_start = token_start - tail.block_start
|
||||
local_end = token_end - tail.block_start
|
||||
assert 0 <= local_start < local_end <= tail.length, (
|
||||
"auxiliary output range is unavailable: "
|
||||
f"request={request_id}, range=[{token_start}, {token_end}), "
|
||||
f"available=[{tail.block_start}, {tail.block_start + tail.length})"
|
||||
)
|
||||
return self._rows[tail.slot, local_start:local_end].copy()
|
||||
|
||||
def retain_block(self, rows: np.ndarray) -> np.ndarray:
|
||||
"""Retain one unkeyed block after the current capture call."""
|
||||
if id(rows) in self._owned_slots:
|
||||
return rows
|
||||
assert self._free_slots, "auxiliary output block pool is exhausted"
|
||||
slot = self._free_slots.pop()
|
||||
retained = self._rows[slot]
|
||||
retained[...] = rows
|
||||
self._owned_slots[id(retained)] = slot
|
||||
return retained
|
||||
|
||||
def release_block(self, rows: np.ndarray) -> None:
|
||||
slot = self._owned_slots.pop(id(rows), None)
|
||||
if slot is not None:
|
||||
self._free_slots.append(slot)
|
||||
|
||||
def discard(self, request_id: Hashable) -> None:
|
||||
tail = self._requests.pop(request_id, None)
|
||||
if tail is not None:
|
||||
self._free_slots.append(tail.slot)
|
||||
|
||||
def reset(self) -> None:
|
||||
self._requests.clear()
|
||||
self._owned_slots.clear()
|
||||
self._free_slots[:] = range(len(self._rows) - 1, -1, -1)
|
||||
|
||||
|
||||
def routed_experts_keys(
|
||||
block_hashes: Iterable[bytes], aux_output_namespace: str
|
||||
) -> list[str]:
|
||||
prefix = f"vllm-artifact/{aux_output_namespace}/"
|
||||
return [prefix + block_hash.hex() for block_hash in block_hashes]
|
||||
|
||||
|
||||
def materialize_routed_experts(
|
||||
store: BackgroundBlockObjectStore | BlockObjectStore,
|
||||
aux_output_keys: list[str],
|
||||
*,
|
||||
shape_per_token: tuple[int, ...],
|
||||
dtype: np.dtype[Any],
|
||||
) -> np.ndarray:
|
||||
if not aux_output_keys:
|
||||
raise ValueError("routed-experts auxiliary output key list must not be empty")
|
||||
payload = store.get_concatenated(aux_output_keys)
|
||||
return np.frombuffer(payload, dtype=dtype).reshape((-1, *shape_per_token))
|
||||
|
||||
|
||||
def publish_routed_experts(
|
||||
store: BackgroundBlockObjectStore | BlockObjectStore,
|
||||
*,
|
||||
batches: Sequence[tuple[Sequence[str], list[tuple[int, np.ndarray]]]],
|
||||
block_size: int,
|
||||
retain_keys: Sequence[str] = (),
|
||||
release_keys: Sequence[str] = (),
|
||||
) -> None:
|
||||
"""Publish immutable full R3 blocks."""
|
||||
objects = []
|
||||
for aux_output_keys, blocks in batches:
|
||||
for block_start, array in blocks:
|
||||
if block_start < 0 or block_start % block_size:
|
||||
raise ValueError(
|
||||
"auxiliary output block start is not hash-block aligned"
|
||||
)
|
||||
block_index = block_start // block_size
|
||||
if block_index >= len(aux_output_keys):
|
||||
raise ValueError(
|
||||
"auxiliary output block has no corresponding key: "
|
||||
f"start={block_start}, index={block_index}, "
|
||||
f"keys={len(aux_output_keys)}, block_size={block_size}"
|
||||
)
|
||||
if len(array) != block_size:
|
||||
raise ValueError(
|
||||
"auxiliary output block length does not match hash block size"
|
||||
)
|
||||
objects.append(
|
||||
BlockObject(
|
||||
key=aux_output_keys[block_index],
|
||||
payload=array.tobytes(order="C"),
|
||||
)
|
||||
)
|
||||
store.put(
|
||||
objects,
|
||||
retain_keys=retain_keys,
|
||||
release_keys=release_keys,
|
||||
)
|
||||
@@ -0,0 +1,234 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""In-process auxiliary-output storage."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import mmap
|
||||
import queue
|
||||
import threading
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BlockObject:
|
||||
"""One immutable auxiliary output object."""
|
||||
|
||||
key: str
|
||||
payload: bytes
|
||||
|
||||
|
||||
class BlockObjectStoreError(RuntimeError):
|
||||
"""AuxOutput storage or retrieval failed."""
|
||||
|
||||
|
||||
class BackgroundBlockObjectStore:
|
||||
"""Serialize store mutations on a background thread."""
|
||||
|
||||
def __init__(self, store: BlockObjectStore, *, max_pending_batches: int) -> None:
|
||||
self._store = store
|
||||
self._queue: queue.Queue[
|
||||
tuple[list[BlockObject], tuple[str, ...], tuple[str, ...]] | None
|
||||
] = queue.Queue(maxsize=max_pending_batches)
|
||||
self._error: BaseException | None = None
|
||||
self._closed = False
|
||||
self._thread = threading.Thread(
|
||||
target=self._run,
|
||||
daemon=True,
|
||||
name="vllm-aux-output-writer",
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
def _run(self) -> None:
|
||||
while True:
|
||||
update = self._queue.get()
|
||||
try:
|
||||
if update is None:
|
||||
return
|
||||
if self._error is None:
|
||||
objects, retain_keys, release_keys = update
|
||||
self._store.put(
|
||||
objects,
|
||||
retain_keys=retain_keys,
|
||||
release_keys=release_keys,
|
||||
)
|
||||
except BaseException as error:
|
||||
self._error = error
|
||||
finally:
|
||||
self._queue.task_done()
|
||||
|
||||
def _raise_if_failed(self) -> None:
|
||||
if self._error is not None:
|
||||
raise BlockObjectStoreError(
|
||||
"auxiliary output publication failed"
|
||||
) from self._error
|
||||
|
||||
def put(
|
||||
self,
|
||||
objects: list[BlockObject],
|
||||
*,
|
||||
retain_keys: Iterable[str] = (),
|
||||
release_keys: Iterable[str] = (),
|
||||
) -> None:
|
||||
retains = tuple(retain_keys)
|
||||
releases = tuple(release_keys)
|
||||
if not objects and not retains and not releases:
|
||||
return
|
||||
if self._closed:
|
||||
raise RuntimeError("auxiliary output store is closed")
|
||||
self._queue.put((objects, retains, releases))
|
||||
self._raise_if_failed()
|
||||
|
||||
def get_concatenated(self, keys: list[str]) -> bytes:
|
||||
self._queue.join()
|
||||
self._raise_if_failed()
|
||||
return self._store.get_concatenated(keys)
|
||||
|
||||
def close(self) -> None:
|
||||
if self._closed:
|
||||
return
|
||||
self._closed = True
|
||||
self._queue.join()
|
||||
self._queue.put(None)
|
||||
self._thread.join()
|
||||
self._store.close()
|
||||
self._raise_if_failed()
|
||||
|
||||
|
||||
class BlockObjectStore:
|
||||
"""Single-owner bounded store that fails closed after an eviction."""
|
||||
|
||||
_UNALLOCATED_SLOT = -1
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
max_bytes: int,
|
||||
object_nbytes: int,
|
||||
) -> None:
|
||||
if object_nbytes <= 0:
|
||||
raise ValueError("auxiliary output object size must be positive")
|
||||
if max_bytes < object_nbytes:
|
||||
raise ValueError("auxiliary output store must fit at least one object")
|
||||
self.object_nbytes = object_nbytes
|
||||
self.num_slots = max_bytes // object_nbytes
|
||||
self._lru: OrderedDict[str, int] = OrderedDict()
|
||||
self._references: dict[str, int] = {}
|
||||
self._free_slots: list[int] = []
|
||||
self._next_slot = 0
|
||||
self._arena: mmap.mmap | None = mmap.mmap(
|
||||
-1, self.num_slots * self.object_nbytes
|
||||
)
|
||||
|
||||
def _evict_to_fit(self, protected: set[str]) -> None:
|
||||
excess = len(self._lru) - self.num_slots
|
||||
if excess <= 0:
|
||||
return
|
||||
victims = []
|
||||
for key in self._lru:
|
||||
if key not in self._references and key not in protected:
|
||||
victims.append(key)
|
||||
if len(victims) == excess:
|
||||
break
|
||||
if len(victims) != excess:
|
||||
raise BlockObjectStoreError(
|
||||
"auxiliary output store cannot retain the requested batch: "
|
||||
f"limit={self.num_slots} objects"
|
||||
)
|
||||
for victim in victims:
|
||||
slot = self._lru.pop(victim)
|
||||
if slot != self._UNALLOCATED_SLOT:
|
||||
self._free_slots.append(slot)
|
||||
|
||||
def _allocate_slot(self) -> int:
|
||||
if self._free_slots:
|
||||
return self._free_slots.pop()
|
||||
slot = self._next_slot
|
||||
self._next_slot += 1
|
||||
return slot
|
||||
|
||||
def put(
|
||||
self,
|
||||
objects: list[BlockObject],
|
||||
*,
|
||||
retain_keys: Iterable[str] = (),
|
||||
release_keys: Iterable[str] = (),
|
||||
) -> None:
|
||||
arena = self._arena
|
||||
if arena is None:
|
||||
raise RuntimeError("auxiliary output store is closed")
|
||||
unique = {obj.key: obj for obj in objects}
|
||||
self._retain(retain_keys)
|
||||
terminal_order = self._release(release_keys)
|
||||
for key in unique:
|
||||
self._lru.setdefault(key, self._UNALLOCATED_SLOT)
|
||||
self._lru.move_to_end(key)
|
||||
for key in terminal_order:
|
||||
if key in self._lru:
|
||||
self._lru.move_to_end(key)
|
||||
try:
|
||||
self._evict_to_fit(set(unique) - set(terminal_order))
|
||||
except BlockObjectStoreError:
|
||||
for key in unique:
|
||||
if self._lru.get(key) == self._UNALLOCATED_SLOT:
|
||||
del self._lru[key]
|
||||
raise
|
||||
for object_id, obj in unique.items():
|
||||
slot = self._lru.get(object_id)
|
||||
if slot is None:
|
||||
continue
|
||||
if slot != self._UNALLOCATED_SLOT:
|
||||
continue
|
||||
slot = self._allocate_slot()
|
||||
offset = slot * self.object_nbytes
|
||||
arena[offset : offset + self.object_nbytes] = obj.payload
|
||||
self._lru[object_id] = slot
|
||||
|
||||
def _retain(self, keys: Iterable[str]) -> None:
|
||||
for key in keys:
|
||||
references = self._references.get(key, 0)
|
||||
self._references[key] = references + 1
|
||||
|
||||
def _release(self, keys: Iterable[str]) -> list[str]:
|
||||
terminal_order = []
|
||||
for key in keys:
|
||||
references = self._references[key] - 1
|
||||
if references:
|
||||
self._references[key] = references
|
||||
continue
|
||||
del self._references[key]
|
||||
terminal_order.append(key)
|
||||
return terminal_order
|
||||
|
||||
def get_concatenated(self, keys: list[str]) -> bytes:
|
||||
arena_obj = self._arena
|
||||
if arena_obj is None:
|
||||
raise RuntimeError("auxiliary output store is closed")
|
||||
try:
|
||||
entries = [self._lru[key] for key in keys]
|
||||
except KeyError as error:
|
||||
raise BlockObjectStoreError(
|
||||
"auxiliary output object does not exist; the object may have been "
|
||||
f"evicted (used={len(self._lru)}, "
|
||||
f"limit={self.num_slots} objects). Increase "
|
||||
"aux_output_config.max_bytes when a KV cache hit requires it."
|
||||
) from error
|
||||
arena = memoryview(arena_obj)
|
||||
try:
|
||||
payload = b"".join(
|
||||
arena[slot * self.object_nbytes : (slot + 1) * self.object_nbytes]
|
||||
for slot in entries
|
||||
)
|
||||
finally:
|
||||
arena.release()
|
||||
for key in keys:
|
||||
self._lru.move_to_end(key)
|
||||
return payload
|
||||
|
||||
def close(self) -> None:
|
||||
arena = self._arena
|
||||
if arena is not None:
|
||||
self._arena = None
|
||||
arena.close()
|
||||
@@ -0,0 +1,412 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Worker-side auxiliary-output data plane."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from threading import Lock
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.aux_output_connector.connector import (
|
||||
AuxOutputConnectorMetadata,
|
||||
AuxRequestOutput,
|
||||
)
|
||||
from vllm.distributed.aux_output_connector.routed_experts import (
|
||||
RoutedExpertsBuffer,
|
||||
materialize_routed_experts,
|
||||
publish_routed_experts,
|
||||
routed_experts_keys,
|
||||
)
|
||||
from vllm.distributed.aux_output_connector.store import (
|
||||
BackgroundBlockObjectStore,
|
||||
BlockObjectStore,
|
||||
)
|
||||
from vllm.distributed.parallel_state import get_tp_group
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
RoutedExpertsCapturer,
|
||||
bind_routed_experts_capturer,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
|
||||
|
||||
@dataclass
|
||||
class _WorkerRequestState:
|
||||
aux_output_keys: list[str] = field(default_factory=list)
|
||||
pending_blocks: list[tuple[int, np.ndarray]] = field(default_factory=list)
|
||||
capture_cursor: int | None = None
|
||||
scheduled_cursor: int = 0
|
||||
emit_cursor: int = 0
|
||||
pending_outputs: int = 0
|
||||
# Terminal event received; teardown waits for in-flight step outputs.
|
||||
finished: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class PendingAuxOutput:
|
||||
"""Own one step's R3 snapshot until its asynchronous copy is consumed."""
|
||||
|
||||
connector: AuxOutputWorkerConnector
|
||||
request_ids: list[str]
|
||||
token_starts: np.ndarray
|
||||
query_start_loc: np.ndarray
|
||||
# GPU snapshot taken on the main stream.
|
||||
routed_experts_gpu: torch.Tensor
|
||||
# Filled by enqueue_cpu_copy on the output copy stream.
|
||||
routed_experts: np.ndarray | None = None
|
||||
num_sampled: np.ndarray | None = None
|
||||
num_rejected: np.ndarray | None = None
|
||||
|
||||
def enqueue_cpu_copy(
|
||||
self, num_sampled: np.ndarray, num_rejected: np.ndarray
|
||||
) -> None:
|
||||
"""Enqueue asynchronous D2H copies; call on the output copy stream."""
|
||||
self.num_sampled = num_sampled
|
||||
self.num_rejected = num_rejected
|
||||
self.routed_experts = self.routed_experts_gpu.to(
|
||||
"cpu", non_blocking=True
|
||||
).numpy()
|
||||
|
||||
def process_output(self) -> dict[str, AuxRequestOutput]:
|
||||
return self.connector.process_output(self)
|
||||
|
||||
|
||||
class AuxOutputWorkerConnector:
|
||||
"""Own capture, request tails, and backend resources on the output worker."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
vllm_config: VllmConfig,
|
||||
model: torch.nn.Module,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
) -> None:
|
||||
max_num_batched_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
capturer = RoutedExpertsCapturer(
|
||||
max_num_batched_tokens=max_num_batched_tokens, vllm_config=vllm_config
|
||||
)
|
||||
bind_routed_experts_capturer(model, capturer)
|
||||
self._capturer = capturer
|
||||
self._store: BackgroundBlockObjectStore | None = None
|
||||
self._buffer: RoutedExpertsBuffer | None = None
|
||||
self._requests: dict[str, _WorkerRequestState] = {}
|
||||
self._generation = 0
|
||||
self._step_metadata: AuxOutputConnectorMetadata | None = None
|
||||
# Steps whose asynchronous copy has not been consumed yet. The engine
|
||||
# consumes step outputs in order, so at most max_concurrent_batches
|
||||
# are outstanding; guarded by _lock against the output thread.
|
||||
self._pending_outputs: list[PendingAuxOutput] = []
|
||||
self._lock = Lock()
|
||||
self._max_concurrent_batches = vllm_config.max_concurrent_batches
|
||||
# Every TP rank participates in capture collectives, but only the
|
||||
# executor output rank owns the auxiliary output data plane.
|
||||
if not get_tp_group().is_first_rank:
|
||||
return
|
||||
|
||||
shape_per_token = self._capturer.shape_per_token
|
||||
dtype: np.dtype[Any] = np.dtype(self._capturer.output_dtype_name)
|
||||
scheduler_block_size, hash_block_size = resolve_kv_cache_block_sizes(
|
||||
kv_cache_config, vllm_config
|
||||
)
|
||||
hashes_per_kv_block = scheduler_block_size // hash_block_size
|
||||
block_nbytes = hash_block_size * int(np.prod(shape_per_token)) * dtype.itemsize
|
||||
max_bytes = vllm_config.aux_output_config.max_bytes
|
||||
if max_bytes is None:
|
||||
max_bytes = kv_cache_config.num_blocks * hashes_per_kv_block * block_nbytes
|
||||
self._store = BackgroundBlockObjectStore(
|
||||
BlockObjectStore(max_bytes=max_bytes, object_nbytes=block_nbytes),
|
||||
max_pending_batches=2 * vllm_config.scheduler_config.max_num_seqs,
|
||||
)
|
||||
self._buffer = RoutedExpertsBuffer(
|
||||
dtype,
|
||||
shape_per_token,
|
||||
hash_block_size,
|
||||
vllm_config.scheduler_config.max_num_seqs,
|
||||
max_num_batched_tokens,
|
||||
vllm_config.max_concurrent_batches,
|
||||
)
|
||||
|
||||
def prepare_output(self, input_batch: InputBatch) -> PendingAuxOutput | None:
|
||||
"""Snapshot one step's R3 tensor for asynchronous CPU transfer."""
|
||||
if self._buffer is None or self._step_metadata is None:
|
||||
return None
|
||||
|
||||
request_ids = list(input_batch.req_ids)
|
||||
query_start_loc = input_batch.query_start_loc_np[: len(request_ids) + 1]
|
||||
num_rows = int(query_start_loc[-1])
|
||||
pending_output = PendingAuxOutput(
|
||||
connector=self,
|
||||
request_ids=request_ids,
|
||||
token_starts=input_batch.num_computed_tokens_np,
|
||||
query_start_loc=query_start_loc,
|
||||
routed_experts_gpu=self._capturer.snapshot_routing_data(num_rows),
|
||||
)
|
||||
with self._lock:
|
||||
assert len(self._pending_outputs) < self._max_concurrent_batches, (
|
||||
"auxiliary output step outputs are not consumed in order"
|
||||
)
|
||||
self._pending_outputs.append(pending_output)
|
||||
for request_id in pending_output.request_ids:
|
||||
self._requests[request_id].pending_outputs += 1
|
||||
return pending_output
|
||||
|
||||
def process_output(self, pending: PendingAuxOutput) -> dict[str, AuxRequestOutput]:
|
||||
"""Commit one consumed R3 snapshot and build request outputs.
|
||||
|
||||
Request teardown deferred by begin_step is completed here once no
|
||||
in-flight step covers the request.
|
||||
"""
|
||||
with self._lock:
|
||||
teardown = []
|
||||
release_keys: list[str] = []
|
||||
try:
|
||||
outputs = self._commit_output(pending)
|
||||
finally:
|
||||
self._pending_outputs.remove(pending)
|
||||
for request_id in pending.request_ids:
|
||||
state = self._requests[request_id]
|
||||
state.pending_outputs -= 1
|
||||
if state.pending_outputs == 0 and state.finished:
|
||||
teardown.append(request_id)
|
||||
release_keys.extend(reversed(state.aux_output_keys))
|
||||
if teardown:
|
||||
self._publish_blocks([], release_keys=release_keys)
|
||||
self._teardown(teardown)
|
||||
return outputs
|
||||
|
||||
def _commit_output(self, pending: PendingAuxOutput) -> dict[str, AuxRequestOutput]:
|
||||
buffer = self._buffer
|
||||
store = self._store
|
||||
assert buffer is not None and store is not None
|
||||
block_size = buffer.block_size
|
||||
|
||||
routed_experts = pending.routed_experts
|
||||
num_sampled = pending.num_sampled
|
||||
num_rejected = pending.num_rejected
|
||||
assert (
|
||||
routed_experts is not None
|
||||
and num_sampled is not None
|
||||
and num_rejected is not None
|
||||
), "auxiliary output CPU copy was not enqueued"
|
||||
|
||||
# Publish the whole batch before materializing any consumer output.
|
||||
materialize_outputs: list[tuple[str, int, int]] = []
|
||||
block_batches = []
|
||||
outputs: dict[str, AuxRequestOutput] = {}
|
||||
|
||||
# Use the ModelRunner's actual batch boundaries rather than rebuilding them.
|
||||
for request_id, token_start, start, end, sampled, rejected in zip(
|
||||
pending.request_ids,
|
||||
pending.token_starts,
|
||||
pending.query_start_loc[:-1],
|
||||
pending.query_start_loc[1:],
|
||||
num_sampled,
|
||||
num_rejected,
|
||||
strict=True,
|
||||
):
|
||||
request_num_tokens = end - start
|
||||
assert request_num_tokens > 0, (
|
||||
"auxiliary output request token count must be positive"
|
||||
)
|
||||
state = self._requests[request_id]
|
||||
|
||||
# Capture precedes speculative acceptance, so discard the rejected
|
||||
# suffix. Batch boundaries still span the full executed range.
|
||||
rejected = int(rejected)
|
||||
assert 0 <= rejected <= request_num_tokens, (
|
||||
"auxiliary output rejected-token count is invalid"
|
||||
)
|
||||
rows = routed_experts[start : end - rejected]
|
||||
|
||||
capture_start = token_start
|
||||
capture_cursor = state.capture_cursor
|
||||
if capture_cursor is None:
|
||||
capture_cursor = capture_start
|
||||
|
||||
assert capture_start >= capture_cursor, (
|
||||
"auxiliary output capture moved backwards"
|
||||
)
|
||||
if capture_start > capture_cursor:
|
||||
# Reattach after an optimistically scheduled suffix was rejected.
|
||||
assert capture_cursor < state.scheduled_cursor, (
|
||||
"auxiliary output capture has an unbacked token gap"
|
||||
)
|
||||
capture_start = capture_cursor
|
||||
|
||||
emit_start = state.emit_cursor
|
||||
# Complete blocks without keys remain pending until a hash update.
|
||||
completed = buffer.capture(request_id, capture_start, rows)
|
||||
token_end = capture_start + len(rows)
|
||||
state.capture_cursor = token_end
|
||||
state.scheduled_cursor = token_start + request_num_tokens
|
||||
block_batches.append((state, completed))
|
||||
|
||||
if sampled > 0 and emit_start <= token_end:
|
||||
if emit_start >= capture_start:
|
||||
outputs[request_id] = AuxRequestOutput(
|
||||
emit_start, rows[emit_start - capture_start :]
|
||||
)
|
||||
state.emit_cursor = token_end
|
||||
else:
|
||||
materialize_outputs.append((request_id, emit_start, token_end))
|
||||
|
||||
# A consumer may reuse a block produced earlier in the same batch.
|
||||
self._publish_blocks(block_batches)
|
||||
|
||||
for request_id, emit_start, token_end in materialize_outputs:
|
||||
state = self._requests[request_id]
|
||||
stored_end = (
|
||||
min(token_end // block_size, len(state.aux_output_keys)) * block_size
|
||||
)
|
||||
if emit_start < stored_end:
|
||||
first_block = emit_start // block_size
|
||||
stored = materialize_routed_experts(
|
||||
store,
|
||||
state.aux_output_keys[first_block : stored_end // block_size],
|
||||
shape_per_token=buffer.shape_per_token,
|
||||
dtype=buffer.dtype,
|
||||
)
|
||||
local_start = emit_start % block_size
|
||||
rows = stored[local_start : local_start + stored_end - emit_start]
|
||||
if stored_end < token_end:
|
||||
rows = np.concatenate(
|
||||
(rows, buffer.read(request_id, stored_end, token_end))
|
||||
)
|
||||
else:
|
||||
rows = buffer.read(request_id, emit_start, token_end)
|
||||
outputs[request_id] = AuxRequestOutput(emit_start, rows)
|
||||
state.emit_cursor = token_end
|
||||
return outputs
|
||||
|
||||
def _publish_blocks(
|
||||
self,
|
||||
batches: list[tuple[_WorkerRequestState, list[tuple[int, np.ndarray]]]],
|
||||
retain_keys: Sequence[str] = (),
|
||||
release_keys: Sequence[str] = (),
|
||||
) -> None:
|
||||
store = self._store
|
||||
buffer = self._buffer
|
||||
assert store is not None and buffer is not None
|
||||
ready_batches = []
|
||||
for state, completed in batches:
|
||||
blocks = state.pending_blocks + completed
|
||||
keyed_end = len(state.aux_output_keys) * buffer.block_size
|
||||
ready = [(start, rows) for start, rows in blocks if start < keyed_end]
|
||||
state.pending_blocks = [
|
||||
(start, buffer.retain_block(rows))
|
||||
for start, rows in blocks
|
||||
if start >= keyed_end
|
||||
]
|
||||
if ready:
|
||||
ready_batches.append((state.aux_output_keys, ready))
|
||||
if ready_batches or retain_keys or release_keys:
|
||||
publish_routed_experts(
|
||||
store,
|
||||
batches=ready_batches,
|
||||
block_size=buffer.block_size,
|
||||
retain_keys=retain_keys,
|
||||
release_keys=release_keys,
|
||||
)
|
||||
for _, blocks in ready_batches:
|
||||
for _, rows in blocks:
|
||||
buffer.release_block(rows)
|
||||
|
||||
def _teardown(self, request_ids: Iterable[str]) -> None:
|
||||
"""Drop finished requests' temporary state after publishing key releases."""
|
||||
buffer = self._buffer
|
||||
assert buffer is not None
|
||||
for request_id in request_ids:
|
||||
state = self._requests.pop(request_id)
|
||||
for _, rows in state.pending_blocks:
|
||||
buffer.release_block(rows)
|
||||
buffer.discard(request_id)
|
||||
|
||||
def begin_step(self, metadata: AuxOutputConnectorMetadata | None) -> None:
|
||||
"""Apply one scheduler step's request and block-hash updates.
|
||||
|
||||
While an unconsumed step covers a finished request, its teardown is
|
||||
deferred to that step's process_output.
|
||||
"""
|
||||
self._step_metadata = metadata
|
||||
if self._buffer is None or metadata is None:
|
||||
return
|
||||
assert not metadata.requests.keys() & metadata.finished_requests, (
|
||||
"auxiliary output request cannot run and finish in one step"
|
||||
)
|
||||
assert metadata.generation >= self._generation, (
|
||||
"auxiliary output metadata generation moved backwards"
|
||||
)
|
||||
with self._lock:
|
||||
release_keys: list[str] = []
|
||||
if metadata.generation > self._generation:
|
||||
# A generation change follows a prefix-cache reset, which the
|
||||
# scheduler only performs once all model output is consumed.
|
||||
assert not self._pending_outputs, (
|
||||
"auxiliary output generation changed with output in flight"
|
||||
)
|
||||
release_keys.extend(
|
||||
key
|
||||
for state in self._requests.values()
|
||||
for key in reversed(state.aux_output_keys)
|
||||
)
|
||||
self._buffer.reset()
|
||||
self._requests.clear()
|
||||
self._generation = metadata.generation
|
||||
for request_id, emit_start in metadata.requests.items():
|
||||
state = self._requests.setdefault(
|
||||
request_id, _WorkerRequestState(emit_cursor=emit_start)
|
||||
)
|
||||
if state.pending_outputs == 0:
|
||||
# In-flight steps leave the worker cursor behind the
|
||||
# scheduler's optimistic view; only check settled requests.
|
||||
assert emit_start <= state.emit_cursor, (
|
||||
"auxiliary output Scheduler emit cursor moved ahead"
|
||||
)
|
||||
block_batches: list[
|
||||
tuple[_WorkerRequestState, list[tuple[int, np.ndarray]]]
|
||||
] = []
|
||||
retained_keys: list[str] = []
|
||||
for request_id, block_hashes in metadata.block_hashes.items():
|
||||
state = self._requests[request_id]
|
||||
keys = routed_experts_keys(block_hashes, str(self._generation))
|
||||
state.aux_output_keys.extend(keys)
|
||||
retained_keys.extend(keys)
|
||||
block_batches.append((state, []))
|
||||
finished_now: list[str] = []
|
||||
for request_id in metadata.finished_requests:
|
||||
state = self._requests[request_id]
|
||||
state.finished = True
|
||||
if state.pending_outputs:
|
||||
continue
|
||||
finished_now.append(request_id)
|
||||
release_keys.extend(reversed(state.aux_output_keys))
|
||||
self._publish_blocks(block_batches, retained_keys, release_keys)
|
||||
self._teardown(finished_now)
|
||||
|
||||
def close(self) -> None:
|
||||
with self._lock:
|
||||
self._pending_outputs.clear()
|
||||
if self._store is not None:
|
||||
try:
|
||||
self._store.close()
|
||||
finally:
|
||||
self._store = None
|
||||
|
||||
|
||||
def get_aux_output_connector(
|
||||
model: torch.nn.Module, vllm_config: VllmConfig, kv_cache_config: KVCacheConfig
|
||||
) -> AuxOutputWorkerConnector:
|
||||
return AuxOutputWorkerConnector(
|
||||
model=model,
|
||||
kv_cache_config=kv_cache_config,
|
||||
vllm_config=vllm_config,
|
||||
)
|
||||
@@ -35,6 +35,7 @@ from typing_extensions import TypeIs
|
||||
import vllm.envs as envs
|
||||
from vllm.config import (
|
||||
AttentionConfig,
|
||||
AuxOutputConfig,
|
||||
CacheConfig,
|
||||
CompilationConfig,
|
||||
ConfigType,
|
||||
@@ -447,7 +448,9 @@ class EngineArgs:
|
||||
"""Arguments for vLLM engine."""
|
||||
|
||||
model: str = ModelConfig.model
|
||||
enable_return_routed_experts: bool = ModelConfig.enable_return_routed_experts
|
||||
# Public compatibility argument. Canonical runtime state lives in
|
||||
# AuxOutputConfig.
|
||||
enable_return_routed_experts: bool = False
|
||||
return_sampling_mask: bool = ModelConfig.return_sampling_mask
|
||||
model_weights: str = ModelConfig.model_weights
|
||||
served_model_name: str | list[str] | None = ModelConfig.served_model_name
|
||||
@@ -674,6 +677,7 @@ class EngineArgs:
|
||||
structured_outputs_config: StructuredOutputsConfig = get_field(
|
||||
VllmConfig, "structured_outputs_config"
|
||||
)
|
||||
aux_output_config: AuxOutputConfig = get_field(VllmConfig, "aux_output_config")
|
||||
reasoning_parser: str = StructuredOutputsConfig.reasoning_parser
|
||||
reasoning_parser_plugin: str | None = None
|
||||
|
||||
@@ -816,6 +820,10 @@ class EngineArgs:
|
||||
self.compilation_config = CompilationConfig(**self.compilation_config)
|
||||
if isinstance(self.attention_config, dict):
|
||||
self.attention_config = AttentionConfig(**self.attention_config)
|
||||
if isinstance(self.aux_output_config, dict):
|
||||
self.aux_output_config = AuxOutputConfig(**self.aux_output_config)
|
||||
if self.enable_return_routed_experts:
|
||||
self.aux_output_config.enable_return_routed_experts = True
|
||||
if isinstance(self.engram_config, dict):
|
||||
self.engram_config = EngramConfig(**self.engram_config)
|
||||
if isinstance(self.mamba_config, dict):
|
||||
@@ -884,6 +892,7 @@ class EngineArgs:
|
||||
"""Shared CLI arguments for vLLM engine."""
|
||||
# Model arguments
|
||||
model_kwargs = get_kwargs(ModelConfig)
|
||||
aux_output_kwargs = get_kwargs(AuxOutputConfig)
|
||||
model_group = parser.add_argument_group(
|
||||
title="ModelConfig",
|
||||
description=ModelConfig.__doc__,
|
||||
@@ -923,7 +932,7 @@ class EngineArgs:
|
||||
model_group.add_argument("--enforce-eager", **model_kwargs["enforce_eager"])
|
||||
model_group.add_argument(
|
||||
"--enable-return-routed-experts",
|
||||
**model_kwargs["enable_return_routed_experts"],
|
||||
**aux_output_kwargs["enable_return_routed_experts"],
|
||||
)
|
||||
model_group.add_argument(
|
||||
"--return-sampling-mask",
|
||||
@@ -1749,6 +1758,9 @@ class EngineArgs:
|
||||
vllm_group.add_argument(
|
||||
"--structured-outputs-config", **vllm_kwargs["structured_outputs_config"]
|
||||
)
|
||||
vllm_group.add_argument(
|
||||
"--aux-output-config", **vllm_kwargs["aux_output_config"]
|
||||
)
|
||||
vllm_group.add_argument("--profiler-config", **vllm_kwargs["profiler_config"])
|
||||
vllm_group.add_argument(
|
||||
"--optimization-level", **vllm_kwargs["optimization_level"]
|
||||
@@ -1856,7 +1868,6 @@ class EngineArgs:
|
||||
quantization_config=self.quantization_config,
|
||||
allow_deprecated_quantization=self.allow_deprecated_quantization,
|
||||
enforce_eager=self.enforce_eager,
|
||||
enable_return_routed_experts=self.enable_return_routed_experts,
|
||||
return_sampling_mask=self.return_sampling_mask,
|
||||
max_logprobs=self.max_logprobs,
|
||||
logprobs_mode=self.logprobs_mode,
|
||||
@@ -2669,6 +2680,7 @@ class EngineArgs:
|
||||
load_config=load_config,
|
||||
offload_config=offload_config,
|
||||
attention_config=attention_config,
|
||||
aux_output_config=self.aux_output_config,
|
||||
engram_config=self.engram_config,
|
||||
mamba_config=mamba_config,
|
||||
kernel_config=kernel_config,
|
||||
|
||||
@@ -10,15 +10,12 @@ from collections.abc import Callable
|
||||
from functools import partial
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.parallel_state import get_tp_group
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, is_full_attention_spec
|
||||
from vllm.v1.outputs import RoutedExpertsTensors
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -46,37 +43,29 @@ def _get_routed_experts_shape(vllm_config: VllmConfig) -> tuple[int, int, int]:
|
||||
class RoutedExpertsCapturer:
|
||||
"""Worker-side capturer for routed experts, lives on GPU.
|
||||
|
||||
Layer-level hooks call :meth:`capture` from inside the forward pass
|
||||
with the per-layer ``topk_ids`` tensor. The tensor is sliced to the
|
||||
tokens owned by this DP rank and written into a preallocated device
|
||||
buffer. At the end of the step, :class:`GPUModelRunner` reads the
|
||||
device buffer, issues a D2H copy into a pinned CPU buffer, and hands
|
||||
the result to the scheduler via :class:`RoutedExpertsLists`.
|
||||
Layer-level hooks call :meth:`capture` inside the forward pass. Routing
|
||||
rows owned by this DP rank are written into a preallocated device buffer.
|
||||
|
||||
The device / pinned-CPU transit buffers use ``torch.int32`` (not a
|
||||
narrow ``uint8``/``uint16`` sized by ``num_experts``). This keeps the
|
||||
SP all-gather path free of dtype casts, matches the router's native
|
||||
``topk_ids`` indices dtype more closely, and costs only a few MB per
|
||||
worker (``max_num_batched_tokens * num_layers * top_k * 4`` bytes).
|
||||
The scheduler-side slot buffer
|
||||
(``RoutedExpertsManager.routed_experts_by_slot``) still uses the
|
||||
narrow dtype -- numpy fancy-index assignment in ``store_batch``
|
||||
narrows the data on the way in.
|
||||
The device buffer uses ``int32``. Stable snapshots use the narrowest dtype
|
||||
that can represent every logical expert ID.
|
||||
|
||||
Invariants:
|
||||
- One instance per worker; shape is fixed at init and covers the
|
||||
worst-case step (``max_num_batched_tokens`` tokens).
|
||||
- Every routed layer overwrites the current step's token rows.
|
||||
- ``device_buffer.dtype`` is ``torch.int32``.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
max_num_batched_tokens: int,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
) -> None:
|
||||
num_layers, _, num_experts_per_tok = _get_routed_experts_shape(vllm_config)
|
||||
num_layers, num_experts, num_experts_per_tok = _get_routed_experts_shape(
|
||||
vllm_config
|
||||
)
|
||||
self.shape_per_token = (num_layers, num_experts_per_tok)
|
||||
self.output_dtype_name = "uint8" if num_experts <= 256 else "uint16"
|
||||
self.output_dtype = getattr(torch, self.output_dtype_name)
|
||||
logger.info(
|
||||
"RoutedExpertsCapturer: allocating buffer with "
|
||||
"max_tokens=%d, num_layers=%d, num_experts_per_tok=%d "
|
||||
@@ -87,23 +76,12 @@ class RoutedExpertsCapturer:
|
||||
vllm_config.model_config.hf_text_config.model_type,
|
||||
)
|
||||
self.device_buffer = torch.zeros(
|
||||
(
|
||||
max_num_batched_tokens,
|
||||
num_layers,
|
||||
num_experts_per_tok,
|
||||
),
|
||||
# Use int32 for the device / host transit buffers: it
|
||||
# matches the router's native topk_ids dtype, is universally
|
||||
# supported by NCCL (uint8/uint16 are version-dependent),
|
||||
# and the extra bytes are small (few MB per worker). The
|
||||
# big scheduler-side slot buffer stays narrow.
|
||||
(max_num_batched_tokens, *self.shape_per_token),
|
||||
dtype=torch.int32,
|
||||
device=current_platform.device_type,
|
||||
)
|
||||
self.dp_rank = vllm_config.parallel_config.data_parallel_rank
|
||||
self.tp_size = vllm_config.parallel_config.tensor_parallel_size
|
||||
# KV cache group whose slot layout the routing data is keyed by.
|
||||
self.attn_gid = get_routed_experts_attn_gid(kv_cache_config)
|
||||
|
||||
def capture(self, layer_id: int, topk_ids: torch.Tensor) -> None:
|
||||
"""Capture expert routing decisions for a specific layer.
|
||||
@@ -139,31 +117,21 @@ class RoutedExpertsCapturer:
|
||||
|
||||
"""
|
||||
ctx = get_forward_context()
|
||||
if ctx.dp_metadata is None: # single dp
|
||||
start_loc = 0
|
||||
end_loc = topk_ids.shape[0]
|
||||
token_num_per_dp = topk_ids.shape[0]
|
||||
else: # multi dp
|
||||
if ctx.dp_metadata is None:
|
||||
local_topk_ids = topk_ids
|
||||
else:
|
||||
num_tokens_dp = ctx.dp_metadata.num_tokens_across_dp_cpu
|
||||
token_num_per_dp = int(num_tokens_dp[self.dp_rank].item())
|
||||
num_local_tokens = int(num_tokens_dp[self.dp_rank].item())
|
||||
total = int(num_tokens_dp.sum().item())
|
||||
n = topk_ids.shape[0]
|
||||
shard_sizes = getattr(ctx.dp_metadata, "local_sizes", None)
|
||||
gathered_size = sum(shard_sizes) if shard_sizes is not None else -1
|
||||
|
||||
if n == total:
|
||||
# Naive dispatch: all DP ranks' tokens concatenated
|
||||
# before routing. This rank owns tokens
|
||||
# [end_loc - token_num_per_dp, end_loc).
|
||||
cumsum = torch.cumsum(num_tokens_dp, dim=0)
|
||||
end_loc = int(cumsum[self.dp_rank].item())
|
||||
start_loc = end_loc - token_num_per_dp
|
||||
elif n == token_num_per_dp:
|
||||
# Modular-kernel path: DP combine happens inside
|
||||
# quant_method.apply; select_experts only sees this
|
||||
# rank's tokens, take the whole tensor.
|
||||
start_loc = 0
|
||||
end_loc = token_num_per_dp
|
||||
end = int(num_tokens_dp[: self.dp_rank + 1].sum().item())
|
||||
local_topk_ids = topk_ids[end - num_local_tokens : end]
|
||||
elif n == num_local_tokens:
|
||||
local_topk_ids = topk_ids
|
||||
elif shard_sizes is not None and n == gathered_size:
|
||||
# Naive DP+EP dispatch gathers the sequence-parallel shards in
|
||||
# flattened DP-then-TP order. ``local_sizes`` is the exact
|
||||
@@ -173,12 +141,11 @@ class RoutedExpertsCapturer:
|
||||
assert len(shard_sizes) % num_dp_ranks == 0
|
||||
shards_per_dp_rank = len(shard_sizes) // num_dp_ranks
|
||||
first_shard = self.dp_rank * shards_per_dp_rank
|
||||
start_loc = sum(shard_sizes[:first_shard])
|
||||
end_loc = start_loc + token_num_per_dp
|
||||
start = sum(shard_sizes[:first_shard])
|
||||
local_topk_ids = topk_ids[start : start + num_local_tokens]
|
||||
elif (
|
||||
self.tp_size > 1
|
||||
and n != token_num_per_dp
|
||||
and n == (token_num_per_dp + self.tp_size - 1) // self.tp_size
|
||||
and n == (num_local_tokens + self.tp_size - 1) // self.tp_size
|
||||
):
|
||||
# SP + modular-kernel path. All-gather across the TP
|
||||
# group along dim=0 to reconstruct the full per-DP-rank
|
||||
@@ -193,60 +160,30 @@ class RoutedExpertsCapturer:
|
||||
#
|
||||
# ``topk_ids`` is already whatever the router produced
|
||||
# (typically int32/int64, both supported by NCCL); the
|
||||
# downstream ``device_buffer[...] = topk_ids[...]``
|
||||
# setitem narrows into int32 automatically.
|
||||
topk_ids = get_tp_group().all_gather(topk_ids, dim=0)
|
||||
start_loc = 0
|
||||
end_loc = token_num_per_dp
|
||||
# downstream buffer assignment narrows to the capture dtype.
|
||||
local_topk_ids = get_tp_group().all_gather(topk_ids, dim=0)[
|
||||
:num_local_tokens
|
||||
]
|
||||
else:
|
||||
sp_expected = (
|
||||
(token_num_per_dp + self.tp_size - 1) // self.tp_size
|
||||
if self.tp_size > 0
|
||||
else -1
|
||||
)
|
||||
sp_expected = (num_local_tokens + self.tp_size - 1) // self.tp_size
|
||||
raise AssertionError(
|
||||
"RoutedExpertsCapturer: unexpected topk_ids batch "
|
||||
f"dim {n} (expected {total}, {token_num_per_dp}, "
|
||||
f"dim {n} (expected {total}, {num_local_tokens}, "
|
||||
f"{gathered_size}, or {sp_expected} for "
|
||||
f"dp_rank={self.dp_rank}, tp_size={self.tp_size})"
|
||||
)
|
||||
|
||||
if layer_id >= self.device_buffer.shape[1]:
|
||||
if not 0 <= layer_id < self.device_buffer.shape[1]:
|
||||
raise IndexError(
|
||||
f"routed-experts layer {layer_id} exceeds capture buffer "
|
||||
f"layer count {self.device_buffer.shape[1]}"
|
||||
)
|
||||
|
||||
self.device_buffer[:token_num_per_dp, layer_id, :] = topk_ids[
|
||||
start_loc:end_loc, :
|
||||
]
|
||||
self.device_buffer[: len(local_topk_ids), layer_id] = local_topk_ids
|
||||
|
||||
def get_device_buffer(self) -> torch.Tensor:
|
||||
"""Return the underlying device buffer so the model runner can
|
||||
issue the D2H copy. The tensor is shared; callers must either
|
||||
clone or fully drain it before the next forward pass overwrites it.
|
||||
"""
|
||||
return self.device_buffer
|
||||
|
||||
def get_routed_experts(
|
||||
self, slot_mappings: torch.Tensor, num_tokens: int
|
||||
) -> RoutedExpertsTensors:
|
||||
"""Snapshot this step's routing data and its attention slot mapping.
|
||||
|
||||
Both tensors are cloned since the capture buffer and the shared
|
||||
``slot_mappings`` are overwritten by the next step, which may race
|
||||
with an in-flight D2H copy.
|
||||
|
||||
Args:
|
||||
slot_mappings: Per-KV-cache-group slot mappings for this step,
|
||||
shape ``(num_kv_cache_groups, max_num_batched_tokens)``.
|
||||
num_tokens: Total number of tokens scheduled in this step.
|
||||
|
||||
"""
|
||||
return RoutedExpertsTensors(
|
||||
routing_data=self.device_buffer[:num_tokens].clone(),
|
||||
slot_mapping=slot_mappings[self.attn_gid, :num_tokens].clone(),
|
||||
)
|
||||
def snapshot_routing_data(self, num_tokens: int) -> torch.Tensor:
|
||||
"""Return a stable snapshot of the current routing data."""
|
||||
return self.device_buffer[:num_tokens].to(self.output_dtype)
|
||||
|
||||
|
||||
def bind_routed_experts_capturer(
|
||||
@@ -268,20 +205,12 @@ def bind_routed_experts_capturer(
|
||||
continue
|
||||
if not isinstance(module, MoERunner):
|
||||
continue
|
||||
layer_id = module.layer_id
|
||||
|
||||
def capture_fn(
|
||||
topk_ids: torch.Tensor,
|
||||
layer_id: int = layer_id,
|
||||
capturer: RoutedExpertsCapturer = capturer,
|
||||
) -> None:
|
||||
capturer.capture(layer_id, topk_ids)
|
||||
|
||||
capture_fn = partial(capturer.capture, module.layer_id)
|
||||
quant_method = module._quant_method
|
||||
moe_kernel = getattr(quant_method, "moe_kernel", None)
|
||||
impl = getattr(moe_kernel, "impl", None)
|
||||
fused_experts = getattr(impl, "fused_experts", None)
|
||||
if quant_method.is_monolithic:
|
||||
moe_kernel = getattr(quant_method, "moe_kernel", None)
|
||||
impl = getattr(moe_kernel, "impl", None)
|
||||
fused_experts = getattr(impl, "fused_experts", None)
|
||||
if not (
|
||||
isinstance(fused_experts, FusedMoEExpertsMonolithic)
|
||||
and fused_experts.supports_routing_replay_capture()
|
||||
@@ -303,132 +232,3 @@ def bind_routed_experts_capturer(
|
||||
|
||||
if num_bound == 0:
|
||||
raise ValueError("No supported MoE router found for routed-experts capture.")
|
||||
|
||||
|
||||
def get_routed_experts_attn_gid(kv_cache_config: KVCacheConfig) -> int:
|
||||
"""Return the full-attention KV cache group used for routed experts."""
|
||||
for gid, group in enumerate(kv_cache_config.kv_cache_groups):
|
||||
if is_full_attention_spec(group.kv_cache_spec):
|
||||
return gid
|
||||
raise ValueError("Routed-experts capture requires a full-attention KV cache group.")
|
||||
|
||||
|
||||
class RoutedExpertsManager:
|
||||
"""Scheduler-side slot-indexed buffer for routed experts.
|
||||
|
||||
Lives on CPU in the scheduler process. Each slot corresponds to
|
||||
``block_id * block_size + offset_in_block`` where ``block_id`` is
|
||||
drawn from the physical KV-cache block pool, so routing data is
|
||||
tied to physical blocks and naturally survives preemption for
|
||||
prefix-cached blocks (prefix hits re-expose the same slots).
|
||||
|
||||
Data flow per step:
|
||||
1. Worker D2Hs its device capture buffer into
|
||||
:class:`RoutedExpertsLists` and returns it via
|
||||
:class:`ModelRunnerOutput`.
|
||||
2. Scheduler calls :meth:`store_batch` with that step's
|
||||
``(routing_data, slot_mapping)`` — a single CPU->CPU
|
||||
fancy-index assign, ~few MB per step.
|
||||
3. On request completion / abort / preemption, the scheduler
|
||||
calls :meth:`get` with the request's block IDs to recover
|
||||
the full per-token routing.
|
||||
|
||||
Memory: ``routed_experts_by_slot`` is sized for the whole block
|
||||
pool (``num_blocks * block_size`` slots). For large block pools
|
||||
this can reach multiple GB; see the init log for the exact size.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
) -> None:
|
||||
# Hybrid models also have KV groups whose slot layout differs.
|
||||
self.attn_gid = get_routed_experts_attn_gid(kv_cache_config)
|
||||
attn_group = kv_cache_config.kv_cache_groups[self.attn_gid]
|
||||
self.block_size = attn_group.kv_cache_spec.block_size
|
||||
|
||||
# All kv_cache_groups share the same physical block pool, so
|
||||
# block IDs span [0, num_blocks) regardless of how many groups
|
||||
# exist. Sizing to the full pool avoids index-out-of-range
|
||||
# when different groups happen to land on the same block.
|
||||
num_layers, num_experts, num_experts_per_tok = _get_routed_experts_shape(
|
||||
vllm_config
|
||||
)
|
||||
max_num_slots = kv_cache_config.num_blocks * self.block_size
|
||||
# Expert IDs are 0..num_experts-1; uint8 fits 256 distinct
|
||||
# values so the boundary is ``<= 256`` (NOT ``< 256``). Keeping
|
||||
# this narrow matters because the slot buffer is sized for the
|
||||
# whole block pool and can reach multiple GB.
|
||||
expert_id_dtype = np.uint8 if num_experts <= 256 else np.uint16
|
||||
self.routed_experts_by_slot = np.zeros(
|
||||
(
|
||||
max_num_slots,
|
||||
num_layers,
|
||||
num_experts_per_tok,
|
||||
),
|
||||
dtype=expert_id_dtype,
|
||||
)
|
||||
logger.info(
|
||||
"RoutedExpertsManager CPU buffer: %.2f GB "
|
||||
"(slots=%d, layers=%d, top_k=%d, dtype=%s)",
|
||||
self.routed_experts_by_slot.nbytes / 1e9,
|
||||
max_num_slots,
|
||||
num_layers,
|
||||
num_experts_per_tok,
|
||||
self.routed_experts_by_slot.dtype.name,
|
||||
)
|
||||
|
||||
def store_batch(self, data: np.ndarray, slot_mapping: np.ndarray) -> None:
|
||||
"""Persist one step's routed experts into the slot buffer.
|
||||
|
||||
Equivalent to ``slot_buffer[slot_mapping] = data``; numpy fancy
|
||||
indexing handles repeated / out-of-order indices. Called once
|
||||
per scheduler step in ``update_from_output``.
|
||||
"""
|
||||
self.routed_experts_by_slot[slot_mapping] = data
|
||||
|
||||
def get(
|
||||
self,
|
||||
block_ids: list[int],
|
||||
num_tokens: int,
|
||||
token_start: int = 0,
|
||||
) -> np.ndarray:
|
||||
"""Read routed experts data for a completed / preempted request.
|
||||
|
||||
Reconstructs a per-token slot_mapping from the request's block
|
||||
IDs and returns the routing slice. Because numpy fancy indexing
|
||||
returns a **copy** (not a view), the returned ndarray is safe
|
||||
to hold across subsequent :meth:`store_batch` calls — do not
|
||||
replace the fancy index with a slice without re-verifying.
|
||||
|
||||
Args:
|
||||
block_ids: Block IDs from the attention KV-cache group.
|
||||
num_tokens: Number of tokens that have gone through a forward
|
||||
pass and therefore have routing data written to their
|
||||
slots (typically ``request.num_tokens - 1``; the last
|
||||
sampled token has not been forwarded yet). Slots beyond
|
||||
``request.num_computed_tokens`` are zero-initialized.
|
||||
token_start: Skip the first ``token_start`` tokens from the
|
||||
result. The slot_mapping is sliced before the fancy-index
|
||||
read, so only the requested slots are fetched — no large
|
||||
intermediate array is allocated. Clamped to
|
||||
``[0, num_tokens]`` automatically.
|
||||
|
||||
Returns:
|
||||
Array of shape (num_tokens - token_start, num_layers,
|
||||
num_experts_per_tok).
|
||||
|
||||
"""
|
||||
bs = self.block_size
|
||||
block_ids_array = np.array(block_ids, dtype=np.int32)
|
||||
block_offsets = np.arange(bs)
|
||||
# slot = block_id * block_size + offset_in_block; flatten the
|
||||
# (num_blocks, block_size) grid and trim to num_tokens, then
|
||||
# skip the first token_start entries so only the requested
|
||||
# range is fetched in a single fancy-index read.
|
||||
slot_mapping = (
|
||||
block_ids_array.reshape(-1, 1) * bs + block_offsets.reshape(1, -1)
|
||||
).flatten()[:num_tokens]
|
||||
slot_mapping = slot_mapping[token_start:]
|
||||
return self.routed_experts_by_slot[slot_mapping]
|
||||
|
||||
@@ -194,6 +194,10 @@ class RequestOutput:
|
||||
completion.cumulative_logprob = (
|
||||
next_completion.cumulative_logprob
|
||||
)
|
||||
# R3 is returned on the terminal output and must survive
|
||||
# aggregation with earlier chunks that have no R3.
|
||||
if next_completion.routed_experts is not None:
|
||||
completion.routed_experts = next_completion.routed_experts
|
||||
completion.finish_reason = next_completion.finish_reason
|
||||
completion.stop_reason = next_completion.stop_reason
|
||||
else:
|
||||
|
||||
@@ -14,6 +14,9 @@ if TYPE_CHECKING:
|
||||
import numpy.typing as npt
|
||||
import torch
|
||||
|
||||
from vllm.distributed.aux_output_connector.connector import (
|
||||
AuxOutputConnectorMetadata,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorMetadata
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata
|
||||
from vllm.lora.request import LoRARequest
|
||||
@@ -23,6 +26,7 @@ if TYPE_CHECKING:
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
|
||||
from vllm.v1.request import Request
|
||||
else:
|
||||
AuxOutputConnectorMetadata = object
|
||||
ECConnectorMetadata = object
|
||||
KVConnectorMetadata = object
|
||||
KVCacheBlockCopy = object
|
||||
@@ -285,6 +289,9 @@ class SchedulerOutput:
|
||||
# synchronously during this step (load_async=False).
|
||||
has_sync_kv_loads: bool = False
|
||||
|
||||
# Execution-auxiliary output control metadata consumed by the worker connector.
|
||||
aux_output_connector_metadata: AuxOutputConnectorMetadata | None = None
|
||||
|
||||
# EC Cache Connector metadata
|
||||
ec_connector_metadata: ECConnectorMetadata | None = None
|
||||
# EC Cache Manager metadata
|
||||
|
||||
@@ -9,6 +9,7 @@ from typing import Any
|
||||
|
||||
from vllm.compilation.cuda_graph import CUDAGraphStat
|
||||
from vllm.config import KVEventsConfig, VllmConfig
|
||||
from vllm.distributed.aux_output_connector.connector import AuxOutputSchedulerConnector
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import (
|
||||
ECConnectorBase,
|
||||
ECConnectorMetadata,
|
||||
@@ -26,9 +27,6 @@ from vllm.distributed.kv_transfer.kv_connector.v1 import (
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
RoutedExpertsManager,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
|
||||
from vllm.multimodal.encoder_budget import MultiModalBudget
|
||||
from vllm.multimodal.utils import get_mm_features_in_window
|
||||
@@ -393,26 +391,13 @@ class Scheduler(SchedulerInterface):
|
||||
if self.log_stats and vllm_config.observability_config.enable_mfu_metrics:
|
||||
self.perf_metrics = ModelMetrics(vllm_config)
|
||||
|
||||
self.enable_return_routed_experts = (
|
||||
vllm_config.model_config.enable_return_routed_experts
|
||||
self.aux_output_connector = (
|
||||
AuxOutputSchedulerConnector()
|
||||
if vllm_config.aux_output_config.enabled
|
||||
else None
|
||||
)
|
||||
self.return_sampling_mask = vllm_config.model_config.return_sampling_mask
|
||||
|
||||
if self.enable_return_routed_experts:
|
||||
assert self.dcp_world_size == 1 and self.pcp_world_size == 1, (
|
||||
"enable_return_routed_experts does not support context parallelism "
|
||||
"(dcp_world_size > 1 or pcp_world_size > 1)"
|
||||
)
|
||||
|
||||
self.routed_experts_mgr = RoutedExpertsManager(
|
||||
vllm_config=vllm_config,
|
||||
kv_cache_config=kv_cache_config,
|
||||
)
|
||||
# Block-ID snapshot taken at schedule time (before forward),
|
||||
# so update_from_output can read slot data even if a later
|
||||
# schedule() frees the blocks (async scheduling race).
|
||||
self._re_block_ids: dict[str, list[int]] = {}
|
||||
|
||||
self._pause_state: PauseState = PauseState.UNPAUSED
|
||||
|
||||
# In-flight requests still prefilling (prefill chunks + in-progress
|
||||
@@ -1487,6 +1472,13 @@ class Scheduler(SchedulerInterface):
|
||||
meta = self._build_kv_connector_meta(self.connector, scheduler_output)
|
||||
scheduler_output.kv_connector_metadata = meta
|
||||
|
||||
if self.aux_output_connector is not None:
|
||||
scheduler_output.aux_output_connector_metadata = (
|
||||
self.aux_output_connector.build_connector_meta(
|
||||
scheduler_output, self.requests
|
||||
)
|
||||
)
|
||||
|
||||
# Build the connector meta for ECConnector
|
||||
if self.ec_connector is not None:
|
||||
ec_meta: ECConnectorMetadata = self.ec_connector.build_connector_meta(
|
||||
@@ -1541,6 +1533,8 @@ class Scheduler(SchedulerInterface):
|
||||
assert request.status == RequestStatus.RUNNING, (
|
||||
"Only running requests can be preempted"
|
||||
)
|
||||
if self.aux_output_connector is not None:
|
||||
self.aux_output_connector.request_finished(request)
|
||||
self._free_request_blocks(request)
|
||||
self.encoder_cache_manager.free(request)
|
||||
self._inflight_prefills.discard(request)
|
||||
@@ -1596,22 +1590,6 @@ class Scheduler(SchedulerInterface):
|
||||
if not request.is_prefill_chunk:
|
||||
self._inflight_prefills.discard(request)
|
||||
|
||||
# Snapshot block IDs for routed experts before forward starts.
|
||||
# A concurrent schedule() may preempt requests and free blocks
|
||||
# before update_from_output runs; the snapshot survives that.
|
||||
# Use update() to preserve entries from the previous step that
|
||||
# have not yet been consumed by update_from_output (async
|
||||
# scheduling may call _update_after_schedule again before the
|
||||
# prior update_from_output runs).
|
||||
if self.enable_return_routed_experts:
|
||||
gid = self.routed_experts_mgr.attn_gid
|
||||
self._re_block_ids.update(
|
||||
{
|
||||
rid: self.kv_cache_manager.get_blocks(rid).get_block_ids()[gid]
|
||||
for rid in num_scheduled_tokens
|
||||
}
|
||||
)
|
||||
|
||||
# Clear the finished and preempted request IDs.
|
||||
# NOTE: We shouldn't just clear() here because it will also affect
|
||||
# the scheduler output.
|
||||
@@ -1962,28 +1940,6 @@ class Scheduler(SchedulerInterface):
|
||||
num_scheduled_tokens,
|
||||
)
|
||||
)
|
||||
|
||||
# Persist per-step routed experts into the scheduler-side slot
|
||||
# buffer (CPU->CPU fancy-index assign; ~few MB per step).
|
||||
# MUST precede the per-request routing reads below: stopped
|
||||
# requests may terminate on tokens generated in this very step,
|
||||
# whose routing was just D2H'd into model_runner_output.
|
||||
routing_data = None
|
||||
routing_offsets: dict[str, int] = {}
|
||||
if model_runner_output.routed_experts is not None:
|
||||
re = model_runner_output.routed_experts
|
||||
self.routed_experts_mgr.store_batch(re.routing_data, re.slot_mapping)
|
||||
routing_data = re.routing_data.astype(
|
||||
self.routed_experts_mgr.routed_experts_by_slot.dtype,
|
||||
copy=False,
|
||||
)
|
||||
# Build offset map using model runner's request order
|
||||
# (input_batch ordering), NOT scheduler dict order.
|
||||
offset = 0
|
||||
for rid in model_runner_output.req_ids:
|
||||
routing_offsets[rid] = offset
|
||||
offset += num_scheduled_tokens[rid]
|
||||
|
||||
# NOTE(woosuk): As len(num_scheduled_tokens) can be up to 1K or more,
|
||||
# the below loop can be a performance bottleneck. We should do our best
|
||||
# to avoid expensive operations inside the loop.
|
||||
@@ -2076,7 +2032,6 @@ class Scheduler(SchedulerInterface):
|
||||
ec_transfer_params = None
|
||||
prefill_stats = None
|
||||
status_before_stop = request.status
|
||||
num_output_tokens_before = len(request._output_token_ids)
|
||||
|
||||
# Check for stop and update request status.
|
||||
if new_token_ids:
|
||||
@@ -2113,48 +2068,13 @@ class Scheduler(SchedulerInterface):
|
||||
stopped = True
|
||||
|
||||
routed_experts = None
|
||||
if (
|
||||
self.enable_return_routed_experts
|
||||
and routing_data is not None
|
||||
and new_token_ids
|
||||
):
|
||||
req_offset = routing_offsets[req_id]
|
||||
end = req_offset + num_tokens_scheduled
|
||||
block_ids = self._re_block_ids.pop(req_id, [])
|
||||
if num_output_tokens_before == 0:
|
||||
# Prefill completed: read full prompt routing from
|
||||
# slot buffer using the block-ID snapshot taken at
|
||||
# schedule time (immune to async preemption).
|
||||
if (
|
||||
request.sampling_params is not None
|
||||
and request.sampling_params.routed_experts_prompt_start
|
||||
is not None
|
||||
):
|
||||
prompt_start = (
|
||||
request.sampling_params.routed_experts_prompt_start
|
||||
)
|
||||
assert prompt_start <= request.num_prompt_tokens
|
||||
else:
|
||||
prompt_start = 0
|
||||
routed_experts = self.routed_experts_mgr.get(
|
||||
block_ids,
|
||||
request.num_prompt_tokens,
|
||||
token_start=prompt_start,
|
||||
)
|
||||
else:
|
||||
if scheduled_spec_token_ids:
|
||||
# Spec decode: accepted tokens at the START of
|
||||
# the scheduled range, rejected at the end.
|
||||
routed_experts = routing_data[
|
||||
req_offset : req_offset + len(new_token_ids)
|
||||
]
|
||||
else:
|
||||
# Normal decode / re-prefill: token(s) at the END.
|
||||
routed_experts = routing_data[end - len(new_token_ids) : end]
|
||||
|
||||
should_emit_output = bool(
|
||||
new_token_ids or pooler_output is not None or stopped
|
||||
)
|
||||
if self.aux_output_connector is not None and should_emit_output:
|
||||
routed_experts = self.aux_output_connector.take_output(
|
||||
request, model_runner_output.aux_output_connector_output
|
||||
)
|
||||
if should_emit_output:
|
||||
prefill_stats = request.take_prefill_stats()
|
||||
if prefill_stats is not None:
|
||||
@@ -2641,6 +2561,8 @@ class Scheduler(SchedulerInterface):
|
||||
) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
|
||||
assert request.is_finished()
|
||||
|
||||
if self.aux_output_connector is not None:
|
||||
self.aux_output_connector.request_finished(request)
|
||||
self._inflight_prefills.discard(request)
|
||||
connector_delay_free_blocks, kv_xfer_params = self._connector_finished(request)
|
||||
|
||||
@@ -2773,6 +2695,16 @@ class Scheduler(SchedulerInterface):
|
||||
Otherwise, this method will only reset the KV prefix cache when there
|
||||
is no running requests taking KV cache.
|
||||
"""
|
||||
if reset_running_requests and self.aux_output_connector is not None:
|
||||
if self._pause_state != PauseState.PAUSED_ALL:
|
||||
raise RuntimeError(
|
||||
"AuxOutput Connector only supports resetting running requests "
|
||||
"after pause(mode='keep')."
|
||||
)
|
||||
if any(request.num_in_flight_tokens for request in self.requests.values()):
|
||||
raise RuntimeError(
|
||||
"AuxOutput Connector cannot reset while model output is in flight."
|
||||
)
|
||||
if reset_running_requests:
|
||||
# For logging.
|
||||
timestamp = time.monotonic()
|
||||
@@ -2803,6 +2735,9 @@ class Scheduler(SchedulerInterface):
|
||||
if reset_connector:
|
||||
reset_successful = self.reset_connector_cache() and reset_successful
|
||||
|
||||
if reset_successful and self.aux_output_connector is not None:
|
||||
self.aux_output_connector.reset()
|
||||
|
||||
return reset_successful
|
||||
|
||||
def reset_connector_cache(self) -> bool:
|
||||
|
||||
@@ -233,14 +233,10 @@ def detach_zero_copy_from_model_runner_output(output: "ModelRunnerOutput") -> No
|
||||
token_ids_c, logprobs_c, ranks_c, cu_num_generated_tokens
|
||||
)
|
||||
|
||||
if output.routed_experts is not None:
|
||||
routing_data, slot_mapping = output.routed_experts
|
||||
routing_data_c = _copy_if_readonly(routing_data)
|
||||
slot_mapping_c = _copy_if_readonly(slot_mapping)
|
||||
if routing_data_c is not routing_data or slot_mapping_c is not slot_mapping:
|
||||
output.routed_experts = type(output.routed_experts)(
|
||||
routing_data_c, slot_mapping_c
|
||||
)
|
||||
aux_output = output.aux_output_connector_output
|
||||
if aux_output is not None:
|
||||
for request_output in aux_output.values():
|
||||
request_output.rows = _copy_if_readonly(request_output.rows)
|
||||
|
||||
|
||||
class FutureWrapper(Future):
|
||||
|
||||
+3
-77
@@ -15,6 +15,7 @@ from vllm.utils.torch_utils import PIN_MEMORY
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.aux_output_connector.connector import AuxRequestOutput
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorWorkerMetadata
|
||||
from vllm.distributed.ec_transfer.ec_connector.metrics import ECConnectorStats
|
||||
from vllm.distributed.kv_events import KVConnectorKVEvents
|
||||
@@ -23,6 +24,7 @@ if TYPE_CHECKING:
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats
|
||||
else:
|
||||
AuxRequestOutput = object
|
||||
KVConnectorStats = object
|
||||
KVConnectorWorkerMetadata = object
|
||||
KVConnectorKVEvents = object
|
||||
@@ -188,73 +190,6 @@ class LogprobsTensors(NamedTuple):
|
||||
)
|
||||
|
||||
|
||||
class RoutedExpertsTensors(NamedTuple):
|
||||
"""Device-side snapshot of routed experts data, pending async D2H.
|
||||
|
||||
Produced by :class:`GPUModelRunner` at the end of each async-scheduled
|
||||
step. The copy stream waits on the default stream, then issues
|
||||
non-blocking D2H via :meth:`to_cpu_nonblocking` into a pinned CPU
|
||||
buffer; :class:`AsyncGPUModelRunnerOutput.get_output` synchronizes
|
||||
the copy before the scheduler reads it.
|
||||
|
||||
Sliced to ``total_num_scheduled_tokens`` (step-level, across all
|
||||
requests — NOT per-request). Both ``routing_data`` and
|
||||
``slot_mapping`` must be private clones when sourced from shared
|
||||
capturer / prepare-input buffers, so the next forward pass /
|
||||
``_prepare_inputs`` on the default stream does not race with a
|
||||
D2H still pending on the copy stream.
|
||||
"""
|
||||
|
||||
# (num_scheduled_tokens, num_layers, num_experts_per_tok)
|
||||
routing_data: torch.Tensor
|
||||
# (num_scheduled_tokens,)
|
||||
slot_mapping: torch.Tensor
|
||||
|
||||
def to_cpu_nonblocking(self) -> "RoutedExpertsTensors":
|
||||
"""Issue non-blocking D2H on the current stream.
|
||||
|
||||
NOTE: ``non_blocking=True`` only delivers true overlap when the
|
||||
CPU target is pinned. The current fallback here allocates a
|
||||
new pageable CPU tensor per call, which silently degrades to a
|
||||
synchronous copy; acceptable because the sync happens on the
|
||||
dedicated copy stream, not the default stream.
|
||||
"""
|
||||
if self.routing_data.device.type == "cpu":
|
||||
return self
|
||||
return RoutedExpertsTensors(
|
||||
self.routing_data.to("cpu", non_blocking=True),
|
||||
self.slot_mapping.to("cpu", non_blocking=True),
|
||||
)
|
||||
|
||||
def tolists(self) -> "RoutedExpertsLists":
|
||||
"""Convert to the numpy-backed form consumed by the scheduler.
|
||||
|
||||
``.cpu()`` is a no-op when the tensor is already on CPU, so this
|
||||
is cheap for the post-D2H case; for raw device tensors it will
|
||||
synchronously block, which is only reached in tests.
|
||||
"""
|
||||
return RoutedExpertsLists(
|
||||
self.routing_data.cpu().numpy(),
|
||||
self.slot_mapping.cpu().numpy(),
|
||||
)
|
||||
|
||||
|
||||
class RoutedExpertsLists(NamedTuple):
|
||||
"""CPU-side routed experts, the form :meth:`RoutedExpertsManager.store_batch`
|
||||
consumes.
|
||||
|
||||
Batched per scheduler step: the leading dim is the number of tokens
|
||||
scheduled across all requests in this step (``total_num_scheduled_tokens``),
|
||||
not per-request tokens. ``slot_mapping[i]`` tells the scheduler which
|
||||
physical KV-cache slot row ``i`` of ``routing_data`` belongs to.
|
||||
"""
|
||||
|
||||
# (num_scheduled_tokens, num_layers, num_experts_per_tok)
|
||||
routing_data: np.ndarray
|
||||
# (num_scheduled_tokens,)
|
||||
slot_mapping: np.ndarray
|
||||
|
||||
|
||||
# [num_reqs, <dynamic>]
|
||||
# The shape of each element depends on the pooler used
|
||||
PoolerOutput: TypeAlias = torch.Tensor | list[torch.Tensor] | list[torch.Tensor | None]
|
||||
@@ -361,16 +296,7 @@ class ModelRunnerOutput:
|
||||
# information related to cudagraph execution
|
||||
cudagraph_stats: CUDAGraphStat | None = None
|
||||
|
||||
# Per-step routed experts data captured by the worker.
|
||||
# ``routing_data`` shape: (num_scheduled_tokens, num_layers,
|
||||
# num_experts_per_tok); expert IDs as uint8/uint16.
|
||||
# ``slot_mapping`` shape: (num_scheduled_tokens,); physical KV-cache
|
||||
# slot for each row of routing_data.
|
||||
# ``num_scheduled_tokens`` is step-level (total across all requests
|
||||
# in this step), not per-request. The scheduler persists this into
|
||||
# its slot buffer via ``slot_buffer[slot_mapping] = routing_data``.
|
||||
# ``None`` when ``enable_return_routed_experts`` is off.
|
||||
routed_experts: RoutedExpertsLists | None = None
|
||||
aux_output_connector_output: dict[str, AuxRequestOutput] | None = None
|
||||
|
||||
# ``None`` when ``return_sampling_mask`` is off.
|
||||
sampling_masks: SamplingMaskLists | None = None
|
||||
|
||||
@@ -15,12 +15,12 @@ from vllm.v1.outputs import (
|
||||
LogprobsTensors,
|
||||
ModelRunnerOutput,
|
||||
PoolerOutput,
|
||||
RoutedExpertsTensors,
|
||||
)
|
||||
from vllm.v1.worker.gpu.sample.output import SamplerOutput, SamplingMaskTensors
|
||||
from vllm.v1.worker.utils import raise_if_nan_logits
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.aux_output_connector.worker import PendingAuxOutput
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
|
||||
|
||||
@@ -120,8 +120,8 @@ class AsyncOutput(AsyncModelRunnerOutput):
|
||||
num_sampled_tokens: torch.Tensor,
|
||||
main_stream: torch.cuda.Stream,
|
||||
copy_stream: torch.cuda.Stream,
|
||||
check_ep_fault: bool = False,
|
||||
routed_experts: RoutedExpertsTensors | None = None,
|
||||
check_ep_fault: bool,
|
||||
pending_aux_output: "PendingAuxOutput | None",
|
||||
):
|
||||
# NOTE(woosuk): We must retain references to the GPU tensors,
|
||||
# as the copy operations are performed on a different CUDA stream than
|
||||
@@ -129,7 +129,7 @@ class AsyncOutput(AsyncModelRunnerOutput):
|
||||
self.model_runner_output = model_runner_output
|
||||
self.sampler_output = sampler_output
|
||||
self.num_sampled_tokens = num_sampled_tokens
|
||||
self.routed_experts = routed_experts
|
||||
self.pending_aux_output = pending_aux_output
|
||||
# Blocking (sleep) event to avoid busy-polling the CUDA driver lock.
|
||||
self.copy_event = torch.cuda.Event(blocking=True)
|
||||
self._has_fault: torch.Tensor | None = None
|
||||
@@ -152,13 +152,15 @@ class AsyncOutput(AsyncModelRunnerOutput):
|
||||
self.sampling_mask_tensors = (
|
||||
sampler_output.sampling_mask_tensors.to_cpu_nonblocking()
|
||||
)
|
||||
self.routed_experts_cpu: RoutedExpertsTensors | None = None
|
||||
if routed_experts is not None:
|
||||
self.routed_experts_cpu = routed_experts.to_cpu_nonblocking()
|
||||
self.prompt_logprobs_dict = {
|
||||
k: v.to_cpu_nonblocking() if v is not None else None
|
||||
for k, v in self.model_runner_output.prompt_logprobs_dict.items()
|
||||
}
|
||||
if self.pending_aux_output is not None:
|
||||
self.pending_aux_output.enqueue_cpu_copy(
|
||||
num_sampled=self.num_sampled_tokens_np,
|
||||
num_rejected=async_copy_to_np(sampler_output.num_rejected),
|
||||
)
|
||||
if check_ep_fault:
|
||||
has_fault = get_ep_all2all_manager().query_fault()
|
||||
self._has_fault = has_fault.to("cpu", non_blocking=True)
|
||||
@@ -192,8 +194,10 @@ class AsyncOutput(AsyncModelRunnerOutput):
|
||||
if self.logprobs_tensors is not None:
|
||||
self.model_runner_output.logprobs = self.logprobs_tensors.tolists()
|
||||
self.model_runner_output.prompt_logprobs_dict = self.prompt_logprobs_dict
|
||||
if self.routed_experts_cpu is not None:
|
||||
self.model_runner_output.routed_experts = self.routed_experts_cpu.tolists()
|
||||
if self.pending_aux_output is not None:
|
||||
self.model_runner_output.aux_output_connector_output = (
|
||||
self.pending_aux_output.process_output()
|
||||
)
|
||||
|
||||
if self._has_fault is not None and self._has_fault.item():
|
||||
mask = get_ep_all2all_manager().query_active_mask()
|
||||
|
||||
@@ -32,6 +32,10 @@ from vllm.compilation.counter import compilation_counter
|
||||
from vllm.compilation.cuda_graph import CUDAGraphStat
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.distributed.aux_output_connector.worker import (
|
||||
AuxOutputWorkerConnector,
|
||||
get_aux_output_connector,
|
||||
)
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_dcp_group,
|
||||
get_pp_group,
|
||||
@@ -39,10 +43,6 @@ from vllm.distributed.parallel_state import (
|
||||
from vllm.forward_context import BatchDescriptor, set_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import get_ep_all2all_manager
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
RoutedExpertsCapturer,
|
||||
bind_routed_experts_capturer,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
|
||||
initialize_mamba_ssu_backend,
|
||||
)
|
||||
@@ -73,7 +73,6 @@ from vllm.v1.outputs import (
|
||||
DraftTokenIds,
|
||||
ECConnectorOutput,
|
||||
ModelRunnerOutput,
|
||||
RoutedExpertsTensors,
|
||||
)
|
||||
from vllm.v1.watermarking import create_watermarker
|
||||
from vllm.v1.watermarking.gpu_sampler import GPUWatermarkSampler
|
||||
@@ -348,7 +347,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
# Expert parallelism load balancer.
|
||||
self.eplb = EPLBController(self.parallel_config, self.device)
|
||||
self.routed_experts_capturer: RoutedExpertsCapturer | None = None
|
||||
# The AuxOutput Connector owns R3 capture, copying, and storage.
|
||||
self.aux_output_connector: AuxOutputWorkerConnector | None = None
|
||||
|
||||
set_offloader(create_offloader(self.vllm_config.offload_config))
|
||||
|
||||
@@ -356,15 +356,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.max_model_len = max_model_len
|
||||
self.req_states.max_model_len = max_model_len
|
||||
|
||||
def init_routed_experts_capturer(self) -> None:
|
||||
"""Initialize target-model capture on every participating worker."""
|
||||
self.routed_experts_capturer = RoutedExpertsCapturer(
|
||||
max_num_batched_tokens=self.max_num_tokens,
|
||||
vllm_config=self.vllm_config,
|
||||
kv_cache_config=self.kv_cache_config,
|
||||
)
|
||||
bind_routed_experts_capturer(self.model, self.routed_experts_capturer)
|
||||
|
||||
def get_supported_tasks(self) -> tuple[SupportedTask, ...]:
|
||||
tasks: list[SupportedTask] = []
|
||||
if self.model_config.runner_type == "generate":
|
||||
@@ -756,6 +747,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
else:
|
||||
self.kv_connector = get_kv_connector(self.vllm_config, kv_caches_dict)
|
||||
|
||||
# AuxOutput connector requires resolved kv_cache_config.
|
||||
if self.vllm_config.aux_output_config.enabled:
|
||||
self.aux_output_connector = get_aux_output_connector(
|
||||
self.model, self.vllm_config, kv_cache_config
|
||||
)
|
||||
|
||||
def _init_kv_zero_meta(self) -> None:
|
||||
"""Build KV-block zeroing metadata; invoked from gpu_worker."""
|
||||
self.kv_block_zeroer = KVBlockZeroer(
|
||||
@@ -1648,6 +1645,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.add_requests(scheduler_output)
|
||||
self.update_requests(scheduler_output)
|
||||
self.block_tables.apply_staged_writes()
|
||||
if self.aux_output_connector is not None:
|
||||
# Register this step before the GPU forward.
|
||||
self.aux_output_connector.begin_step(
|
||||
scheduler_output.aux_output_connector_metadata
|
||||
)
|
||||
if scheduler_output.total_num_scheduled_tokens == 0:
|
||||
# No need to run the model.
|
||||
empty_output = self.kv_connector.no_forward(scheduler_output)
|
||||
@@ -1969,11 +1971,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
aux_hidden_states = None
|
||||
output_intermediate_tensors = model_output
|
||||
|
||||
routed_experts = None
|
||||
if not dummy_run and (capturer := self.routed_experts_capturer) is not None:
|
||||
assert slot_mappings is not None
|
||||
routed_experts = capturer.get_routed_experts(slot_mappings, num_toks)
|
||||
|
||||
finished_req_ids = scheduler_output.finished_req_ids
|
||||
self.execute_model_state = ExecuteModelState(
|
||||
input_batch=input_batch,
|
||||
@@ -1984,7 +1981,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
dp_sync=dp_sync,
|
||||
finished_req_ids=finished_req_ids,
|
||||
ec_connector_output=ec_connector_output,
|
||||
routed_experts=routed_experts,
|
||||
cudagraph_stats=cudagraph_stats,
|
||||
)
|
||||
|
||||
@@ -2014,7 +2010,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
dp_sync = self.execute_model_state.dp_sync
|
||||
finished_req_ids = self.execute_model_state.finished_req_ids
|
||||
ec_connector_output = self.execute_model_state.ec_connector_output
|
||||
routed_experts = self.execute_model_state.routed_experts
|
||||
cudagraph_stats = self.execute_model_state.cudagraph_stats
|
||||
self.execute_model_state = None
|
||||
|
||||
@@ -2083,6 +2078,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
prompt_logprobs_dict=prompt_logprobs_dict, # type: ignore[arg-type]
|
||||
cudagraph_stats=cudagraph_stats,
|
||||
)
|
||||
pending_aux_output = None
|
||||
if self.aux_output_connector is not None:
|
||||
pending_aux_output = self.aux_output_connector.prepare_output(input_batch)
|
||||
|
||||
# Start async output copy here so that it can overlap with speculator proposal.
|
||||
async_output = AsyncOutput(
|
||||
model_runner_output=model_runner_output,
|
||||
@@ -2091,7 +2090,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
main_stream=self.main_stream,
|
||||
copy_stream=self.output_copy_stream,
|
||||
check_ep_fault=self.check_ep_fault,
|
||||
routed_experts=routed_experts,
|
||||
pending_aux_output=pending_aux_output,
|
||||
)
|
||||
|
||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None
|
||||
@@ -2244,6 +2243,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
"""Release GPU tensors (model weights, KV caches, workspace) so that
|
||||
memory is reclaimable when running in the same process."""
|
||||
torch.accelerator.synchronize()
|
||||
if self.aux_output_connector is not None:
|
||||
self.aux_output_connector.close()
|
||||
self.cudagraph_manager = None
|
||||
self.fast_prefill = None
|
||||
if hasattr(self, "kv_caches"):
|
||||
@@ -2316,7 +2317,6 @@ class ExecuteModelState(NamedTuple):
|
||||
dp_sync: DPSyncState | None
|
||||
finished_req_ids: set[str]
|
||||
ec_connector_output: ECConnectorOutput | None
|
||||
routed_experts: RoutedExpertsTensors | None
|
||||
cudagraph_stats: CUDAGraphStat | None
|
||||
|
||||
|
||||
|
||||
@@ -617,8 +617,6 @@ def gather_sampler_output(
|
||||
device=device,
|
||||
)
|
||||
if local_output is not None:
|
||||
assert local_output.num_sampled is not None
|
||||
assert local_output.num_rejected is not None
|
||||
assert not gather_num_nans or local_output.num_nans is not None
|
||||
num_src_cols = min(
|
||||
local_output.sampled_token_ids.shape[1], max_num_logits_per_req
|
||||
|
||||
@@ -17,8 +17,8 @@ class SamplerOutput:
|
||||
sampled_token_ids: torch.Tensor
|
||||
logprobs_tensors: LogprobsTensors | None
|
||||
num_nans: torch.Tensor | None
|
||||
num_sampled: torch.Tensor | None
|
||||
num_rejected: torch.Tensor | None = None
|
||||
num_sampled: torch.Tensor
|
||||
num_rejected: torch.Tensor
|
||||
sampling_mask_tensors: SamplingMaskTensors | None = None
|
||||
|
||||
|
||||
|
||||
@@ -61,10 +61,6 @@ from vllm.lora.layers import BaseLayerWithLoRA, LoRAMapping, LoRAMappingType
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import get_ep_all2all_manager
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
RoutedExpertsCapturer,
|
||||
bind_routed_experts_capturer,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFuncsByType
|
||||
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
|
||||
initialize_mamba_ssu_backend,
|
||||
@@ -176,8 +172,6 @@ from vllm.v1.outputs import (
|
||||
LogprobsTensors,
|
||||
ModelRunnerOutput,
|
||||
PoolerOutput,
|
||||
RoutedExpertsLists,
|
||||
RoutedExpertsTensors,
|
||||
SamplerOutput,
|
||||
make_empty_encoder_model_runner_output,
|
||||
)
|
||||
@@ -293,7 +287,6 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput):
|
||||
invalid_req_indices: list[int],
|
||||
async_output_copy_stream: torch.cuda.Stream,
|
||||
vocab_size: int,
|
||||
routed_experts: RoutedExpertsTensors | None = None,
|
||||
check_ep_fault: bool = False,
|
||||
num_nans: torch.Tensor | None = None,
|
||||
):
|
||||
@@ -309,7 +302,6 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput):
|
||||
self._sampled_token_ids = sampled_token_ids
|
||||
self.vocab_size = vocab_size
|
||||
self._logprobs_tensors = logprobs_tensors
|
||||
self._routed_experts = routed_experts
|
||||
self._num_nans = num_nans
|
||||
self._has_fault: torch.Tensor | None = None
|
||||
|
||||
@@ -325,11 +317,6 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput):
|
||||
if self._logprobs_tensors
|
||||
else None
|
||||
)
|
||||
self._routed_experts_cpu = (
|
||||
self._routed_experts.to_cpu_nonblocking()
|
||||
if self._routed_experts is not None
|
||||
else None
|
||||
)
|
||||
self._num_nans_cpu = (
|
||||
self._num_nans.to("cpu", non_blocking=True)
|
||||
if self._num_nans is not None
|
||||
@@ -370,10 +357,6 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput):
|
||||
output.sampled_token_ids = valid_sampled_token_ids
|
||||
output.logprobs = logprobs_lists
|
||||
|
||||
if self._routed_experts_cpu is not None:
|
||||
output.routed_experts = self._routed_experts_cpu.tolists()
|
||||
del self._routed_experts
|
||||
|
||||
if self._num_nans_cpu is not None:
|
||||
output.num_nans_in_logits = nans_to_dict(
|
||||
self._num_nans_cpu.tolist(), output.req_id_to_index
|
||||
@@ -537,9 +520,6 @@ class GPUModelRunner(
|
||||
# These will be overridden in load_model()
|
||||
self.is_multimodal_pruning_enabled = False
|
||||
self.requires_sequential_video_encoding = False
|
||||
# Set to True after init_routed_experts_capturer() completes.
|
||||
# Prevents routed experts code from running during profiling/dummy run.
|
||||
self.routed_experts_initialized = False
|
||||
self.max_model_len = model_config.max_model_len
|
||||
|
||||
# Always set to false after the first forward pass
|
||||
@@ -2360,18 +2340,6 @@ class GPUModelRunner(
|
||||
block_table_gid_0 = _get_block_table(0)
|
||||
slot_mapping_gid_0 = slot_mappings[0]
|
||||
|
||||
if self.routed_experts_initialized:
|
||||
# Copy this step's attention slot_mapping into our private
|
||||
# device buffer. The shared ``slot_mappings[attn_gid]`` is
|
||||
# owned by the attention block table and will be overwritten
|
||||
# by the next ``_prepare_inputs``; we need a stable snapshot
|
||||
# because the async D2H may still be in flight on the copy
|
||||
# stream when the next step runs.
|
||||
slot_mapping_attn = slot_mappings[self.routed_experts_capturer.attn_gid]
|
||||
self.routed_experts_slot_mapping_device[:num_tokens].copy_(
|
||||
slot_mapping_attn[:num_tokens]
|
||||
)
|
||||
|
||||
num_computed_tokens_cpu = self.input_batch.num_computed_tokens_cpu_tensor[
|
||||
:num_reqs_padded
|
||||
]
|
||||
@@ -3745,20 +3713,6 @@ class GPUModelRunner(
|
||||
invalid_req_indices = []
|
||||
logprobs_lists = None
|
||||
if not self.use_async_scheduling:
|
||||
# Sync scheduling: issue routed experts D2H into the pinned
|
||||
# CPU buffer BEFORE ``_to_list`` below. ``_to_list`` does
|
||||
# ``event.synchronize()`` on the async copy stream which
|
||||
# waits for every D2H queued on the default stream since
|
||||
# the last sync, so this enqueue is naturally covered
|
||||
# without requiring its own synchronize.
|
||||
if self.routed_experts_initialized:
|
||||
buf = self.routed_experts_capturer.get_device_buffer()
|
||||
total = scheduler_output.total_num_scheduled_tokens
|
||||
self.routed_experts_cpu[:total].copy_(buf[:total], non_blocking=True)
|
||||
self.routed_experts_slot_mapping_cpu[:total].copy_(
|
||||
self.routed_experts_slot_mapping_device[:total],
|
||||
non_blocking=True,
|
||||
)
|
||||
with gpu_sync_allowed():
|
||||
# Get the valid generated tokens.
|
||||
max_gen_len = sampled_token_ids.shape[-1]
|
||||
@@ -4802,39 +4756,14 @@ class GPUModelRunner(
|
||||
else None,
|
||||
num_nans_in_logits=num_nans_in_logits,
|
||||
cudagraph_stats=cudagraph_stats,
|
||||
routed_experts=None,
|
||||
)
|
||||
|
||||
if not self.use_async_scheduling:
|
||||
if self.routed_experts_initialized:
|
||||
# Sync path: D2H was issued in ``_bookkeeping_sync`` and
|
||||
# synchronized by ``_to_list``'s event.synchronize(), so
|
||||
# the pinned buffers are ready to be wrapped as numpy.
|
||||
total = scheduler_output.total_num_scheduled_tokens
|
||||
output.routed_experts = RoutedExpertsLists(
|
||||
routing_data=self.routed_experts_cpu[:total].numpy(),
|
||||
slot_mapping=self.routed_experts_slot_mapping_cpu[:total].numpy(),
|
||||
)
|
||||
return output
|
||||
|
||||
with record_function_or_nullcontext(
|
||||
"gpu_model_runner: AsyncGPUModelRunnerOutput"
|
||||
):
|
||||
# Async path: produce a device-side snapshot that the async
|
||||
# copy stream can D2H later. Both tensors must be private
|
||||
# clones because:
|
||||
# - ``routing_data`` source is the shared capturer buffer,
|
||||
# which the next forward overwrites on the default stream.
|
||||
# - ``slot_mapping`` source is our own
|
||||
# ``routed_experts_slot_mapping_device``, which the
|
||||
# next ``_prepare_inputs`` overwrites on the default
|
||||
# stream while the D2H is still pending on the copy
|
||||
# stream.
|
||||
# Without clones, the copy stream would read torn data.
|
||||
routed_experts_snapshot = self.get_routed_experts(
|
||||
scheduler_output.total_num_scheduled_tokens
|
||||
)
|
||||
|
||||
async_output = AsyncGPUModelRunnerOutput(
|
||||
model_runner_output=output,
|
||||
sampled_token_ids=sampler_output.sampled_token_ids,
|
||||
@@ -4842,7 +4771,6 @@ class GPUModelRunner(
|
||||
invalid_req_indices=invalid_req_indices,
|
||||
async_output_copy_stream=self._get_or_create_async_output_copy_stream(),
|
||||
vocab_size=self.input_batch.vocab_size,
|
||||
routed_experts=routed_experts_snapshot,
|
||||
check_ep_fault=self.check_ep_fault,
|
||||
num_nans=num_nans_device,
|
||||
)
|
||||
@@ -7468,61 +7396,6 @@ class GPUModelRunner(
|
||||
kv_transfer_group.register_kv_caches(kv_caches)
|
||||
kv_transfer_group.set_host_xfer_buffer_ops(copy_kv_blocks)
|
||||
|
||||
def get_routed_experts(
|
||||
self,
|
||||
num_tokens: int,
|
||||
) -> RoutedExpertsTensors | None:
|
||||
if not self.routed_experts_initialized:
|
||||
return None
|
||||
|
||||
device_buffer = self.routed_experts_capturer.get_device_buffer()
|
||||
return RoutedExpertsTensors(
|
||||
routing_data=device_buffer[:num_tokens].clone(),
|
||||
slot_mapping=self.routed_experts_slot_mapping_device[:num_tokens].clone(),
|
||||
)
|
||||
|
||||
def init_routed_experts_capturer(self):
|
||||
logger.info(
|
||||
"Initializing routed experts capturer, enable_return_routed_experts: %s",
|
||||
self.model_config.enable_return_routed_experts,
|
||||
)
|
||||
self.routed_experts_capturer = RoutedExpertsCapturer(
|
||||
max_num_batched_tokens=self.scheduler_config.max_num_batched_tokens,
|
||||
vllm_config=self.vllm_config,
|
||||
kv_cache_config=self.kv_cache_config,
|
||||
)
|
||||
bind_routed_experts_capturer(self.model, self.routed_experts_capturer)
|
||||
|
||||
# Pinned CPU buffer for non-blocking D2H of ``routing_data`` on
|
||||
# the sync scheduling path. Shape / dtype mirror the device
|
||||
# capturer exactly so ``copy_`` is a straight memcpy.
|
||||
self.routed_experts_cpu = torch.empty(
|
||||
self.routed_experts_capturer.device_buffer.shape,
|
||||
dtype=self.routed_experts_capturer.device_buffer.dtype,
|
||||
device="cpu",
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
# ``slot_mapping`` dtype is fixed to int64 by
|
||||
# ``block_table.slot_mapping``; we mirror that here.
|
||||
max_tokens = self.scheduler_config.max_num_batched_tokens
|
||||
self.routed_experts_slot_mapping_cpu = torch.empty(
|
||||
(max_tokens,),
|
||||
dtype=torch.int64,
|
||||
device="cpu",
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
# Private device buffer so the shared ``block_table.slot_mapping``
|
||||
# can be overwritten by the next ``_prepare_inputs`` while the
|
||||
# D2H is still pending on the copy stream. Written in
|
||||
# ``_prepare_inputs``, read in ``_bookkeeping_sync`` (sync path)
|
||||
# or cloned into a snapshot (async path).
|
||||
self.routed_experts_slot_mapping_device = torch.empty(
|
||||
(max_tokens,),
|
||||
dtype=torch.int64,
|
||||
device=self.device,
|
||||
)
|
||||
self.routed_experts_initialized = True
|
||||
|
||||
def may_add_encoder_only_layers_to_kv_cache_config(self) -> None:
|
||||
"""Add encoder-only layers to the KV cache config."""
|
||||
block_size = self.vllm_config.cache_config.block_size
|
||||
|
||||
@@ -761,9 +761,6 @@ class Worker(WorkerBase):
|
||||
),
|
||||
)
|
||||
|
||||
if self.model_config.enable_return_routed_experts:
|
||||
self.model_runner.init_routed_experts_capturer()
|
||||
|
||||
# Build KV-zero metadata outside the CuMem pool so the bookkeeping
|
||||
# GPU tensors (seg_addrs, block-id buffers) use the standard PyTorch
|
||||
# allocator and are not discarded during sleep/wake cycles.
|
||||
|
||||
Reference in New Issue
Block a user