[AuxOutput] Add block-keyed storage for routed-expert outputs (#45635)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
aoshen02
OpenAI Codex
Roger Wang
Nick Hill
Codex
parent
0aee727ff6
commit
0b7f11a1ee
@@ -0,0 +1,119 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config import AuxOutputConfig, VllmConfig
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
def _config(
|
||||
*,
|
||||
use_v2: bool = True,
|
||||
pp: int = 1,
|
||||
dcp: int = 1,
|
||||
pcp: int = 1,
|
||||
connector: str | None = None,
|
||||
runner_type: str = "generate",
|
||||
is_moe: bool = True,
|
||||
sliding_window: int | None = None,
|
||||
attention_chunk_size: int | None = None,
|
||||
enable_prefix_caching: bool = True,
|
||||
adaptive_verification: bool = False,
|
||||
):
|
||||
return SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
runner_type=runner_type,
|
||||
is_moe=is_moe,
|
||||
get_sliding_window=lambda: sliding_window,
|
||||
attention_chunk_size=attention_chunk_size,
|
||||
),
|
||||
use_v2_model_runner=use_v2,
|
||||
parallel_config=SimpleNamespace(
|
||||
pipeline_parallel_size=pp,
|
||||
decode_context_parallel_size=dcp,
|
||||
prefill_context_parallel_size=pcp,
|
||||
),
|
||||
aux_output_config=AuxOutputConfig(enable_return_routed_experts=True),
|
||||
cache_config=SimpleNamespace(
|
||||
enable_prefix_caching=enable_prefix_caching,
|
||||
),
|
||||
speculative_config=(
|
||||
SimpleNamespace(enable_adaptive_verification=True)
|
||||
if adaptive_verification
|
||||
else None
|
||||
),
|
||||
kv_transfer_config=(
|
||||
None
|
||||
if connector is None
|
||||
else SimpleNamespace(
|
||||
is_kv_transfer_instance=True,
|
||||
kv_connector=connector,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_aux_output_config_defaults():
|
||||
config = AuxOutputConfig()
|
||||
|
||||
assert not config.enabled
|
||||
assert not config.enable_return_routed_experts
|
||||
assert config.max_bytes is None
|
||||
|
||||
|
||||
def test_aux_output_capture_changes_compilation_hash():
|
||||
disabled = AuxOutputConfig()
|
||||
enabled = AuxOutputConfig(enable_return_routed_experts=True)
|
||||
|
||||
assert disabled.compute_hash() != enabled.compute_hash()
|
||||
|
||||
|
||||
def test_legacy_routed_experts_flag_updates_aux_output_config():
|
||||
args = EngineArgs(enable_return_routed_experts=True)
|
||||
|
||||
assert args.aux_output_config.enabled
|
||||
assert args.aux_output_config.enable_return_routed_experts
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kwargs", "error"),
|
||||
[
|
||||
({"use_v2": False}, "requires Model Runner V2"),
|
||||
({"runner_type": "pooling"}, "only supports generate runners"),
|
||||
({"is_moe": False}, "only supports MoE models"),
|
||||
({"enable_prefix_caching": False}, "requires prefix caching"),
|
||||
(
|
||||
{"adaptive_verification": True},
|
||||
"adaptive speculative verification",
|
||||
),
|
||||
({"pp": 2}, "pipeline parallelism"),
|
||||
({"dcp": 2}, "context parallelism"),
|
||||
({"pcp": 2}, "context parallelism"),
|
||||
({"connector": "MooncakeConnector"}, "incompatible with KV connectors"),
|
||||
],
|
||||
)
|
||||
def test_aux_output_connector_rejects_unsupported_configuration(kwargs, error):
|
||||
with pytest.raises(ValueError, match=error):
|
||||
VllmConfig._verify_aux_output_compatibility(_config(**kwargs))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"kwargs",
|
||||
[{"sliding_window": 4096}, {"attention_chunk_size": 4096}],
|
||||
)
|
||||
def test_aux_output_config_defers_attention_layout_to_kv_config(kwargs):
|
||||
VllmConfig._verify_aux_output_compatibility(_config(**kwargs))
|
||||
|
||||
|
||||
def test_aux_output_guards_are_inactive_when_capture_is_disabled():
|
||||
config: Any = VllmConfig.__new__(VllmConfig)
|
||||
config.model_config = SimpleNamespace()
|
||||
config.aux_output_config = AuxOutputConfig()
|
||||
|
||||
config._verify_aux_output_compatibility()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -12,15 +12,13 @@ from ...utils import RemoteOpenAIServer
|
||||
MODEL_NAME = "TitanML/tiny-mixtral"
|
||||
|
||||
# tiny-mixtral config: 8 local experts, top-2 routing, 2 hidden layers.
|
||||
# The published config has sliding_window=4096, which produces
|
||||
# SlidingWindowSpec kv-cache groups; RoutedExpertsManager requires a
|
||||
# FullAttentionSpec group, so we override sliding_window=null below.
|
||||
# Its published sliding window is incompatible with AuxOutput retention.
|
||||
NUM_LOCAL_EXPERTS = 8
|
||||
NUM_EXPERTS_PER_TOK = 2
|
||||
NUM_HIDDEN_LAYERS = 2
|
||||
|
||||
|
||||
def assert_valid_routed_experts(encoded: str | None) -> None:
|
||||
def decode_routed_experts(encoded: str | None) -> np.ndarray:
|
||||
assert encoded is not None
|
||||
routed_experts = np.load(io.BytesIO(base64.b64decode(encoded)))
|
||||
assert routed_experts.ndim == 3
|
||||
@@ -30,6 +28,7 @@ def assert_valid_routed_experts(encoded: str | None) -> None:
|
||||
assert topk == NUM_EXPERTS_PER_TOK
|
||||
assert (routed_experts >= 0).all()
|
||||
assert (routed_experts < NUM_LOCAL_EXPERTS).all()
|
||||
return routed_experts
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
@@ -44,7 +43,11 @@ def server():
|
||||
"--hf-overrides",
|
||||
'{"sliding_window": null}',
|
||||
]
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME,
|
||||
args,
|
||||
env_dict={"VLLM_USE_V2_MODEL_RUNNER": "1"},
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@@ -63,4 +66,8 @@ async def test_routed_experts(server):
|
||||
choice = result.model_dump()["choices"][0]
|
||||
|
||||
assert choice["token_ids"] is not None
|
||||
assert_valid_routed_experts(choice["routed_experts"])
|
||||
assert result.usage is not None
|
||||
routed_experts = decode_routed_experts(choice["routed_experts"])
|
||||
assert len(routed_experts) == (
|
||||
result.usage.prompt_tokens + len(choice["token_ids"]) - 1
|
||||
)
|
||||
|
||||
@@ -15,9 +15,7 @@ MODEL_NAME = "TitanML/tiny-mixtral"
|
||||
GEN_ENDPOINT = "/inference/v1/generate"
|
||||
|
||||
# tiny-mixtral config: 8 local experts, top-2 routing, 2 hidden layers.
|
||||
# The published config has sliding_window=4096, which produces
|
||||
# SlidingWindowSpec kv-cache groups; RoutedExpertsManager requires a
|
||||
# FullAttentionSpec group, so we override sliding_window=null below.
|
||||
# Its published sliding window is incompatible with AuxOutput retention.
|
||||
NUM_LOCAL_EXPERTS = 8
|
||||
NUM_EXPERTS_PER_TOK = 2
|
||||
NUM_HIDDEN_LAYERS = 2
|
||||
@@ -36,7 +34,13 @@ def server():
|
||||
'{"sliding_window": null}',
|
||||
"--enable-scale-out",
|
||||
]
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME,
|
||||
args,
|
||||
env_dict={
|
||||
"VLLM_USE_V2_MODEL_RUNNER": "1",
|
||||
},
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@@ -56,9 +60,10 @@ async def client(server: RemoteOpenAIServer):
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_routed_experts(client):
|
||||
"""Test that /inference/v1/generate returns routed_experts when enabled."""
|
||||
prompt_token_ids = [1, 2, 3]
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": [1, 2, 3],
|
||||
"token_ids": prompt_token_ids,
|
||||
"sampling_params": {"max_tokens": 10, "temperature": 0.0},
|
||||
"stream": False,
|
||||
}
|
||||
@@ -80,3 +85,4 @@ async def test_generate_routed_experts(client):
|
||||
assert topk == NUM_EXPERTS_PER_TOK
|
||||
assert (routed_experts >= 0).all()
|
||||
assert (routed_experts < NUM_LOCAL_EXPERTS).all()
|
||||
assert num_tokens == len(prompt_token_ids) + len(choice["token_ids"]) - 1
|
||||
|
||||
@@ -7,26 +7,19 @@ from unittest.mock import Mock, patch
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CompilationMode
|
||||
from vllm.distributed.eplb.eplb_state import EplbLayerState
|
||||
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
RoutedExpertsCapturer,
|
||||
RoutedExpertsManager,
|
||||
bind_routed_experts_capturer,
|
||||
get_routed_experts_attn_gid,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.base_router import BaseRouter
|
||||
from vllm.transformers_utils.model_arch_config_convertor import (
|
||||
ModelArchConfigConvertorBase,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowMLASpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
|
||||
@@ -42,6 +35,7 @@ def _capturer_with_buffer(
|
||||
num_experts_per_tok: int = 2,
|
||||
dp_rank: int = 0,
|
||||
tp_size: int = 1,
|
||||
dtype: torch.dtype = torch.int32,
|
||||
) -> RoutedExpertsCapturer:
|
||||
# Bypass __init__ so the test can use a CPU buffer and skip the
|
||||
# VllmConfig dependency. The CUDA device-tensor allocation in the
|
||||
@@ -49,10 +43,11 @@ def _capturer_with_buffer(
|
||||
c = RoutedExpertsCapturer.__new__(RoutedExpertsCapturer)
|
||||
c.dp_rank = dp_rank
|
||||
c.tp_size = tp_size
|
||||
c.output_dtype = torch.uint8
|
||||
c.device_buffer = torch.full(
|
||||
(max_tokens, num_layers, num_experts_per_tok),
|
||||
-1,
|
||||
dtype=torch.int32,
|
||||
dtype=dtype,
|
||||
)
|
||||
return c
|
||||
|
||||
@@ -88,74 +83,36 @@ def _make_modular_routed_experts():
|
||||
)
|
||||
|
||||
|
||||
def _make_model_config(hf_config):
|
||||
num_experts_per_token = ModelArchConfigConvertorBase(
|
||||
hf_config, hf_config
|
||||
).get_num_experts_per_token()
|
||||
model_config = SimpleNamespace(
|
||||
hf_text_config=hf_config,
|
||||
model_arch_config=SimpleNamespace(
|
||||
num_experts_per_token=num_experts_per_token,
|
||||
def _full_attention_kv_group(
|
||||
spec_type: type[FullAttentionSpec] = FullAttentionSpec,
|
||||
) -> KVCacheGroupSpec:
|
||||
full_attention = spec_type(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
return KVCacheGroupSpec(
|
||||
["layer"],
|
||||
UniformTypeKVCacheSpecs(
|
||||
block_size=16,
|
||||
kv_cache_specs={"layer": full_attention},
|
||||
),
|
||||
)
|
||||
model_config.get_num_experts = lambda: hf_config.num_experts
|
||||
model_config.get_num_experts_per_tok = lambda: (
|
||||
ModelConfig.get_num_experts_per_tok(model_config)
|
||||
)
|
||||
model_config.get_total_num_hidden_layers = lambda: hf_config.num_hidden_layers
|
||||
return model_config
|
||||
|
||||
|
||||
def test_routed_experts_manager_uses_gemma4_top_k_experts():
|
||||
hf_config = SimpleNamespace(
|
||||
num_experts=8,
|
||||
top_k_experts=2,
|
||||
num_hidden_layers=3,
|
||||
)
|
||||
vllm_config = SimpleNamespace(model_config=_make_model_config(hf_config))
|
||||
kv_cache_spec = FullAttentionSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=2,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[KVCacheGroupSpec(["layer"], kv_cache_spec)],
|
||||
)
|
||||
@pytest.mark.parametrize("eplb_enabled", [False, True])
|
||||
def test_base_router_capture_pre_eplb_mapping(eplb_enabled):
|
||||
eplb_state = None
|
||||
if eplb_enabled:
|
||||
eplb_state = EplbLayerState()
|
||||
eplb_state.expert_load_view = torch.zeros(32, dtype=torch.int64)
|
||||
eplb_state.logical_to_physical_map = torch.arange(32).view(32, 1)
|
||||
eplb_state.logical_replica_count = torch.ones(32, dtype=torch.int64)
|
||||
eplb_state.should_record_tensor = torch.ones((), dtype=torch.bool)
|
||||
eplb_state.num_unpadded_tokens_tensors = [torch.tensor(0, dtype=torch.int32)]
|
||||
router = _make_router(eplb_state)
|
||||
|
||||
manager = RoutedExpertsManager(vllm_config, kv_cache_config)
|
||||
|
||||
assert manager.routed_experts_by_slot.shape == (8, 3, 2)
|
||||
|
||||
|
||||
def test_routed_experts_manager_uses_kimi_k3_experts_per_token():
|
||||
hf_config = SimpleNamespace(
|
||||
num_experts=8,
|
||||
num_experts_per_token=2,
|
||||
num_hidden_layers=3,
|
||||
)
|
||||
vllm_config = SimpleNamespace(model_config=_make_model_config(hf_config))
|
||||
kv_cache_spec = FullAttentionSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=2,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[KVCacheGroupSpec(["layer"], kv_cache_spec)],
|
||||
)
|
||||
|
||||
manager = RoutedExpertsManager(vllm_config, kv_cache_config)
|
||||
|
||||
assert manager.routed_experts_by_slot.shape == (8, 3, 2)
|
||||
|
||||
|
||||
def test_base_router_capture_pre_eplb_mapping():
|
||||
router = _make_router()
|
||||
captured = []
|
||||
|
||||
def capture_fn(ids):
|
||||
@@ -173,34 +130,7 @@ def test_base_router_capture_pre_eplb_mapping():
|
||||
assert torch.equal(topk_ids, torch.tensor([[11, 12], [13, 14]]))
|
||||
|
||||
|
||||
def test_base_router_capture_with_eplb_enabled():
|
||||
eplb_state = EplbLayerState()
|
||||
eplb_state.expert_load_view = torch.zeros(32, dtype=torch.int64)
|
||||
eplb_state.logical_to_physical_map = torch.arange(32).view(32, 1)
|
||||
eplb_state.logical_replica_count = torch.ones(32, dtype=torch.int64)
|
||||
eplb_state.should_record_tensor = torch.ones((), dtype=torch.bool)
|
||||
eplb_state.num_unpadded_tokens_tensors = [torch.tensor(0, dtype=torch.int32)]
|
||||
router = _make_router(eplb_state=eplb_state)
|
||||
|
||||
captured = []
|
||||
|
||||
def capture_fn(ids):
|
||||
captured.append(ids.clone())
|
||||
|
||||
router.set_capture_fn(capture_fn)
|
||||
_, topk_ids = router.select_experts(
|
||||
hidden_states=torch.empty(1),
|
||||
router_logits=torch.empty(1),
|
||||
)
|
||||
|
||||
assert len(captured) == 1
|
||||
# Capture should see logical ids pre-EPLB mapping.
|
||||
assert torch.equal(captured[0], torch.tensor([[1, 2], [3, 4]]))
|
||||
# Our DummyRouter mapping adds +10.
|
||||
assert torch.equal(topk_ids, torch.tensor([[11, 12], [13, 14]]))
|
||||
|
||||
|
||||
def test_public_binding_only_visits_target_model(monkeypatch):
|
||||
def test_public_binding_binds_target_model_router(monkeypatch):
|
||||
class DummyFusedMoE:
|
||||
def __init__(self, layer_id):
|
||||
self.layer_id = layer_id
|
||||
@@ -208,7 +138,6 @@ def test_public_binding_only_visits_target_model(monkeypatch):
|
||||
self._quant_method = _make_modular_routed_experts().quant_method
|
||||
|
||||
target_module = DummyFusedMoE(layer_id=7)
|
||||
draft_module = DummyFusedMoE(layer_id=0)
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.layer as fused_moe_layer
|
||||
|
||||
@@ -221,12 +150,30 @@ def test_public_binding_only_visits_target_model(monkeypatch):
|
||||
)
|
||||
|
||||
assert target_module.router.capture_fn is not None
|
||||
assert draft_module.router.capture_fn is None
|
||||
topk_ids = torch.tensor([[5, 6]])
|
||||
target_module.router.capture_fn(topk_ids)
|
||||
assert calls == [(7, topk_ids)]
|
||||
|
||||
|
||||
def test_public_binding_supports_direct_capture_source():
|
||||
class DirectCaptureSource(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.layer_id = 3
|
||||
self.capture_fn = None
|
||||
|
||||
source = DirectCaptureSource()
|
||||
calls = []
|
||||
capturer = types.SimpleNamespace(capture=lambda *args: calls.append(args))
|
||||
|
||||
bind_routed_experts_capturer(source, capturer)
|
||||
|
||||
assert source.capture_fn is not None
|
||||
topk_ids = torch.tensor([[1, 2]])
|
||||
source.capture_fn(topk_ids)
|
||||
assert calls == [(3, topk_ids)]
|
||||
|
||||
|
||||
def test_public_binding_rejects_monolithic_without_replay_support(monkeypatch):
|
||||
class DummyFusedMoE:
|
||||
def __init__(self):
|
||||
@@ -277,6 +224,51 @@ def test_routed_experts_capturer_single_dp_no_metadata():
|
||||
assert capturer.device_buffer[3, 0, 0].item() == -1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_experts", "dtype_name", "torch_dtype"),
|
||||
[(256, "uint8", torch.uint8), (257, "uint16", torch.uint16)],
|
||||
)
|
||||
def test_routed_experts_capturer_exposes_output_profile(
|
||||
monkeypatch, num_experts, dtype_name, torch_dtype
|
||||
):
|
||||
import vllm.model_executor.layers.fused_moe.routed_experts_capturer as module
|
||||
|
||||
monkeypatch.setattr(module, "current_platform", SimpleNamespace(device_type="cpu"))
|
||||
config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
get_total_num_hidden_layers=lambda: 3,
|
||||
get_num_experts=lambda: num_experts,
|
||||
get_num_experts_per_tok=lambda: 2,
|
||||
hf_text_config=SimpleNamespace(model_type="test"),
|
||||
),
|
||||
parallel_config=SimpleNamespace(data_parallel_rank=0, tensor_parallel_size=1),
|
||||
)
|
||||
|
||||
capturer = RoutedExpertsCapturer(8, config)
|
||||
|
||||
assert capturer.shape_per_token == (3, 2)
|
||||
assert capturer.output_dtype_name == dtype_name
|
||||
assert capturer.output_dtype == torch_dtype
|
||||
assert capturer.device_buffer.shape == (8, 3, 2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("output_dtype", [torch.uint8, torch.uint16])
|
||||
def test_routed_experts_capturer_narrows_snapshot(output_dtype):
|
||||
capturer = _capturer_with_buffer(dtype=torch.int32)
|
||||
capturer.output_dtype = output_dtype
|
||||
topk = torch.tensor([[1, 2], [254, 255]], dtype=torch.int64)
|
||||
ctx = SimpleNamespace(dp_metadata=None)
|
||||
with patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx):
|
||||
capturer.capture(layer_id=0, topk_ids=topk)
|
||||
|
||||
output = capturer.snapshot_routing_data(2)
|
||||
assert capturer.device_buffer.dtype == torch.int32
|
||||
assert capturer.device_buffer[:2, 0, :].tolist() == topk.tolist()
|
||||
assert output.dtype == output_dtype
|
||||
assert output[:, 0, :].tolist() == topk.tolist()
|
||||
assert output.data_ptr() != capturer.device_buffer.data_ptr()
|
||||
|
||||
|
||||
def test_routed_experts_capturer_dp_naive_concatenated_all_ranks():
|
||||
"""N == sum(num_tokens_dp): slice this rank's segment from concatenated topk."""
|
||||
capturer = _capturer_with_buffer(dp_rank=1)
|
||||
@@ -307,6 +299,47 @@ def test_routed_experts_capturer_dp_modular_local_tokens():
|
||||
assert torch.equal(capturer.device_buffer[:3, 0, :], topk)
|
||||
|
||||
|
||||
def test_routed_experts_capturer_dp_ep_gathered_shards():
|
||||
capturer = _capturer_with_buffer(dp_rank=1, tp_size=2)
|
||||
num_tokens_dp = torch.tensor([3, 2], dtype=torch.int32)
|
||||
ctx = SimpleNamespace(
|
||||
dp_metadata=SimpleNamespace(
|
||||
num_tokens_across_dp_cpu=num_tokens_dp,
|
||||
local_sizes=[2, 2, 1, 1],
|
||||
)
|
||||
)
|
||||
topk = torch.tensor(
|
||||
[[0, 1], [2, 3], [4, 5], [-1, -1], [10, 11], [12, 13]],
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
with patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx):
|
||||
capturer.capture(layer_id=0, topk_ids=topk)
|
||||
|
||||
assert torch.equal(capturer.device_buffer[:2, 0], topk[4:])
|
||||
|
||||
|
||||
def test_routed_experts_capturer_sp_modular_gathers_tp_shards():
|
||||
capturer = _capturer_with_buffer(dp_rank=1, tp_size=2)
|
||||
num_tokens_dp = torch.tensor([2, 3], dtype=torch.int32)
|
||||
ctx = SimpleNamespace(
|
||||
dp_metadata=SimpleNamespace(num_tokens_across_dp_cpu=num_tokens_dp)
|
||||
)
|
||||
local_shard = torch.tensor([[10, 11], [12, 13]], dtype=torch.int32)
|
||||
gathered = torch.tensor([[10, 11], [12, 13], [14, 15], [-1, -1]], dtype=torch.int32)
|
||||
tp_group = Mock()
|
||||
tp_group.all_gather.return_value = gathered
|
||||
|
||||
with (
|
||||
patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx),
|
||||
patch(f"{_REC_MODULE}.get_tp_group", return_value=tp_group),
|
||||
):
|
||||
capturer.capture(layer_id=0, topk_ids=local_shard)
|
||||
|
||||
tp_group.all_gather.assert_called_once_with(local_shard, dim=0)
|
||||
assert torch.equal(capturer.device_buffer[:3, 0], gathered[:3])
|
||||
|
||||
|
||||
def test_routed_experts_capturer_dp_unexpected_batch_raises():
|
||||
"""Mismatch between topk batch dim and DP layout: fail fast."""
|
||||
capturer = _capturer_with_buffer(dp_rank=0)
|
||||
@@ -324,128 +357,119 @@ def test_routed_experts_capturer_dp_unexpected_batch_raises():
|
||||
assert capturer.device_buffer[0, 0, 0].item() == -1
|
||||
|
||||
|
||||
def test_routed_experts_attention_group_is_shared_and_fail_closed():
|
||||
"""Both sides key routing data by this gid, so it must skip non-full-attention
|
||||
groups rather than defaulting to 0, and fail closed when none exists."""
|
||||
common = dict(num_kv_heads=1, head_size=1, dtype=torch.float32)
|
||||
def test_get_aux_output_connector_passes_config(monkeypatch):
|
||||
import vllm.distributed.aux_output_connector.worker as aux_output_worker
|
||||
|
||||
connector = Mock()
|
||||
constructor = Mock(return_value=connector)
|
||||
monkeypatch.setattr(aux_output_worker, "AuxOutputWorkerConnector", constructor)
|
||||
config = SimpleNamespace(
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["swa_layer"],
|
||||
SlidingWindowMLASpec(block_size=4, sliding_window=8, **common),
|
||||
),
|
||||
KVCacheGroupSpec(["full_layer"], FullAttentionSpec(block_size=4, **common)),
|
||||
]
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=32)
|
||||
)
|
||||
assert get_routed_experts_attn_gid(config) == 1
|
||||
model = Mock()
|
||||
kv_cache_config = Mock()
|
||||
|
||||
with pytest.raises(ValueError, match="requires a full-attention KV cache group"):
|
||||
get_routed_experts_attn_gid(SimpleNamespace(kv_cache_groups=[]))
|
||||
result = aux_output_worker.get_aux_output_connector(model, config, kv_cache_config)
|
||||
|
||||
|
||||
def test_routed_experts_attention_group_unwraps_uniform_type_specs():
|
||||
"""DeepSeek-V4-shaped groups wrap their specs in ``UniformTypeKVCacheSpecs``.
|
||||
|
||||
The wrapper is not a ``FullAttentionSpec``, so a bare isinstance check finds
|
||||
no group and fails closed on every worker. Unwrap semantics themselves are
|
||||
covered by ``test_is_full_attention_spec_*`` in tests/v1/core.
|
||||
"""
|
||||
common = dict(num_kv_heads=1, head_size=1, dtype=torch.float32)
|
||||
swa_spec = SlidingWindowMLASpec(block_size=4, sliding_window=8, **common)
|
||||
mla_spec = MLAAttentionSpec(block_size=4, **common)
|
||||
config = SimpleNamespace(
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["swa_layer"],
|
||||
UniformTypeKVCacheSpecs(
|
||||
block_size=4, kv_cache_specs={"swa_layer": swa_spec}
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mla_layer"],
|
||||
UniformTypeKVCacheSpecs(
|
||||
block_size=4, kv_cache_specs={"mla_layer": mla_spec}
|
||||
),
|
||||
),
|
||||
]
|
||||
constructor.assert_called_once_with(
|
||||
model=model,
|
||||
kv_cache_config=kv_cache_config,
|
||||
vllm_config=config,
|
||||
)
|
||||
|
||||
assert get_routed_experts_attn_gid(config) == 1
|
||||
|
||||
swa_only = SimpleNamespace(kv_cache_groups=[config.kv_cache_groups[0]])
|
||||
with pytest.raises(ValueError, match="requires a full-attention KV cache group"):
|
||||
get_routed_experts_attn_gid(swa_only)
|
||||
assert result is connector
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
def test_mrv2_async_output_returns_existing_routed_experts_field():
|
||||
from vllm.v1.outputs import ModelRunnerOutput, RoutedExpertsTensors
|
||||
from vllm.v1.worker.gpu.async_utils import AsyncOutput
|
||||
from vllm.v1.worker.gpu.sample.output import SamplerOutput
|
||||
|
||||
routed_experts = RoutedExpertsTensors(
|
||||
routing_data=torch.arange(6, dtype=torch.int32, device="cuda").reshape(3, 1, 2),
|
||||
slot_mapping=torch.tensor([11, 12, 13], device="cuda"),
|
||||
)
|
||||
num_sampled = torch.tensor([1], dtype=torch.int32, device="cuda")
|
||||
sampler_output = SamplerOutput(
|
||||
sampled_token_ids=torch.tensor([[1]], device="cuda"),
|
||||
logprobs_tensors=None,
|
||||
num_nans=None,
|
||||
num_sampled=num_sampled,
|
||||
num_rejected=torch.tensor([0], dtype=torch.int32, device="cuda"),
|
||||
)
|
||||
output = AsyncOutput(
|
||||
model_runner_output=ModelRunnerOutput(req_ids=["req"], req_id_to_index={}),
|
||||
sampler_output=sampler_output,
|
||||
num_sampled_tokens=num_sampled,
|
||||
main_stream=torch.cuda.current_stream(),
|
||||
copy_stream=torch.cuda.Stream(),
|
||||
routed_experts=routed_experts,
|
||||
).get_output()
|
||||
|
||||
assert output.routed_experts is not None
|
||||
assert output.routed_experts.routing_data[:, 0, 0].tolist() == [0, 2, 4]
|
||||
assert output.routed_experts.slot_mapping.tolist() == [11, 12, 13]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rank", [0, 1])
|
||||
def test_all_tp_ranks_initialize_capture(monkeypatch, rank):
|
||||
pytest.importorskip("vllm.vllm_flash_attn", exc_type=ImportError)
|
||||
import vllm.v1.worker.gpu.model_runner as model_runner
|
||||
def test_aux_output_worker_connector_binds_capture_on_non_output_rank(monkeypatch):
|
||||
import vllm.distributed.aux_output_connector.worker as aux_output_worker
|
||||
|
||||
capturer = Mock()
|
||||
constructor = Mock(return_value=capturer)
|
||||
bind = Mock()
|
||||
monkeypatch.setattr(model_runner, "RoutedExpertsCapturer", constructor)
|
||||
monkeypatch.setattr(model_runner, "bind_routed_experts_capturer", bind)
|
||||
monkeypatch.setattr(aux_output_worker, "RoutedExpertsCapturer", constructor)
|
||||
monkeypatch.setattr(aux_output_worker, "bind_routed_experts_capturer", bind)
|
||||
monkeypatch.setattr(
|
||||
aux_output_worker,
|
||||
"get_tp_group",
|
||||
lambda: SimpleNamespace(is_first_rank=False, world_size=1),
|
||||
)
|
||||
|
||||
runner = model_runner.GPUModelRunner.__new__(model_runner.GPUModelRunner)
|
||||
runner.max_num_tokens = 32
|
||||
runner.vllm_config = SimpleNamespace(parallel_config=SimpleNamespace(rank=rank))
|
||||
runner.kv_cache_config = SimpleNamespace()
|
||||
runner.model = Mock()
|
||||
|
||||
runner.init_routed_experts_capturer()
|
||||
config = SimpleNamespace(
|
||||
aux_output_config=SimpleNamespace(enable_return_routed_experts=True),
|
||||
kv_transfer_config=None,
|
||||
max_concurrent_batches=2,
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=32),
|
||||
)
|
||||
model = Mock()
|
||||
connector = aux_output_worker.AuxOutputWorkerConnector(
|
||||
vllm_config=config,
|
||||
model=model,
|
||||
kv_cache_config=SimpleNamespace(kv_cache_groups=[_full_attention_kv_group()]),
|
||||
)
|
||||
|
||||
constructor.assert_called_once_with(
|
||||
max_num_batched_tokens=32,
|
||||
vllm_config=runner.vllm_config,
|
||||
kv_cache_config=runner.kv_cache_config,
|
||||
vllm_config=config,
|
||||
)
|
||||
bind.assert_called_once_with(runner.model, capturer)
|
||||
assert runner.routed_experts_capturer is capturer
|
||||
bind.assert_called_once_with(model, capturer)
|
||||
connector.begin_step(Mock())
|
||||
assert connector.prepare_output(Mock()) is None
|
||||
capturer.snapshot_routing_data.assert_not_called()
|
||||
|
||||
|
||||
def test_aux_output_worker_connector_default_capacity(monkeypatch):
|
||||
import vllm.distributed.aux_output_connector.worker as aux_output_worker
|
||||
|
||||
tp_group = SimpleNamespace(is_first_rank=True, world_size=1)
|
||||
store_constructor = Mock()
|
||||
background_store_constructor = Mock(side_effect=lambda store, **_: store)
|
||||
capturer = SimpleNamespace(shape_per_token=(2,), output_dtype_name="int32")
|
||||
monkeypatch.setattr(aux_output_worker, "get_tp_group", lambda: tp_group)
|
||||
monkeypatch.setattr(
|
||||
aux_output_worker, "RoutedExpertsCapturer", Mock(return_value=capturer)
|
||||
)
|
||||
monkeypatch.setattr(aux_output_worker, "bind_routed_experts_capturer", Mock())
|
||||
monkeypatch.setattr(
|
||||
aux_output_worker,
|
||||
"resolve_kv_cache_block_sizes",
|
||||
lambda *_: (32, 16),
|
||||
)
|
||||
monkeypatch.setattr(aux_output_worker, "BlockObjectStore", store_constructor)
|
||||
monkeypatch.setattr(
|
||||
aux_output_worker, "BackgroundBlockObjectStore", background_store_constructor
|
||||
)
|
||||
monkeypatch.setattr(aux_output_worker, "RoutedExpertsBuffer", Mock())
|
||||
|
||||
config = SimpleNamespace(
|
||||
aux_output_config=SimpleNamespace(max_bytes=None),
|
||||
kv_transfer_config=None,
|
||||
cache_config=SimpleNamespace(enable_prefix_caching=True),
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=8, max_num_batched_tokens=32),
|
||||
max_concurrent_batches=2,
|
||||
)
|
||||
kwargs = dict(
|
||||
vllm_config=config,
|
||||
model=Mock(),
|
||||
kv_cache_config=SimpleNamespace(
|
||||
num_blocks=10,
|
||||
kv_cache_groups=[_full_attention_kv_group(MLAAttentionSpec)],
|
||||
),
|
||||
)
|
||||
|
||||
aux_output_worker.AuxOutputWorkerConnector(**kwargs)
|
||||
assert store_constructor.call_args.kwargs["max_bytes"] == 2560
|
||||
assert store_constructor.call_args.kwargs["object_nbytes"] == 128
|
||||
assert background_store_constructor.call_args.kwargs["max_pending_batches"] == 16
|
||||
|
||||
|
||||
def test_v2_model_runner_accepts_routed_experts(monkeypatch):
|
||||
monkeypatch.setattr("importlib.metadata.entry_points", lambda **_: ())
|
||||
config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
enable_return_routed_experts=True,
|
||||
use_mla=False,
|
||||
logits_processors=None,
|
||||
enable_prompt_embeds=False,
|
||||
),
|
||||
aux_output_config=SimpleNamespace(enable_return_routed_experts=True),
|
||||
speculative_config=None,
|
||||
parallel_config=SimpleNamespace(
|
||||
prefill_context_parallel_size=1,
|
||||
|
||||
@@ -354,8 +354,7 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
|
||||
scheduler.finished_req_ids_dict = None
|
||||
scheduler.grammar_compile_error_reqs = set()
|
||||
scheduler.vllm_config = Mock()
|
||||
scheduler.vllm_config.model_config.enable_return_routed_experts = False
|
||||
scheduler.enable_return_routed_experts = False
|
||||
scheduler.aux_output_connector = None
|
||||
scheduler.return_sampling_mask = False
|
||||
scheduler.recompute_kv_load_failures = False
|
||||
scheduler.defer_block_free = False
|
||||
|
||||
@@ -19,6 +19,10 @@ from vllm.config import (
|
||||
SpeculativeConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.distributed.aux_output_connector.connector import (
|
||||
AuxOutputSchedulerConnector,
|
||||
AuxRequestOutput,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.metrics import ECConnectorStats
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import (
|
||||
HiSparseConnector,
|
||||
@@ -55,7 +59,6 @@ from vllm.v1.outputs import (
|
||||
ECConnectorOutput,
|
||||
KVConnectorOutput,
|
||||
ModelRunnerOutput,
|
||||
RoutedExpertsLists,
|
||||
SamplingMaskLists,
|
||||
make_empty_encoder_model_runner_output,
|
||||
)
|
||||
@@ -136,21 +139,18 @@ def test_add_requests():
|
||||
assert len(scheduler.waiting) == i + 1
|
||||
|
||||
|
||||
def test_routed_experts_prompt_start_at_prompt_end():
|
||||
@pytest.mark.parametrize("max_tokens", [1, 2])
|
||||
def test_routed_experts_prompt_start_at_prompt_end(max_tokens):
|
||||
"""A prompt-end offset should return empty routing data, not crash."""
|
||||
scheduler = create_scheduler()
|
||||
(request,) = create_requests(num_requests=1)
|
||||
(request,) = create_requests(num_requests=1, max_tokens=max_tokens)
|
||||
request.sampling_params.routed_experts_prompt_start = request.num_prompt_tokens
|
||||
scheduler.aux_output_connector = AuxOutputSchedulerConnector()
|
||||
scheduler.add_request(request)
|
||||
scheduler_output = scheduler.schedule()
|
||||
num_tokens = scheduler_output.num_scheduled_tokens[request.request_id]
|
||||
|
||||
manager = Mock()
|
||||
manager.routed_experts_by_slot = np.empty((0, 1, 1), dtype=np.uint8)
|
||||
manager.get.return_value = np.empty((0, 1, 1), dtype=np.uint8)
|
||||
scheduler.enable_return_routed_experts = True
|
||||
scheduler.routed_experts_mgr = manager
|
||||
scheduler._re_block_ids = {request.request_id: []}
|
||||
assert scheduler_output.aux_output_connector_metadata.requests == {
|
||||
request.request_id: request.num_prompt_tokens
|
||||
}
|
||||
|
||||
outputs = scheduler.update_from_output(
|
||||
scheduler_output,
|
||||
@@ -161,21 +161,23 @@ def test_routed_experts_prompt_start_at_prompt_end():
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
routed_experts=RoutedExpertsLists(
|
||||
routing_data=np.zeros((num_tokens, 1, 1), dtype=np.uint8),
|
||||
slot_mapping=np.arange(num_tokens),
|
||||
),
|
||||
aux_output_connector_output={
|
||||
request.request_id: AuxRequestOutput(
|
||||
request.num_prompt_tokens,
|
||||
np.empty((0, 1, 1), dtype=np.uint8),
|
||||
)
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
manager.get.assert_called_once_with(
|
||||
[], request.num_prompt_tokens, token_start=request.num_prompt_tokens
|
||||
)
|
||||
assert outputs[request.client_index].outputs[0].routed_experts.shape == (0, 1, 1)
|
||||
output = outputs[request.client_index].outputs[0]
|
||||
assert output.new_token_ids == [0]
|
||||
assert output.routed_experts.shape == (0, 1, 1)
|
||||
|
||||
|
||||
def test_finish_request():
|
||||
scheduler = create_scheduler()
|
||||
scheduler.aux_output_connector = Mock()
|
||||
requests = create_requests(num_requests=10)
|
||||
for request in requests:
|
||||
scheduler.add_request(request)
|
||||
@@ -184,6 +186,7 @@ def test_finish_request():
|
||||
scheduler.finish_requests(request.request_id, RequestStatus.FINISHED_ABORTED)
|
||||
assert request.request_id not in scheduler.requests
|
||||
assert len(scheduler.waiting) == 9 - i
|
||||
scheduler.aux_output_connector.request_finished.assert_called_with(request)
|
||||
|
||||
|
||||
def test_get_num_unfinished_requests():
|
||||
@@ -1373,6 +1376,31 @@ def test_preemption_re_records_prefix_cache_query():
|
||||
assert stats.preempted_requests == 1
|
||||
|
||||
|
||||
def test_preemption_processes_stale_aux_output():
|
||||
scheduler = create_scheduler(enable_prefix_caching=True)
|
||||
request = create_requests(num_requests=1)[0]
|
||||
scheduler.add_request(request)
|
||||
scheduler_output = scheduler.schedule()
|
||||
scheduler.running.remove(request)
|
||||
scheduler.aux_output_connector = Mock()
|
||||
scheduler._preempt_request(request, 0.0)
|
||||
scheduler.aux_output_connector.request_finished.assert_called_once_with(request)
|
||||
|
||||
scheduler.update_from_output(
|
||||
scheduler_output,
|
||||
ModelRunnerOutput(
|
||||
req_ids=[request.request_id],
|
||||
req_id_to_index={request.request_id: 0},
|
||||
sampled_token_ids=[[1000]],
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
),
|
||||
)
|
||||
|
||||
scheduler.aux_output_connector.take_output.assert_called_once_with(request, None)
|
||||
|
||||
|
||||
def test_prefix_cache_stats_not_recorded_when_caching_disabled():
|
||||
"""With prefix caching off there is no local lookup, so admitting a request
|
||||
records no phantom miss."""
|
||||
@@ -1461,6 +1489,7 @@ def test_prefix_cache_stats_counted_once_for_retried_then_scheduled_request():
|
||||
|
||||
def test_scheduler_reset_prefix_cache():
|
||||
scheduler = create_scheduler(enable_prefix_caching=True)
|
||||
scheduler.aux_output_connector = Mock()
|
||||
requests = create_requests(num_requests=10)
|
||||
for request in requests:
|
||||
scheduler.add_request(request)
|
||||
@@ -1477,10 +1506,20 @@ def test_scheduler_reset_prefix_cache():
|
||||
# Reset prefix cache should fail since there are still running requests
|
||||
# and they are taking KV cache
|
||||
assert not scheduler.reset_prefix_cache()
|
||||
scheduler.aux_output_connector.reset.assert_not_called()
|
||||
|
||||
with pytest.raises(RuntimeError, match=r"pause\(mode='keep'\)"):
|
||||
scheduler.reset_prefix_cache(reset_running_requests=True)
|
||||
|
||||
# pause(mode="keep") also waits for scheduled model outputs to drain.
|
||||
scheduler.set_pause_state(PauseState.PAUSED_ALL)
|
||||
with pytest.raises(RuntimeError, match="model output is in flight"):
|
||||
scheduler.reset_prefix_cache(reset_running_requests=True)
|
||||
for request in requests:
|
||||
request.num_in_flight_tokens = 0
|
||||
|
||||
# Reset prefix cache with reset_running_requests=True. All running requests
|
||||
# Should be pushed back to the waiting queue and kv cache should be freed
|
||||
assert scheduler.reset_prefix_cache(reset_running_requests=True)
|
||||
scheduler.aux_output_connector.reset.assert_called_once_with()
|
||||
|
||||
# Verify requests moved from running to waiting
|
||||
assert len(scheduler.waiting) == len(requests)
|
||||
@@ -1490,6 +1529,16 @@ def test_scheduler_reset_prefix_cache():
|
||||
assert scheduler.waiting[i] == request
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reset_successful", [False, True])
|
||||
def test_aux_output_reset_follows_kv_reset_result(reset_successful: bool):
|
||||
scheduler = create_scheduler(enable_prefix_caching=True)
|
||||
scheduler.aux_output_connector = Mock()
|
||||
scheduler.kv_cache_manager.reset_prefix_cache = Mock(return_value=reset_successful)
|
||||
|
||||
assert scheduler.reset_prefix_cache() is reset_successful
|
||||
assert scheduler.aux_output_connector.reset.call_count == int(reset_successful)
|
||||
|
||||
|
||||
def test_kv_cache_release_after_keep_pause_preserves_requests():
|
||||
"""A completed keep pause permits release and recomputation of live requests."""
|
||||
scheduler = create_scheduler(max_num_seqs=1, enable_prefix_caching=True)
|
||||
@@ -3791,10 +3840,9 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
|
||||
scheduler.kv_event_publisher = Mock()
|
||||
scheduler.finished_req_ids = set()
|
||||
scheduler.finished_req_ids_dict = None
|
||||
scheduler.aux_output_connector = None
|
||||
scheduler.grammar_compile_error_reqs = set()
|
||||
scheduler.vllm_config = Mock()
|
||||
scheduler.vllm_config.model_config.enable_return_routed_experts = False
|
||||
scheduler.enable_return_routed_experts = False
|
||||
scheduler.return_sampling_mask = False
|
||||
scheduler.recompute_kv_load_failures = False
|
||||
scheduler.defer_block_free = False
|
||||
|
||||
@@ -1122,6 +1122,7 @@ def _run_mamba_prefix_cache_mrv2(
|
||||
logprobs_tensors=None,
|
||||
num_nans=None,
|
||||
num_sampled=num_sampled,
|
||||
num_rejected=num_rejected,
|
||||
)
|
||||
return sampler_output, num_sampled, num_rejected
|
||||
|
||||
|
||||
@@ -3,8 +3,9 @@
|
||||
|
||||
import math
|
||||
import time
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import MagicMock, Mock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from tests.v1.engine.utils import (
|
||||
@@ -1307,6 +1308,7 @@ def test_lora_request_tracking(log_stats: bool, dummy_test_vectors):
|
||||
async def test_request_output_collector():
|
||||
NUM_REQS = 3
|
||||
TEXT = "a"
|
||||
routed_experts = np.arange(12, dtype=np.uint8).reshape(2, 3, 2)
|
||||
|
||||
def make_outputs() -> list[RequestOutput]:
|
||||
return [
|
||||
@@ -1322,6 +1324,9 @@ async def test_request_output_collector():
|
||||
token_ids=[idx],
|
||||
cumulative_logprob=(idx + 1 * 1.0),
|
||||
logprobs=[{"a": idx, "b": idx}],
|
||||
routed_experts=(
|
||||
routed_experts if idx == NUM_REQS - 1 else None
|
||||
),
|
||||
finish_reason="length" if (idx == NUM_REQS - 1) else None,
|
||||
)
|
||||
],
|
||||
@@ -1374,6 +1379,7 @@ async def test_request_output_collector():
|
||||
|
||||
assert output.finished
|
||||
assert output.outputs[0].finish_reason == "length"
|
||||
np.testing.assert_array_equal(output.outputs[0].routed_experts, routed_experts)
|
||||
# Text, token_ids, and logprobs should get merged.
|
||||
assert output.outputs[0].text == TEXT * num_to_put
|
||||
for tok_0, tok_1 in zip(output.outputs[0].token_ids, list(range(num_to_put))):
|
||||
@@ -1385,6 +1391,46 @@ async def test_request_output_collector():
|
||||
assert output.outputs[0].cumulative_logprob == cumulative_logprob_expected
|
||||
|
||||
|
||||
def test_routed_experts_are_accumulated_until_finish():
|
||||
state = RequestState(
|
||||
request_id="request-int",
|
||||
external_req_id="request",
|
||||
parent_req=None,
|
||||
request_index=0,
|
||||
lora_request=None,
|
||||
output_kind=RequestOutputKind.DELTA,
|
||||
prompt="prompt",
|
||||
prompt_token_ids=[1, 2, 3, 4],
|
||||
prompt_embeds=None,
|
||||
logprobs_processor=Mock(
|
||||
logprobs=None,
|
||||
cumulative_logprob=0.0,
|
||||
pop_prompt_logprobs=Mock(return_value=None),
|
||||
),
|
||||
detokenizer=Mock(get_next_output_text=Mock(return_value="")),
|
||||
max_tokens_param=2,
|
||||
arrival_time=0.0,
|
||||
queue=None,
|
||||
log_stats=False,
|
||||
stream_interval=1,
|
||||
)
|
||||
prompt_chunk = np.arange(24, dtype=np.uint8).reshape(4, 3, 2)
|
||||
decode_chunk = np.arange(12, dtype=np.uint8).reshape(2, 3, 2) + 24
|
||||
state.routed_experts_chunks.append(prompt_chunk)
|
||||
|
||||
partial = state.make_request_output([5], None, None, None)
|
||||
assert partial is not None
|
||||
assert partial.outputs[0].routed_experts is None
|
||||
|
||||
state.routed_experts_chunks.append(decode_chunk)
|
||||
finished = state.make_request_output([6], None, FinishReason.LENGTH, None)
|
||||
assert finished is not None
|
||||
np.testing.assert_array_equal(
|
||||
finished.outputs[0].routed_experts,
|
||||
np.concatenate((prompt_chunk, decode_chunk)),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cumulative_output_collector_n():
|
||||
"""Test collector correctly handles multiple outputs by index."""
|
||||
|
||||
@@ -3,12 +3,14 @@
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.distributed.aux_output_connector.connector import (
|
||||
AuxRequestOutput,
|
||||
)
|
||||
from vllm.v1.executor.ray_utils import detach_zero_copy_from_model_runner_output
|
||||
from vllm.v1.outputs import (
|
||||
LogprobsLists,
|
||||
LogprobsTensors,
|
||||
ModelRunnerOutput,
|
||||
RoutedExpertsLists,
|
||||
)
|
||||
|
||||
|
||||
@@ -59,26 +61,19 @@ def test_detach_zero_copy_from_model_runner_output_copies_only_numpy_views():
|
||||
assert output.prompt_logprobs_dict["req-0"] is prompt_logprobs
|
||||
|
||||
|
||||
def test_detach_zero_copy_routed_experts_without_logprobs():
|
||||
def test_detach_zero_copy_aux_output_without_logprobs():
|
||||
rows = _make_readonly(np.arange(12, dtype=np.uint8).reshape(2, 3, 2))
|
||||
output = ModelRunnerOutput(
|
||||
req_ids=["req-0"],
|
||||
req_id_to_index={"req-0": 0},
|
||||
routed_experts=RoutedExpertsLists(
|
||||
routing_data=_make_readonly(np.arange(12, dtype=np.int32).reshape(2, 3, 2)),
|
||||
slot_mapping=_make_readonly(np.array([7, 8], dtype=np.int64)),
|
||||
),
|
||||
aux_output_connector_output={"req-0": AuxRequestOutput(0, rows)},
|
||||
)
|
||||
original = output.routed_experts
|
||||
assert output.logprobs is None
|
||||
|
||||
detach_zero_copy_from_model_runner_output(output)
|
||||
|
||||
detached = output.routed_experts
|
||||
assert detached is not None
|
||||
assert detached is not original
|
||||
assert detached.routing_data is not original.routing_data
|
||||
assert detached.slot_mapping is not original.slot_mapping
|
||||
assert detached.routing_data.flags.writeable
|
||||
assert detached.slot_mapping.flags.writeable
|
||||
np.testing.assert_array_equal(detached.routing_data, original.routing_data)
|
||||
np.testing.assert_array_equal(detached.slot_mapping, original.slot_mapping)
|
||||
aux_output = output.aux_output_connector_output
|
||||
assert aux_output is not None
|
||||
detached = aux_output["req-0"].rows
|
||||
assert detached is not rows
|
||||
assert detached.flags.writeable
|
||||
np.testing.assert_array_equal(detached, rows)
|
||||
|
||||
Reference in New Issue
Block a user