Merge remote-tracking branch 'origin/main' into sync/lesj/qwen4exp-bugfixes

Signed-off-by: lesj0610 <lesj0610@users.noreply.github.com>
This commit is contained in:
lesj0610
2026-09-23 00:40:48 +09:00
512 changed files with 29298 additions and 6783 deletions
@@ -30,10 +30,19 @@ DAEMON_TIMEOUT_S = 600
class WeightCacheDaemon:
"""Context manager running the real weight cache daemon as a subprocess."""
def __init__(self, model: str, tp_size: int, extra_args: list[str] | None = None):
def __init__(
self,
model: str,
tp_size: int,
extra_args: list[str] | None = None,
num_groups: int = 1,
):
# Short base path: Unix socket paths are limited to ~107 characters.
self.socket_dir = tempfile.mkdtemp(prefix="vllm_ipc_")
self.tp_size = tp_size
# Each daemon group (target plus cached drafts) binds one socket per
# local rank.
self.num_sockets = tp_size * num_groups
self._cmd = [
sys.executable,
"-m",
@@ -87,7 +96,7 @@ class WeightCacheDaemon:
f"Weight cache daemon exited with {self._proc.returncode}:\n"
f"{self._logs()}"
)
if len(glob.glob(pattern)) >= self.tp_size:
if len(glob.glob(pattern)) >= self.num_sockets:
return
time.sleep(1.0)
raise TimeoutError(
@@ -112,6 +121,9 @@ class ModelCase:
images: list | None = None
llm_kwargs: dict[str, Any] = field(default_factory=dict)
daemon_args: list[str] = field(default_factory=list)
# Daemon groups the launcher starts; 2 when a cached draft group joins the
# target group (MTP/EAGLE speculative decoding).
daemon_groups: int = 1
def generate(
@@ -172,14 +184,41 @@ K3_CASE = ModelCase(
daemon_args=["--trust-remote-code"],
)
# Qwen3.5-0.8B ships one MTP layer in the target checkpoint, so method="mtp"
# loads the draft from the same model. The daemon must cache it in its draft
# group for the warm runs (fallback=False) to succeed.
QWEN_MTP_CASE = ModelCase(
model="Qwen/Qwen3.5-0.8B",
prompts=[
"Hello, my name is",
"The capital of France is",
],
llm_kwargs=dict(
gpu_memory_utilization=0.3,
enforce_eager=True,
enable_chunked_prefill=True,
speculative_config={"method": "mtp", "num_speculative_tokens": 1},
),
daemon_args=[
"--speculative-config",
'{"method": "mtp", "num_speculative_tokens": 1}',
],
daemon_groups=2,
)
@pytest.mark.parametrize("case", [QWEN_CASE, K3_CASE], ids=["qwen3.5", "kimi-k3"])
@pytest.mark.parametrize(
"case",
[QWEN_CASE, K3_CASE, QWEN_MTP_CASE],
ids=["qwen3.5", "kimi-k3", "qwen3.5-mtp"],
)
def test_ipc_cache_cold_start_and_warm_restart(vllm_runner, case: ModelCase):
"""Cold start falls back to disk; warm restarts load weights via CUDA IPC.
All runs must produce outputs identical to a default-loader baseline. The
warm runs disable the disk fallback, so they only pass if the weights
really came from the daemon.
really came from the daemon — for the MTP case, both the target's and the
draft's daemon groups.
"""
if not current_platform.is_cuda_alike():
pytest.skip("Weight cache IPC sharing requires CUDA or ROCm")
@@ -199,7 +238,12 @@ def test_ipc_cache_cold_start_and_warm_restart(vllm_runner, case: ModelCase):
with tempfile.TemporaryDirectory(prefix="vllm_ipc_empty_") as empty_socket_dir:
cold_outputs = generate(vllm_runner, case, empty_socket_dir, fallback=True)
with WeightCacheDaemon(case.model, tp_size=1, extra_args=case.daemon_args) as d:
with WeightCacheDaemon(
case.model,
tp_size=1,
extra_args=case.daemon_args,
num_groups=case.daemon_groups,
) as d:
warm_outputs = generate(vllm_runner, case, d.socket_dir, fallback=False)
# Warm restart: a second engine lifetime against the same daemon.
restart_outputs = generate(vllm_runner, case, d.socket_dir, fallback=False)
+2 -1
View File
@@ -369,11 +369,12 @@ def test_dspark_shares_target_embedding_with_smaller_draft_vocabulary():
draft_parallel_config=SimpleNamespace(tensor_parallel_size=1),
attention_backend=None,
kv_cache_dtype=None,
draft_load_config=None,
),
parallel_config=ParallelConfig(),
attention_config=SimpleNamespace(backend=None),
cache_config=SimpleNamespace(),
load_config=SimpleNamespace(),
load_config=SimpleNamespace(load_format="auto"),
model_config=SimpleNamespace(get_vocab_size=Mock(return_value=100)),
)
@@ -7,26 +7,19 @@ from unittest.mock import Mock, patch
import pytest
import torch
from vllm.config import ModelConfig, VllmConfig
from vllm.config import VllmConfig
from vllm.config.compilation import CompilationMode
from vllm.distributed.eplb.eplb_state import EplbLayerState
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
RoutedExpertsCapturer,
RoutedExpertsManager,
bind_routed_experts_capturer,
get_routed_experts_attn_gid,
)
from vllm.model_executor.layers.fused_moe.router.base_router import BaseRouter
from vllm.transformers_utils.model_arch_config_convertor import (
ModelArchConfigConvertorBase,
)
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
MLAAttentionSpec,
SlidingWindowMLASpec,
UniformTypeKVCacheSpecs,
)
@@ -42,6 +35,7 @@ def _capturer_with_buffer(
num_experts_per_tok: int = 2,
dp_rank: int = 0,
tp_size: int = 1,
dtype: torch.dtype = torch.int32,
) -> RoutedExpertsCapturer:
# Bypass __init__ so the test can use a CPU buffer and skip the
# VllmConfig dependency. The CUDA device-tensor allocation in the
@@ -49,10 +43,11 @@ def _capturer_with_buffer(
c = RoutedExpertsCapturer.__new__(RoutedExpertsCapturer)
c.dp_rank = dp_rank
c.tp_size = tp_size
c.output_dtype = torch.uint8
c.device_buffer = torch.full(
(max_tokens, num_layers, num_experts_per_tok),
-1,
dtype=torch.int32,
dtype=dtype,
)
return c
@@ -88,74 +83,36 @@ def _make_modular_routed_experts():
)
def _make_model_config(hf_config):
num_experts_per_token = ModelArchConfigConvertorBase(
hf_config, hf_config
).get_num_experts_per_token()
model_config = SimpleNamespace(
hf_text_config=hf_config,
model_arch_config=SimpleNamespace(
num_experts_per_token=num_experts_per_token,
def _full_attention_kv_group(
spec_type: type[FullAttentionSpec] = FullAttentionSpec,
) -> KVCacheGroupSpec:
full_attention = spec_type(
block_size=16,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
)
return KVCacheGroupSpec(
["layer"],
UniformTypeKVCacheSpecs(
block_size=16,
kv_cache_specs={"layer": full_attention},
),
)
model_config.get_num_experts = lambda: hf_config.num_experts
model_config.get_num_experts_per_tok = lambda: (
ModelConfig.get_num_experts_per_tok(model_config)
)
model_config.get_total_num_hidden_layers = lambda: hf_config.num_hidden_layers
return model_config
def test_routed_experts_manager_uses_gemma4_top_k_experts():
hf_config = SimpleNamespace(
num_experts=8,
top_k_experts=2,
num_hidden_layers=3,
)
vllm_config = SimpleNamespace(model_config=_make_model_config(hf_config))
kv_cache_spec = FullAttentionSpec(
block_size=4,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
)
kv_cache_config = KVCacheConfig(
num_blocks=2,
kv_cache_tensors=[],
kv_cache_groups=[KVCacheGroupSpec(["layer"], kv_cache_spec)],
)
@pytest.mark.parametrize("eplb_enabled", [False, True])
def test_base_router_capture_pre_eplb_mapping(eplb_enabled):
eplb_state = None
if eplb_enabled:
eplb_state = EplbLayerState()
eplb_state.expert_load_view = torch.zeros(32, dtype=torch.int64)
eplb_state.logical_to_physical_map = torch.arange(32).view(32, 1)
eplb_state.logical_replica_count = torch.ones(32, dtype=torch.int64)
eplb_state.should_record_tensor = torch.ones((), dtype=torch.bool)
eplb_state.num_unpadded_tokens_tensors = [torch.tensor(0, dtype=torch.int32)]
router = _make_router(eplb_state)
manager = RoutedExpertsManager(vllm_config, kv_cache_config)
assert manager.routed_experts_by_slot.shape == (8, 3, 2)
def test_routed_experts_manager_uses_kimi_k3_experts_per_token():
hf_config = SimpleNamespace(
num_experts=8,
num_experts_per_token=2,
num_hidden_layers=3,
)
vllm_config = SimpleNamespace(model_config=_make_model_config(hf_config))
kv_cache_spec = FullAttentionSpec(
block_size=4,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
)
kv_cache_config = KVCacheConfig(
num_blocks=2,
kv_cache_tensors=[],
kv_cache_groups=[KVCacheGroupSpec(["layer"], kv_cache_spec)],
)
manager = RoutedExpertsManager(vllm_config, kv_cache_config)
assert manager.routed_experts_by_slot.shape == (8, 3, 2)
def test_base_router_capture_pre_eplb_mapping():
router = _make_router()
captured = []
def capture_fn(ids):
@@ -173,34 +130,7 @@ def test_base_router_capture_pre_eplb_mapping():
assert torch.equal(topk_ids, torch.tensor([[11, 12], [13, 14]]))
def test_base_router_capture_with_eplb_enabled():
eplb_state = EplbLayerState()
eplb_state.expert_load_view = torch.zeros(32, dtype=torch.int64)
eplb_state.logical_to_physical_map = torch.arange(32).view(32, 1)
eplb_state.logical_replica_count = torch.ones(32, dtype=torch.int64)
eplb_state.should_record_tensor = torch.ones((), dtype=torch.bool)
eplb_state.num_unpadded_tokens_tensors = [torch.tensor(0, dtype=torch.int32)]
router = _make_router(eplb_state=eplb_state)
captured = []
def capture_fn(ids):
captured.append(ids.clone())
router.set_capture_fn(capture_fn)
_, topk_ids = router.select_experts(
hidden_states=torch.empty(1),
router_logits=torch.empty(1),
)
assert len(captured) == 1
# Capture should see logical ids pre-EPLB mapping.
assert torch.equal(captured[0], torch.tensor([[1, 2], [3, 4]]))
# Our DummyRouter mapping adds +10.
assert torch.equal(topk_ids, torch.tensor([[11, 12], [13, 14]]))
def test_public_binding_only_visits_target_model(monkeypatch):
def test_public_binding_binds_target_model_router(monkeypatch):
class DummyFusedMoE:
def __init__(self, layer_id):
self.layer_id = layer_id
@@ -208,7 +138,6 @@ def test_public_binding_only_visits_target_model(monkeypatch):
self._quant_method = _make_modular_routed_experts().quant_method
target_module = DummyFusedMoE(layer_id=7)
draft_module = DummyFusedMoE(layer_id=0)
import vllm.model_executor.layers.fused_moe.layer as fused_moe_layer
@@ -221,12 +150,30 @@ def test_public_binding_only_visits_target_model(monkeypatch):
)
assert target_module.router.capture_fn is not None
assert draft_module.router.capture_fn is None
topk_ids = torch.tensor([[5, 6]])
target_module.router.capture_fn(topk_ids)
assert calls == [(7, topk_ids)]
def test_public_binding_supports_direct_capture_source():
class DirectCaptureSource(torch.nn.Module):
def __init__(self):
super().__init__()
self.layer_id = 3
self.capture_fn = None
source = DirectCaptureSource()
calls = []
capturer = types.SimpleNamespace(capture=lambda *args: calls.append(args))
bind_routed_experts_capturer(source, capturer)
assert source.capture_fn is not None
topk_ids = torch.tensor([[1, 2]])
source.capture_fn(topk_ids)
assert calls == [(3, topk_ids)]
def test_public_binding_rejects_monolithic_without_replay_support(monkeypatch):
class DummyFusedMoE:
def __init__(self):
@@ -277,6 +224,51 @@ def test_routed_experts_capturer_single_dp_no_metadata():
assert capturer.device_buffer[3, 0, 0].item() == -1
@pytest.mark.parametrize(
("num_experts", "dtype_name", "torch_dtype"),
[(256, "uint8", torch.uint8), (257, "uint16", torch.uint16)],
)
def test_routed_experts_capturer_exposes_output_profile(
monkeypatch, num_experts, dtype_name, torch_dtype
):
import vllm.model_executor.layers.fused_moe.routed_experts_capturer as module
monkeypatch.setattr(module, "current_platform", SimpleNamespace(device_type="cpu"))
config = SimpleNamespace(
model_config=SimpleNamespace(
get_total_num_hidden_layers=lambda: 3,
get_num_experts=lambda: num_experts,
get_num_experts_per_tok=lambda: 2,
hf_text_config=SimpleNamespace(model_type="test"),
),
parallel_config=SimpleNamespace(data_parallel_rank=0, tensor_parallel_size=1),
)
capturer = RoutedExpertsCapturer(8, config)
assert capturer.shape_per_token == (3, 2)
assert capturer.output_dtype_name == dtype_name
assert capturer.output_dtype == torch_dtype
assert capturer.device_buffer.shape == (8, 3, 2)
@pytest.mark.parametrize("output_dtype", [torch.uint8, torch.uint16])
def test_routed_experts_capturer_narrows_snapshot(output_dtype):
capturer = _capturer_with_buffer(dtype=torch.int32)
capturer.output_dtype = output_dtype
topk = torch.tensor([[1, 2], [254, 255]], dtype=torch.int64)
ctx = SimpleNamespace(dp_metadata=None)
with patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx):
capturer.capture(layer_id=0, topk_ids=topk)
output = capturer.snapshot_routing_data(2)
assert capturer.device_buffer.dtype == torch.int32
assert capturer.device_buffer[:2, 0, :].tolist() == topk.tolist()
assert output.dtype == output_dtype
assert output[:, 0, :].tolist() == topk.tolist()
assert output.data_ptr() != capturer.device_buffer.data_ptr()
def test_routed_experts_capturer_dp_naive_concatenated_all_ranks():
"""N == sum(num_tokens_dp): slice this rank's segment from concatenated topk."""
capturer = _capturer_with_buffer(dp_rank=1)
@@ -307,6 +299,47 @@ def test_routed_experts_capturer_dp_modular_local_tokens():
assert torch.equal(capturer.device_buffer[:3, 0, :], topk)
def test_routed_experts_capturer_dp_ep_gathered_shards():
capturer = _capturer_with_buffer(dp_rank=1, tp_size=2)
num_tokens_dp = torch.tensor([3, 2], dtype=torch.int32)
ctx = SimpleNamespace(
dp_metadata=SimpleNamespace(
num_tokens_across_dp_cpu=num_tokens_dp,
local_sizes=[2, 2, 1, 1],
)
)
topk = torch.tensor(
[[0, 1], [2, 3], [4, 5], [-1, -1], [10, 11], [12, 13]],
dtype=torch.int32,
)
with patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx):
capturer.capture(layer_id=0, topk_ids=topk)
assert torch.equal(capturer.device_buffer[:2, 0], topk[4:])
def test_routed_experts_capturer_sp_modular_gathers_tp_shards():
capturer = _capturer_with_buffer(dp_rank=1, tp_size=2)
num_tokens_dp = torch.tensor([2, 3], dtype=torch.int32)
ctx = SimpleNamespace(
dp_metadata=SimpleNamespace(num_tokens_across_dp_cpu=num_tokens_dp)
)
local_shard = torch.tensor([[10, 11], [12, 13]], dtype=torch.int32)
gathered = torch.tensor([[10, 11], [12, 13], [14, 15], [-1, -1]], dtype=torch.int32)
tp_group = Mock()
tp_group.all_gather.return_value = gathered
with (
patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx),
patch(f"{_REC_MODULE}.get_tp_group", return_value=tp_group),
):
capturer.capture(layer_id=0, topk_ids=local_shard)
tp_group.all_gather.assert_called_once_with(local_shard, dim=0)
assert torch.equal(capturer.device_buffer[:3, 0], gathered[:3])
def test_routed_experts_capturer_dp_unexpected_batch_raises():
"""Mismatch between topk batch dim and DP layout: fail fast."""
capturer = _capturer_with_buffer(dp_rank=0)
@@ -324,128 +357,119 @@ def test_routed_experts_capturer_dp_unexpected_batch_raises():
assert capturer.device_buffer[0, 0, 0].item() == -1
def test_routed_experts_attention_group_is_shared_and_fail_closed():
"""Both sides key routing data by this gid, so it must skip non-full-attention
groups rather than defaulting to 0, and fail closed when none exists."""
common = dict(num_kv_heads=1, head_size=1, dtype=torch.float32)
def test_get_aux_output_connector_passes_config(monkeypatch):
import vllm.distributed.aux_output_connector.worker as aux_output_worker
connector = Mock()
constructor = Mock(return_value=connector)
monkeypatch.setattr(aux_output_worker, "AuxOutputWorkerConnector", constructor)
config = SimpleNamespace(
kv_cache_groups=[
KVCacheGroupSpec(
["swa_layer"],
SlidingWindowMLASpec(block_size=4, sliding_window=8, **common),
),
KVCacheGroupSpec(["full_layer"], FullAttentionSpec(block_size=4, **common)),
]
scheduler_config=SimpleNamespace(max_num_batched_tokens=32)
)
assert get_routed_experts_attn_gid(config) == 1
model = Mock()
kv_cache_config = Mock()
with pytest.raises(ValueError, match="requires a full-attention KV cache group"):
get_routed_experts_attn_gid(SimpleNamespace(kv_cache_groups=[]))
result = aux_output_worker.get_aux_output_connector(model, config, kv_cache_config)
def test_routed_experts_attention_group_unwraps_uniform_type_specs():
"""DeepSeek-V4-shaped groups wrap their specs in ``UniformTypeKVCacheSpecs``.
The wrapper is not a ``FullAttentionSpec``, so a bare isinstance check finds
no group and fails closed on every worker. Unwrap semantics themselves are
covered by ``test_is_full_attention_spec_*`` in tests/v1/core.
"""
common = dict(num_kv_heads=1, head_size=1, dtype=torch.float32)
swa_spec = SlidingWindowMLASpec(block_size=4, sliding_window=8, **common)
mla_spec = MLAAttentionSpec(block_size=4, **common)
config = SimpleNamespace(
kv_cache_groups=[
KVCacheGroupSpec(
["swa_layer"],
UniformTypeKVCacheSpecs(
block_size=4, kv_cache_specs={"swa_layer": swa_spec}
),
),
KVCacheGroupSpec(
["mla_layer"],
UniformTypeKVCacheSpecs(
block_size=4, kv_cache_specs={"mla_layer": mla_spec}
),
),
]
constructor.assert_called_once_with(
model=model,
kv_cache_config=kv_cache_config,
vllm_config=config,
)
assert get_routed_experts_attn_gid(config) == 1
swa_only = SimpleNamespace(kv_cache_groups=[config.kv_cache_groups[0]])
with pytest.raises(ValueError, match="requires a full-attention KV cache group"):
get_routed_experts_attn_gid(swa_only)
assert result is connector
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
def test_mrv2_async_output_returns_existing_routed_experts_field():
from vllm.v1.outputs import ModelRunnerOutput, RoutedExpertsTensors
from vllm.v1.worker.gpu.async_utils import AsyncOutput
from vllm.v1.worker.gpu.sample.output import SamplerOutput
routed_experts = RoutedExpertsTensors(
routing_data=torch.arange(6, dtype=torch.int32, device="cuda").reshape(3, 1, 2),
slot_mapping=torch.tensor([11, 12, 13], device="cuda"),
)
num_sampled = torch.tensor([1], dtype=torch.int32, device="cuda")
sampler_output = SamplerOutput(
sampled_token_ids=torch.tensor([[1]], device="cuda"),
logprobs_tensors=None,
num_nans=None,
num_sampled=num_sampled,
num_rejected=torch.tensor([0], dtype=torch.int32, device="cuda"),
)
output = AsyncOutput(
model_runner_output=ModelRunnerOutput(req_ids=["req"], req_id_to_index={}),
sampler_output=sampler_output,
num_sampled_tokens=num_sampled,
main_stream=torch.cuda.current_stream(),
copy_stream=torch.cuda.Stream(),
routed_experts=routed_experts,
).get_output()
assert output.routed_experts is not None
assert output.routed_experts.routing_data[:, 0, 0].tolist() == [0, 2, 4]
assert output.routed_experts.slot_mapping.tolist() == [11, 12, 13]
@pytest.mark.parametrize("rank", [0, 1])
def test_all_tp_ranks_initialize_capture(monkeypatch, rank):
pytest.importorskip("vllm.vllm_flash_attn", exc_type=ImportError)
import vllm.v1.worker.gpu.model_runner as model_runner
def test_aux_output_worker_connector_binds_capture_on_non_output_rank(monkeypatch):
import vllm.distributed.aux_output_connector.worker as aux_output_worker
capturer = Mock()
constructor = Mock(return_value=capturer)
bind = Mock()
monkeypatch.setattr(model_runner, "RoutedExpertsCapturer", constructor)
monkeypatch.setattr(model_runner, "bind_routed_experts_capturer", bind)
monkeypatch.setattr(aux_output_worker, "RoutedExpertsCapturer", constructor)
monkeypatch.setattr(aux_output_worker, "bind_routed_experts_capturer", bind)
monkeypatch.setattr(
aux_output_worker,
"get_tp_group",
lambda: SimpleNamespace(is_first_rank=False, world_size=1),
)
runner = model_runner.GPUModelRunner.__new__(model_runner.GPUModelRunner)
runner.max_num_tokens = 32
runner.vllm_config = SimpleNamespace(parallel_config=SimpleNamespace(rank=rank))
runner.kv_cache_config = SimpleNamespace()
runner.model = Mock()
runner.init_routed_experts_capturer()
config = SimpleNamespace(
aux_output_config=SimpleNamespace(enable_return_routed_experts=True),
kv_transfer_config=None,
max_concurrent_batches=2,
scheduler_config=SimpleNamespace(max_num_batched_tokens=32),
)
model = Mock()
connector = aux_output_worker.AuxOutputWorkerConnector(
vllm_config=config,
model=model,
kv_cache_config=SimpleNamespace(kv_cache_groups=[_full_attention_kv_group()]),
)
constructor.assert_called_once_with(
max_num_batched_tokens=32,
vllm_config=runner.vllm_config,
kv_cache_config=runner.kv_cache_config,
vllm_config=config,
)
bind.assert_called_once_with(runner.model, capturer)
assert runner.routed_experts_capturer is capturer
bind.assert_called_once_with(model, capturer)
connector.begin_step(Mock())
assert connector.prepare_output(Mock()) is None
capturer.snapshot_routing_data.assert_not_called()
def test_aux_output_worker_connector_default_capacity(monkeypatch):
import vllm.distributed.aux_output_connector.worker as aux_output_worker
tp_group = SimpleNamespace(is_first_rank=True, world_size=1)
store_constructor = Mock()
background_store_constructor = Mock(side_effect=lambda store, **_: store)
capturer = SimpleNamespace(shape_per_token=(2,), output_dtype_name="int32")
monkeypatch.setattr(aux_output_worker, "get_tp_group", lambda: tp_group)
monkeypatch.setattr(
aux_output_worker, "RoutedExpertsCapturer", Mock(return_value=capturer)
)
monkeypatch.setattr(aux_output_worker, "bind_routed_experts_capturer", Mock())
monkeypatch.setattr(
aux_output_worker,
"resolve_kv_cache_block_sizes",
lambda *_: (32, 16),
)
monkeypatch.setattr(aux_output_worker, "BlockObjectStore", store_constructor)
monkeypatch.setattr(
aux_output_worker, "BackgroundBlockObjectStore", background_store_constructor
)
monkeypatch.setattr(aux_output_worker, "RoutedExpertsBuffer", Mock())
config = SimpleNamespace(
aux_output_config=SimpleNamespace(max_bytes=None),
kv_transfer_config=None,
cache_config=SimpleNamespace(enable_prefix_caching=True),
scheduler_config=SimpleNamespace(max_num_seqs=8, max_num_batched_tokens=32),
max_concurrent_batches=2,
)
kwargs = dict(
vllm_config=config,
model=Mock(),
kv_cache_config=SimpleNamespace(
num_blocks=10,
kv_cache_groups=[_full_attention_kv_group(MLAAttentionSpec)],
),
)
aux_output_worker.AuxOutputWorkerConnector(**kwargs)
assert store_constructor.call_args.kwargs["max_bytes"] == 2560
assert store_constructor.call_args.kwargs["object_nbytes"] == 128
assert background_store_constructor.call_args.kwargs["max_pending_batches"] == 16
def test_v2_model_runner_accepts_routed_experts(monkeypatch):
monkeypatch.setattr("importlib.metadata.entry_points", lambda **_: ())
config = SimpleNamespace(
model_config=SimpleNamespace(
enable_return_routed_experts=True,
use_mla=False,
logits_processors=None,
enable_prompt_embeds=False,
),
aux_output_config=SimpleNamespace(enable_return_routed_experts=True),
speculative_config=None,
parallel_config=SimpleNamespace(
prefill_context_parallel_size=1,