From 0b7f11a1eece3dc9c689f1b5ab90d60c03b10793 Mon Sep 17 00:00:00 2001 From: Hongxin Xu <70438206+xhx1022@users.noreply.github.com> Date: Mon, 21 Sep 2026 18:25:05 +0800 Subject: [PATCH] [AuxOutput] Add block-keyed storage for routed-expert outputs (#45635) Signed-off-by: aoshen02 Signed-off-by: Nick Hill Co-authored-by: aoshen02 Co-authored-by: OpenAI Codex Co-authored-by: Roger Wang Co-authored-by: Nick Hill Co-authored-by: Codex --- tests/config/test_aux_output_config.py | 119 ++ .../aux_output_connector/test_store.py | 1854 +++++++++++++++++ .../openai/test_return_routed_experts.py | 19 +- .../test_return_routed_experts.py | 16 +- .../test_routed_experts_capture.py | 420 ++-- tests/v1/core/test_async_scheduler.py | 3 +- tests/v1/core/test_scheduler.py | 94 +- .../v1/e2e/general/test_mamba_prefix_cache.py | 1 + tests/v1/engine/test_output_processor.py | 48 +- tests/v1/executor/test_ray_utils.py | 29 +- vllm/config/__init__.py | 3 + vllm/config/aux_output.py | 31 + vllm/config/model.py | 2 - vllm/config/vllm.py | 86 +- .../aux_output_connector/__init__.py | 2 + .../aux_output_connector/connector.py | 157 ++ .../aux_output_connector/routed_experts.py | 219 ++ .../distributed/aux_output_connector/store.py | 234 +++ .../aux_output_connector/worker.py | 412 ++++ vllm/engine/arg_utils.py | 18 +- .../fused_moe/routed_experts_capturer.py | 274 +-- vllm/outputs.py | 4 + vllm/v1/core/sched/output.py | 7 + vllm/v1/core/sched/scheduler.py | 131 +- vllm/v1/executor/ray_utils.py | 12 +- vllm/v1/outputs.py | 80 +- vllm/v1/worker/gpu/async_utils.py | 22 +- vllm/v1/worker/gpu/model_runner.py | 48 +- vllm/v1/worker/gpu/sample/batch_shard.py | 2 - vllm/v1/worker/gpu/sample/output.py | 4 +- vllm/v1/worker/gpu_model_runner.py | 127 -- vllm/v1/worker/gpu_worker.py | 3 - 32 files changed, 3604 insertions(+), 877 deletions(-) create mode 100644 tests/config/test_aux_output_config.py create mode 100644 tests/distributed/aux_output_connector/test_store.py create mode 100644 vllm/config/aux_output.py create mode 100644 vllm/distributed/aux_output_connector/__init__.py create mode 100644 vllm/distributed/aux_output_connector/connector.py create mode 100644 vllm/distributed/aux_output_connector/routed_experts.py create mode 100644 vllm/distributed/aux_output_connector/store.py create mode 100644 vllm/distributed/aux_output_connector/worker.py diff --git a/tests/config/test_aux_output_config.py b/tests/config/test_aux_output_config.py new file mode 100644 index 0000000000..bf428ea2dd --- /dev/null +++ b/tests/config/test_aux_output_config.py @@ -0,0 +1,119 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace +from typing import Any + +import pytest + +from vllm.config import AuxOutputConfig, VllmConfig +from vllm.engine.arg_utils import EngineArgs + +pytestmark = pytest.mark.cpu_test + + +def _config( + *, + use_v2: bool = True, + pp: int = 1, + dcp: int = 1, + pcp: int = 1, + connector: str | None = None, + runner_type: str = "generate", + is_moe: bool = True, + sliding_window: int | None = None, + attention_chunk_size: int | None = None, + enable_prefix_caching: bool = True, + adaptive_verification: bool = False, +): + return SimpleNamespace( + model_config=SimpleNamespace( + runner_type=runner_type, + is_moe=is_moe, + get_sliding_window=lambda: sliding_window, + attention_chunk_size=attention_chunk_size, + ), + use_v2_model_runner=use_v2, + parallel_config=SimpleNamespace( + pipeline_parallel_size=pp, + decode_context_parallel_size=dcp, + prefill_context_parallel_size=pcp, + ), + aux_output_config=AuxOutputConfig(enable_return_routed_experts=True), + cache_config=SimpleNamespace( + enable_prefix_caching=enable_prefix_caching, + ), + speculative_config=( + SimpleNamespace(enable_adaptive_verification=True) + if adaptive_verification + else None + ), + kv_transfer_config=( + None + if connector is None + else SimpleNamespace( + is_kv_transfer_instance=True, + kv_connector=connector, + ) + ), + ) + + +def test_aux_output_config_defaults(): + config = AuxOutputConfig() + + assert not config.enabled + assert not config.enable_return_routed_experts + assert config.max_bytes is None + + +def test_aux_output_capture_changes_compilation_hash(): + disabled = AuxOutputConfig() + enabled = AuxOutputConfig(enable_return_routed_experts=True) + + assert disabled.compute_hash() != enabled.compute_hash() + + +def test_legacy_routed_experts_flag_updates_aux_output_config(): + args = EngineArgs(enable_return_routed_experts=True) + + assert args.aux_output_config.enabled + assert args.aux_output_config.enable_return_routed_experts + + +@pytest.mark.parametrize( + ("kwargs", "error"), + [ + ({"use_v2": False}, "requires Model Runner V2"), + ({"runner_type": "pooling"}, "only supports generate runners"), + ({"is_moe": False}, "only supports MoE models"), + ({"enable_prefix_caching": False}, "requires prefix caching"), + ( + {"adaptive_verification": True}, + "adaptive speculative verification", + ), + ({"pp": 2}, "pipeline parallelism"), + ({"dcp": 2}, "context parallelism"), + ({"pcp": 2}, "context parallelism"), + ({"connector": "MooncakeConnector"}, "incompatible with KV connectors"), + ], +) +def test_aux_output_connector_rejects_unsupported_configuration(kwargs, error): + with pytest.raises(ValueError, match=error): + VllmConfig._verify_aux_output_compatibility(_config(**kwargs)) + + +@pytest.mark.parametrize( + "kwargs", + [{"sliding_window": 4096}, {"attention_chunk_size": 4096}], +) +def test_aux_output_config_defers_attention_layout_to_kv_config(kwargs): + VllmConfig._verify_aux_output_compatibility(_config(**kwargs)) + + +def test_aux_output_guards_are_inactive_when_capture_is_disabled(): + config: Any = VllmConfig.__new__(VllmConfig) + config.model_config = SimpleNamespace() + config.aux_output_config = AuxOutputConfig() + + config._verify_aux_output_compatibility() diff --git a/tests/distributed/aux_output_connector/test_store.py b/tests/distributed/aux_output_connector/test_store.py new file mode 100644 index 0000000000..39277f3e7e --- /dev/null +++ b/tests/distributed/aux_output_connector/test_store.py @@ -0,0 +1,1854 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import threading +from contextlib import nullcontext +from dataclasses import dataclass, field +from types import SimpleNamespace +from unittest.mock import Mock + +import numpy as np +import pytest +import torch + +from vllm.distributed.aux_output_connector.connector import ( + AuxOutputConnectorMetadata, + AuxOutputSchedulerConnector, + AuxRequestOutput, + PackedBlockHashes, +) +from vllm.distributed.aux_output_connector.routed_experts import ( + RoutedExpertsBuffer, + materialize_routed_experts, + publish_routed_experts, + routed_experts_keys, +) +from vllm.distributed.aux_output_connector.store import ( + BackgroundBlockObjectStore, + BlockObject, + BlockObjectStore, + BlockObjectStoreError, +) +from vllm.distributed.aux_output_connector.worker import ( + AuxOutputWorkerConnector, +) +from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput +from vllm.v1.outputs import ModelRunnerOutput +from vllm.v1.worker.gpu import async_utils +from vllm.v1.worker.gpu.sample.output import SamplerOutput + +pytestmark = pytest.mark.cpu_test + +_SHAPE = (3, 2) +_DTYPE = np.dtype("uint8") +_BLOCK_SIZE = 4 + + +def test_background_store_publishes_without_blocking_caller(): + started = threading.Event() + release = threading.Event() + underlying = Mock() + + def put(*_args, **_kwargs): + started.set() + release.wait() + + underlying.put.side_effect = put + store = BackgroundBlockObjectStore(underlying, max_pending_batches=1) + + store.put([BlockObject("key", b"value")]) + assert started.wait(timeout=1) + release.set() + store.close() + + underlying.close.assert_called_once_with() + + +def _make_connector(): + return AuxOutputSchedulerConnector() + + +@dataclass +class _Execution: + request_id: str + token_start: int + num_tokens: int + emit_start: int + block_hashes: list[bytes] | PackedBlockHashes + + +@dataclass +class _WorkerStep: + metadata: AuxOutputConnectorMetadata + executions: list[_Execution] + + +@dataclass +class _SchedulerRequest: + request_id: str + block_hashes: list[bytes] + num_tokens: int + num_output_tokens: int = 1 + num_computed_tokens: int = 0 + num_in_flight_tokens: int = 0 + finished: bool = False + sampling_params: SimpleNamespace = field( + default_factory=lambda: SimpleNamespace(routed_experts_prompt_start=0) + ) + + def is_finished(self) -> bool: + return self.finished + + +def _request_metadata( + request_id: str, + token_start: int, + num_tokens: int, + emit_start: int, + block_hashes, +) -> _Execution: + return _Execution(request_id, token_start, num_tokens, emit_start, block_hashes) + + +def _metadata( + generation: int, + requests: list[_Execution], + finished_requests: dict[str, list[bytes] | PackedBlockHashes], + hash_updates: dict[str, list[bytes] | PackedBlockHashes] | None = None, +) -> _WorkerStep: + block_hashes = { + request.request_id: request.block_hashes + for request in requests + if request.block_hashes + } + if hash_updates: + block_hashes.update( + (request_id, hashes) + for request_id, hashes in hash_updates.items() + if hashes + ) + block_hashes.update( + (request_id, hashes) + for request_id, hashes in finished_requests.items() + if hashes + ) + return _WorkerStep( + AuxOutputConnectorMetadata( + generation, + {request.request_id: request.emit_start for request in requests}, + block_hashes, + tuple(finished_requests), + ), + requests, + ) + + +def _execution_ranges(metadata, request_ids): + by_request = {request.request_id: request for request in metadata.executions} + token_starts = tuple( + by_request[request_id].token_start for request_id in request_ids + ) + num_tokens = tuple(by_request[request_id].num_tokens for request_id in request_ids) + return token_starts, num_tokens + + +def _begin_step(worker: AuxOutputWorkerConnector, step: _WorkerStep) -> None: + worker.begin_step(step.metadata) + + +def _make_worker( + max_num_seqs: int, + max_num_batched_tokens: int | None = None, + max_concurrent_batches: int = 2, + max_store_blocks: int | None = None, +) -> AuxOutputWorkerConnector: + object_nbytes = _BLOCK_SIZE * int(np.prod(_SHAPE)) + store = _make_store( + max_bytes=( + 1 << 20 if max_store_blocks is None else max_store_blocks * object_nbytes + ), + object_nbytes=object_nbytes, + ) + worker = object.__new__(AuxOutputWorkerConnector) + worker._store = store + worker._buffer = RoutedExpertsBuffer( + _DTYPE, + _SHAPE, + _BLOCK_SIZE, + max_num_seqs, + max_num_batched_tokens or max_num_seqs * _BLOCK_SIZE, + max_concurrent_batches, + ) + worker._requests = {} + worker._generation = 0 + worker._step_metadata = None + worker._pending_outputs = [] + worker._lock = threading.Lock() + worker._max_concurrent_batches = max_concurrent_batches + return worker + + +def test_worker_rejects_metadata_generation_rollback(): + worker = _make_worker(1) + _begin_step(worker, _metadata(1, [], {})) + + with pytest.raises(AssertionError, match="generation moved backwards"): + _begin_step(worker, _metadata(0, [], {})) + + worker.close() + + +def test_worker_rejects_run_and_finish_in_one_step(): + worker = _make_worker(1) + request = _request_metadata("request", 0, 1, 0, []) + metadata = _metadata( + 0, + [request], + {"request": []}, + ) + + with pytest.raises(AssertionError, match="cannot run and finish"): + _begin_step(worker, metadata) + + worker.close() + + +def _input_batch(request_ids, token_starts, query_start_loc): + return SimpleNamespace( + req_ids=request_ids, + num_computed_tokens_np=token_starts, + query_start_loc_np=query_start_loc, + ) + + +def test_non_output_rank_skips_capture_snapshot(): + worker = object.__new__(AuxOutputWorkerConnector) + worker._store = None + worker._buffer = None + worker._capturer = Mock() + worker._step_metadata = Mock() + assert worker.prepare_output(_input_batch([], np.array([]), np.array([]))) is None + worker._capturer.snapshot_routing_data.assert_not_called() + + +def test_worker_skips_aux_outputs_for_internal_warmup_step(): + worker = _make_worker(1) + worker._capturer = Mock() + + worker.begin_step(None) + + assert worker.prepare_output(_input_batch([], np.array([]), np.array([]))) is None + worker._capturer.snapshot_routing_data.assert_not_called() + worker.close() + + +def test_next_step_does_not_consume_pending_output(monkeypatch): + """begin_step must not wait for or consume an unconsumed step output.""" + event = Mock() + monkeypatch.setattr(torch.cuda, "Event", lambda **kwargs: event) + monkeypatch.setattr(async_utils, "stream", lambda *args: nullcontext()) + monkeypatch.setattr( + async_utils, "async_copy_to_np", lambda tensor: tensor.numpy().copy() + ) + worker = _make_worker(1) + worker.begin_step( + _metadata(0, [_request_metadata("request", 0, 1, 0, [])], {}).metadata + ) + rows = torch.ones((1, *_SHAPE), dtype=torch.uint8) + worker._capturer = Mock() + worker._capturer.snapshot_routing_data.return_value = rows + process_output = Mock(wraps=worker.process_output) + monkeypatch.setattr(worker, "process_output", process_output) + pending = worker.prepare_output( + _input_batch(["request"], np.array([0]), np.array([0, 1])) + ) + assert pending is not None + output = async_utils.AsyncOutput( + ModelRunnerOutput(["request"], {"request": 0}), + SamplerOutput( + torch.tensor([[7]]), None, None, None, num_rejected=torch.tensor([0]) + ), + torch.tensor([1]), + Mock(), + Mock(), + check_ep_fault=False, + pending_aux_output=pending, + ) + event.record.assert_called_once() + assert worker._pending_outputs == [pending] + + worker.begin_step(_metadata(0, [], {}).metadata) + process_output.assert_not_called() + assert worker._pending_outputs == [pending] + + result = output.get_output() + + assert result.sampled_token_ids == [[7]] + np.testing.assert_array_equal( + result.aux_output_connector_output["request"].rows, rows.numpy() + ) + process_output.assert_called_once() + assert worker._pending_outputs == [] + worker.close() + + +@pytest.mark.parametrize("num_pending", [1, 2]) +def test_finished_request_teardown_waits_for_pending_output(num_pending): + """Terminal cleanup waits for every outstanding output, not just the first.""" + worker = _make_worker(1) + rows = np.arange(4 * 3 * 2, dtype=np.uint8).reshape(4, *_SHAPE) + worker._capturer = Mock() + worker._capturer.snapshot_routing_data.return_value = torch.from_numpy(rows) + pending_outputs = [] + for i in range(num_pending): + request = _request_metadata("request", 4 * i, 4, 4 * i, [bytes([i]) * 4]) + worker.begin_step(_metadata(0, [request], {}).metadata) + pending = worker.prepare_output( + _input_batch(["request"], np.array([4 * i]), np.array([0, 4])) + ) + assert pending is not None + pending.enqueue_cpu_copy(np.array([1]), np.array([0])) + pending_outputs.append(pending) + + # The next step finishes the request while its output is unconsumed. + worker.begin_step(_metadata(0, [], {"request": []}).metadata) + assert "request" in worker._requests + + for i, pending in enumerate(pending_outputs): + output = worker.process_output(pending) + np.testing.assert_array_equal(output["request"].rows, rows) + if i + 1 < num_pending: + assert "request" in worker._requests + assert worker._store._references + assert "request" not in worker._requests + # The finished request's store keys must be released after commit. + assert not worker._store._references + worker.close() + + +def test_pending_outputs_bounded_by_max_concurrent_batches(): + worker = _make_worker(1, max_concurrent_batches=2) + worker._capturer = Mock() + worker._capturer.snapshot_routing_data.side_effect = lambda num_rows: torch.zeros( + (num_rows, *_SHAPE), dtype=torch.uint8 + ) + metadata = _metadata(0, [_request_metadata("request", 0, 1, 0, [])], {}).metadata + batch = _input_batch(["request"], np.array([0]), np.array([0, 1])) + for _ in range(2): + worker.begin_step(metadata) + assert worker.prepare_output(batch) is not None + + worker.begin_step(metadata) + with pytest.raises(AssertionError, match="not consumed in order"): + worker.prepare_output(batch) + + worker.close() + + +def test_worker_rejects_invalid_rejected_token_count(): + worker = _make_worker(1) + metadata = _metadata( + 0, + [_request_metadata("request", 0, 1, 0, [])], + {}, + ) + with pytest.raises(AssertionError, match="rejected-token count is invalid"): + _process_output( + worker, + metadata, + np.zeros((1, *_SHAPE), dtype=_DTYPE), + ["request"], + np.array([-1]), + token_starts=(0,), + num_tokens=(1,), + ) + assert worker._pending_outputs == [] + + worker.close() + + +def _process_output( + worker, + step, + rows, + request_ids, + num_rejected, + num_sampled=None, + *, + token_starts=None, + num_tokens=None, + query_start_loc=None, +): + if num_sampled is None: + num_sampled = np.ones(len(request_ids), dtype=np.int32) + if token_starts is None or num_tokens is None: + assert isinstance(step, _WorkerStep) + token_starts, num_tokens = _execution_ranges(step, request_ids) + worker._capturer = Mock() + worker._capturer.snapshot_routing_data.return_value = torch.from_numpy(rows) + metadata = step.metadata if isinstance(step, _WorkerStep) else step + worker.begin_step(metadata) + if query_start_loc is None: + query_start_loc = np.concatenate( + (np.zeros(1, dtype=np.int32), np.cumsum(num_tokens, dtype=np.int32)) + ) + pending = worker.prepare_output( + _input_batch(request_ids, np.asarray(token_starts), query_start_loc) + ) + assert pending is not None + pending.enqueue_cpu_copy(num_sampled, num_rejected) + return worker.process_output(pending) + + +def test_worker_ignores_cudagraph_query_padding(): + worker = _make_worker(2) + logical = np.arange(2 * 3 * 2, dtype=np.uint8).reshape(2, 3, 2) + metadata = _metadata( + 0, + [ + _request_metadata("first", 0, 1, 0, []), + _request_metadata("second", 0, 1, 0, []), + ], + {}, + ) + + output = _process_output( + worker, + metadata, + logical, + ["first", "second"], + np.zeros(2, dtype=np.int32), + query_start_loc=np.array([0, 1, 2, 2], dtype=np.int32), + ) + + np.testing.assert_array_equal(output["first"].rows, logical[:1]) + np.testing.assert_array_equal(output["second"].rows, logical[1:]) + worker.close() + + +def test_worker_rejects_scheduler_emit_cursor_ahead(): + worker = _make_worker(1) + _begin_step( + worker, + _metadata( + 0, + [_request_metadata("request", 0, 1, 0, [])], + {}, + ), + ) + + with pytest.raises(AssertionError, match="Scheduler emit cursor moved ahead"): + _begin_step( + worker, + _metadata( + 0, + [_request_metadata("request", 1, 1, 1, [])], + {}, + ), + ) + + worker.close() + + +@pytest.mark.parametrize( + "routing", + [ + np.zeros((1, _SHAPE[0], _SHAPE[1] + 1), dtype=_DTYPE), + np.zeros((1, *_SHAPE), dtype=np.int32), + ], + ids=["shape", "dtype"], +) +def test_worker_rejects_mismatched_capture_profile(routing): + worker = _make_worker(1) + metadata = _metadata( + 0, + [_request_metadata("request", 0, 1, 0, [])], + {}, + ) + + with pytest.raises(AssertionError, match="capture profile changed"): + _process_output( + worker, + metadata, + routing, + ["request"], + np.array([0]), + ) + worker.close() + + +@pytest.mark.parametrize( + ("initial_start", "initial_rows", "invalid_start"), + [ + (4, [4, 5, 6], 6), + (0, [0], 2), + (None, [], 3), + ], + ids=["overlap", "gap", "unaligned-initial-capture"], +) +def test_logical_buffer_rejects_noncontiguous_capture( + initial_start, initial_rows, invalid_start +): + buffer = RoutedExpertsBuffer(np.dtype("uint8"), (1,), 4, 1, 4, 1) + if initial_start is not None: + rows = np.asarray(initial_rows, dtype=np.uint8).reshape(-1, 1) + assert buffer.capture("request", initial_start, rows) == [] + + with pytest.raises(AssertionError, match="not contiguous"): + buffer.capture( + "request", invalid_start, np.array([[invalid_start]], dtype=np.uint8) + ) + + +def test_logical_buffer_captures_one_row_per_decode_step(): + buffer = RoutedExpertsBuffer(_DTYPE, _SHAPE, _BLOCK_SIZE, 1, 8, 2) + logical = np.arange(_BLOCK_SIZE * 3 * 2, dtype=_DTYPE).reshape(_BLOCK_SIZE, 3, 2) + + completed = [] + for step in range(_BLOCK_SIZE): + completed += buffer.capture("request", step, logical[step : step + 1]) + + assert len(completed) == 1 + np.testing.assert_array_equal(completed[0][1], logical) + + +def test_logical_buffer_copies_borrowed_block_when_retained(): + buffer = RoutedExpertsBuffer(_DTYPE, _SHAPE, _BLOCK_SIZE, 1, 4, 2) + logical = np.arange(_BLOCK_SIZE * 3 * 2, dtype=_DTYPE).reshape(_BLOCK_SIZE, *_SHAPE) + + expected = logical.copy() + completed = buffer.capture("request", 0, logical) + retained = buffer.retain_block(completed[0][1]) + logical[:] = 0 + + assert not np.shares_memory(retained, logical) + np.testing.assert_array_equal(retained, expected) + buffer.release_block(retained) + + +def _make_store( + *, + max_bytes: int = 1 << 20, + object_nbytes: int = 4, +): + return BlockObjectStore( + max_bytes=max_bytes, + object_nbytes=object_nbytes, + ) + + +def test_publish_routed_experts_publishes_full_blocks(): + store = _make_store(object_nbytes=_BLOCK_SIZE * int(np.prod(_SHAPE))) + buffer = RoutedExpertsBuffer(_DTYPE, _SHAPE, _BLOCK_SIZE, 1, 8, 2) + logical = np.arange(8 * 3 * 2, dtype=np.uint8).reshape(8, 3, 2) + hashes = [b"a" * 32, b"b" * 32] + keys = routed_experts_keys(hashes, "0") + blocks = buffer.capture("request", 0, logical) + publish_routed_experts( + store, + batches=[(keys, blocks)], + block_size=_BLOCK_SIZE, + ) + expected = logical.copy() + logical[:] = 0 + + np.testing.assert_array_equal( + materialize_routed_experts( + store, + keys, + shape_per_token=_SHAPE, + dtype=_DTYPE, + ), + expected, + ) + store.close() + + +def test_worker_data_plane_publishes_blocks_and_reuses_prefix(): + worker = _make_worker(2) + + hashes = [b"a" * 32, b"b" * 32, b"c" * 32] + logical = np.arange(10 * 3 * 2, dtype=np.uint8).reshape(10, 3, 2) + first = _metadata( + generation=0, + requests=[_request_metadata("first", 0, 8, 0, hashes)], + finished_requests={}, + ) + output = _process_output(worker, first, logical[:8], ["first"], np.array([0])) + assert output is not None + np.testing.assert_array_equal(output["first"].rows, logical[:8]) + + second = _metadata( + generation=0, + requests=[_request_metadata("second", 8, 2, 0, hashes)], + finished_requests={"first": []}, + ) + output = _process_output(worker, second, logical[8:], ["second"], np.array([0])) + assert output is not None + np.testing.assert_array_equal(output["second"].rows, logical) + worker.close() + + +def test_worker_matches_kv_eviction_order_after_requests_finish(): + worker = _make_worker(2, max_store_blocks=3) + hashes = [bytes([value]) * 32 for value in range(5)] + blocks = [ + np.full((_BLOCK_SIZE, *_SHAPE), value, dtype=_DTYPE) for value in range(5) + ] + + def publish_and_finish(request_id, block_indexes): + rows = np.concatenate([blocks[index] for index in block_indexes]) + step = _metadata( + 0, + [ + _request_metadata( + request_id, + 0, + len(rows), + 0, + [hashes[index] for index in block_indexes], + ) + ], + {}, + ) + _process_output(worker, step, rows, [request_id], np.array([0])) + _begin_step(worker, _metadata(0, [], {request_id: []})) + + publish_and_finish("s", [0]) + publish_and_finish("ab", [1, 2]) + publish_and_finish("c", [3]) + publish_and_finish("d", [4]) + + assert worker._store is not None + np.testing.assert_array_equal( + materialize_routed_experts( + worker._store, + routed_experts_keys([hashes[1], hashes[3], hashes[4]], "0"), + shape_per_token=_SHAPE, + dtype=_DTYPE, + ), + np.concatenate([blocks[1], blocks[3], blocks[4]]), + ) + with pytest.raises(BlockObjectStoreError, match="does not exist"): + materialize_routed_experts( + worker._store, + routed_experts_keys([hashes[2]], "0"), + shape_per_token=_SHAPE, + dtype=_DTYPE, + ) + worker.close() + + +def test_worker_rejects_unbacked_capture_gap(): + worker = _make_worker(1) + logical = np.arange(4 * 3 * 2, dtype=np.uint8).reshape(4, 3, 2) + first = _metadata( + 0, + [_request_metadata("request", 0, 1, 0, [])], + {}, + ) + _process_output( + worker, + first, + logical[:1], + ["request"], + np.array([0]), + np.zeros(1, dtype=np.int32), + ) + second = _metadata( + 0, + [_request_metadata("request", 2, 1, 0, [])], + {}, + ) + + with pytest.raises(AssertionError, match="unbacked token gap"): + _process_output( + worker, + second, + logical[2:3], + ["request"], + np.array([0]), + np.zeros(1, dtype=np.int32), + ) + worker.close() + + +def test_worker_replays_rejected_speculative_gap(): + worker = _make_worker(1) + logical = np.arange(5 * 3 * 2, dtype=np.uint8).reshape(5, 3, 2) + first = _metadata( + 0, + [_request_metadata("request", 0, 5, 0, [])], + {}, + ) + first_output = _process_output( + worker, first, logical[:5], ["request"], np.array([4]) + ) + + queued = _metadata( + 0, + [_request_metadata("request", 5, 4, 0, [])], + {}, + ) + queued_output = _process_output( + worker, + queued, + logical[1:], + ["request"], + np.array([3]), + ) + + rolled_back = _metadata( + 0, + [_request_metadata("request", 6, 3, 0, [b"a" * 32])], + {}, + ) + + rolled_back_output = _process_output( + worker, + rolled_back, + logical[2:], + ["request"], + np.array([0]), + ) + + np.testing.assert_array_equal(first_output["request"].rows, logical[:1]) + assert queued_output["request"].token_start == 1 + np.testing.assert_array_equal( + queued_output["request"].rows, + logical[1:2], + ) + np.testing.assert_array_equal( + rolled_back_output["request"].rows, + logical[2:], + ) + assert worker._requests["request"].capture_cursor == len(logical) + worker.close() + + +def test_worker_rejects_overlapping_capture_rows(): + worker = _make_worker(1) + logical = np.arange(4 * 3 * 2, dtype=np.uint8).reshape(4, 3, 2) + first = _metadata( + 0, + [_request_metadata("request", 0, 3, 0, [])], + {}, + ) + _process_output( + worker, + first, + logical[:3], + ["request"], + np.array([0]), + np.zeros(1, dtype=np.int32), + ) + stale_overlap = logical[2:].copy() + stale_overlap[0] += 100 + second = _metadata( + 0, + [_request_metadata("request", 2, 2, 0, [b"a" * 32])], + {}, + ) + with pytest.raises(AssertionError, match="capture moved backwards"): + _process_output( + worker, + second, + stale_overlap, + ["request"], + np.array([0]), + ) + worker.close() + + +def test_worker_publishes_entire_batch_before_materializing_prefix(): + worker = _make_worker(2) + hashes = [b"a" * 32, b"b" * 32] + logical = np.arange(8 * 3 * 2, dtype=np.uint8).reshape(8, 3, 2) + metadata = _metadata( + generation=0, + requests=[ + _request_metadata("consumer", 4, 4, 0, hashes), + _request_metadata("producer", 0, 4, 0, hashes), + ], + finished_requests={}, + ) + + output = _process_output( + worker, + metadata, + np.concatenate((logical[4:], logical[:4])), + ["consumer", "producer"], + np.array([0, 0]), + ) + + assert output is not None + np.testing.assert_array_equal(output["consumer"].rows, logical) + np.testing.assert_array_equal(output["producer"].rows, logical[:4]) + worker.close() + + +def test_worker_defers_full_block_until_kv_hash_arrives(): + worker = _make_worker(1) + logical = np.arange(5 * 3 * 2, dtype=np.uint8).reshape(5, 3, 2) + first = _metadata( + 0, + [_request_metadata("request", 0, 4, 0, [])], + {}, + ) + + output = _process_output( + worker, + first, + logical[:4], + ["request"], + np.array([0]), + ) + + assert output is not None + np.testing.assert_array_equal(output["request"].rows, logical[:4]) + assert worker._requests["request"].pending_blocks[0][0] == 0 + + block_hash = b"a" * 32 + second = _metadata( + 0, + [_request_metadata("request", 4, 1, 4, [block_hash])], + {}, + ) + output = _process_output(worker, second, logical[4:], ["request"], np.array([0])) + + assert output is not None + np.testing.assert_array_equal(output["request"].rows, logical[4:]) + assert not worker._requests["request"].pending_blocks + assert worker._store is not None + np.testing.assert_array_equal( + materialize_routed_experts( + worker._store, + routed_experts_keys([block_hash], "0"), + shape_per_token=_SHAPE, + dtype=_DTYPE, + ), + logical[:4], + ) + worker.close() + + +def test_worker_releases_newly_keyed_blocks_before_capture(): + worker = _make_worker(1, max_num_batched_tokens=16) + connector = _make_connector() + block_hashes = [b"a" * 32] + request = _scheduler_request("request", block_hashes, num_tokens=100) + logical = np.arange(52 * 3 * 2, dtype=np.uint8).reshape(52, 3, 2) + + for token_start, num_tokens in [(0, 4), (4, 16), (20, 16)]: + request.num_computed_tokens = token_start + metadata = connector.build_connector_meta( + _step_output([request.request_id], [token_start], [num_tokens]), + {request.request_id: request}, + ) + _process_output( + worker, + metadata, + logical[token_start : token_start + num_tokens], + [request.request_id], + np.array([0]), + token_starts=(token_start,), + num_tokens=(num_tokens,), + ) + + block_hashes.extend(bytes([value]) * 32 for value in range(1, 5)) + request.num_computed_tokens = 36 + metadata = connector.build_connector_meta( + _step_output([request.request_id], [36], [16]), + {request.request_id: request}, + ) + _process_output( + worker, + metadata, + logical[36:52], + [request.request_id], + np.array([0]), + token_starts=(36,), + num_tokens=(16,), + ) + + state = worker._requests[request.request_id] + assert [start for start, _ in state.pending_blocks] == [ + 20, + 24, + 28, + 32, + 36, + 40, + 44, + 48, + ] + worker.close() + + +def test_worker_publishes_pending_block_without_forward(): + worker = _make_worker(1) + _begin_step( + worker, + _metadata( + 0, + [_request_metadata("request", 0, 1, 0, [])], + {}, + ), + ) + logical = np.arange(_BLOCK_SIZE * 3 * 2, dtype=np.uint8).reshape(_BLOCK_SIZE, 3, 2) + assert worker._buffer is not None + worker._requests["request"].pending_blocks = [ + (0, worker._buffer.retain_block(logical)) + ] + block_hash = b"a" * 32 + + _begin_step(worker, _metadata(0, [], {}, {"request": [block_hash]})) + + assert not worker._requests["request"].pending_blocks + assert worker._store is not None + np.testing.assert_array_equal( + materialize_routed_experts( + worker._store, + routed_experts_keys([block_hash], "0"), + shape_per_token=_SHAPE, + dtype=_DTYPE, + ), + logical, + ) + worker.close() + + +def test_worker_releases_unscheduled_request_blocks_before_capture(): + worker = _make_worker(2, 20, max_concurrent_batches=1) + logical = np.arange(48 * 3 * 2, dtype=np.uint8).reshape(48, 3, 2) + prompt_hashes = [b"a" * 32, b"b" * 32] + prompt = _metadata( + 0, + [ + _request_metadata("first", 0, 4, 0, prompt_hashes[:1]), + _request_metadata("second", 0, 4, 0, prompt_hashes[1:]), + ], + {}, + ) + _process_output( + worker, + prompt, + logical[:8], + ["first", "second"], + np.zeros(2), + np.zeros(2, dtype=np.int32), + ) + + first_decode = _metadata( + 0, + [_request_metadata("first", 4, 20, 0, [])], + {}, + ) + _process_output( + worker, + first_decode, + logical[8:28], + ["first"], + np.zeros(1), + np.zeros(1, dtype=np.int32), + ) + assert len(worker._requests["first"].pending_blocks) == 5 + + first_hashes = [bytes([value]) * 32 for value in range(2, 7)] + second_decode = _metadata( + 0, + [_request_metadata("second", 4, 20, 0, [])], + {}, + {"first": first_hashes}, + ) + _process_output( + worker, + second_decode, + logical[28:48], + ["second"], + np.zeros(1), + np.zeros(1, dtype=np.int32), + ) + + assert not worker._requests["first"].pending_blocks + assert len(worker._requests["second"].pending_blocks) == 5 + worker.close() + + +def test_worker_does_not_rematerialize_emitted_rows(): + worker = _make_worker(1) + hashes = [b"a" * 32, b"b" * 32] + logical = np.arange(8 * 3 * 2, dtype=np.uint8).reshape(8, 3, 2) + + first = _metadata( + 0, + [_request_metadata("request", 0, 4, 0, hashes)], + {}, + ) + output = _process_output(worker, first, logical[:4], ["request"], np.array([0])) + assert output is not None + + assert worker._store is not None + worker._store.get_concatenated = Mock(wraps=worker._store.get_concatenated) + second = _metadata( + 0, + # Async scheduling may build this before the scheduler consumes first. + [_request_metadata("request", 4, 4, 0, [])], + {}, + ) + output = _process_output(worker, second, logical[4:], ["request"], np.array([0])) + + assert output is not None + np.testing.assert_array_equal(output["request"].rows, logical[4:]) + worker._store.get_concatenated.assert_not_called() + worker.close() + + +def test_worker_resumes_from_scheduler_emit_boundary(): + worker = _make_worker(1) + logical = np.arange(5 * 3 * 2, dtype=np.uint8).reshape(5, 3, 2) + resumed = _metadata( + 0, + [_request_metadata("request", 4, 1, 4, [b"a" * 32])], + {}, + ) + + output = _process_output(worker, resumed, logical[4:], ["request"], np.array([0])) + + assert output["request"].token_start == 4 + np.testing.assert_array_equal(output["request"].rows, logical[4:]) + assert worker._requests["request"].emit_cursor == 5 + worker.close() + + +def test_worker_emits_mid_block_chunked_prefill(): + worker = _make_worker(1) + logical = np.arange(3 * 3 * 2, dtype=np.uint8).reshape(3, 3, 2) + + first = _metadata( + 0, + [_request_metadata("request", 0, 2, 0, [])], + {}, + ) + output = _process_output( + worker, + first, + logical[:2], + ["request"], + np.array([0]), + np.zeros(1, dtype=np.int32), + ) + assert output is not None and not output + + second = _metadata( + 0, + [_request_metadata("request", 2, 1, 0, [])], + {}, + ) + output = _process_output(worker, second, logical[2:], ["request"], np.array([0])) + + np.testing.assert_array_equal(output["request"].rows, logical) + worker.close() + + +def test_worker_emits_published_block_and_mid_block_tail(): + worker = _make_worker(1) + logical = np.arange(7 * 3 * 2, dtype=np.uint8).reshape(7, 3, 2) + + first = _metadata( + 0, + [_request_metadata("request", 0, 6, 0, [b"a" * 32])], + {}, + ) + output = _process_output( + worker, + first, + logical[:6], + ["request"], + np.array([0]), + np.zeros(1, dtype=np.int32), + ) + assert output is not None and not output + + second = _metadata( + 0, + [_request_metadata("request", 6, 1, 0, [])], + {}, + ) + output = _process_output(worker, second, logical[6:], ["request"], np.array([0])) + + np.testing.assert_array_equal(output["request"].rows, logical) + worker.close() + + +def test_worker_output_does_not_alias_released_tail_buffer(): + worker = _make_worker(1) + logical = np.arange(4 * 3 * 2, dtype=np.uint8).reshape(4, 3, 2) + first = _metadata( + 0, + [_request_metadata("request", 0, 2, 0, [b"a" * 32])], + {}, + ) + _process_output( + worker, + first, + logical[:2], + ["request"], + np.array([0]), + np.zeros(1, dtype=np.int32), + ) + second = _metadata( + 0, + [_request_metadata("request", 2, 2, 0, [])], + {}, + ) + + output = _process_output(worker, second, logical[2:], ["request"], np.array([0])) + emitted = output["request"].rows + expected = logical.copy() + + replacement = np.full_like(logical, 99) + reuse = _metadata( + 0, + [_request_metadata("reuse", 0, 4, 0, [b"b" * 32])], + {}, + ) + _process_output( + worker, + reuse, + replacement, + ["reuse"], + np.array([0]), + np.zeros(1, dtype=np.int32), + ) + + np.testing.assert_array_equal(emitted, expected) + worker.close() + + +def test_worker_merges_block_hash_deltas(): + worker = _make_worker(1) + _begin_step( + worker, + _metadata( + 0, + [_request_metadata("request", 0, 1, 0, [b"a" * 32, b"b" * 32])], + {}, + ), + ) + _begin_step( + worker, + _metadata( + 0, + [ + _request_metadata( + "request", + 1, + 1, + 0, + [b"c" * 32], + ) + ], + {}, + ), + ) + + assert worker._requests["request"].aux_output_keys == [ + "vllm-artifact/0/" + (b"a" * 32).hex(), + "vllm-artifact/0/" + (b"b" * 32).hex(), + "vllm-artifact/0/" + (b"c" * 32).hex(), + ] + worker.close() + + +def test_worker_finish_publishes_keyed_block_and_discards_request_state(): + worker = _make_worker(1) + logical = np.arange(2 * _BLOCK_SIZE * 3 * 2, dtype=np.uint8).reshape( + 2 * _BLOCK_SIZE, 3, 2 + ) + running = _metadata( + 0, + [_request_metadata("request", 0, len(logical), 0, [])], + {}, + ) + _process_output( + worker, + running, + logical, + ["request"], + np.array([0]), + np.array([0]), + ) + block_hash = b"a" * 32 + + _begin_step( + worker, + _metadata( + 0, + [], + {"request": [block_hash]}, + ), + ) + + assert "request" not in worker._requests + assert worker._buffer is not None + assert "request" not in worker._buffer._requests + assert worker._store is not None + np.testing.assert_array_equal( + materialize_routed_experts( + worker._store, + routed_experts_keys([block_hash], "0"), + shape_per_token=_SHAPE, + dtype=_DTYPE, + ), + logical[:_BLOCK_SIZE], + ) + worker.close() + + +def test_worker_fails_when_cached_aux_output_is_missing(): + worker = _make_worker(1) + worker._generation = 0 + metadata = _metadata( + 0, + [ + _request_metadata( + "request", + 8, + 1, + 0, + [b"a" * 32, b"b" * 32], + ) + ], + {}, + ) + + with pytest.raises(BlockObjectStoreError, match="does not exist"): + _process_output( + worker, + metadata, + np.zeros((1, *_SHAPE), dtype=_DTYPE), + ["request"], + np.array([0]), + ) + worker.close() + + +def test_store_rejects_oversized_batch_without_partial_write(): + store = _make_store(max_bytes=6, object_nbytes=3) + store.put([BlockObject("retained", b"rrr")]) + + with pytest.raises(BlockObjectStoreError, match="cannot retain"): + store.put( + [ + BlockObject("first", b"111"), + BlockObject("second", b"222"), + BlockObject("third", b"333"), + ] + ) + + assert store.get_concatenated(["retained"]) == b"rrr" + with pytest.raises(BlockObjectStoreError, match="does not exist"): + store.get_concatenated(["first"]) + store.close() + + +def test_store_lru_and_immutable_put(): + store = _make_store(max_bytes=8) + store.put([BlockObject("first", b"1111"), BlockObject("second", b"2222")]) + assert store.get_concatenated(["first"]) == b"1111" + + store.put([BlockObject("first", b"1111"), BlockObject("third", b"3333")]) + + assert store.get_concatenated(["first", "third"]) == b"11113333" + with pytest.raises(BlockObjectStoreError, match="Increase aux_output_config"): + store.get_concatenated(["second"]) + store.close() + + +def test_store_does_not_evict_referenced_aux_outputs(): + store = _make_store(max_bytes=8) + store.put([BlockObject("first", b"1111"), BlockObject("second", b"2222")]) + store.put([], retain_keys=["first"]) + store.put([], retain_keys=["first"], release_keys=["first"]) + + store.put([BlockObject("third", b"3333")]) + + assert store.get_concatenated(["first", "third"]) == b"11113333" + with pytest.raises(BlockObjectStoreError, match="does not exist"): + store.get_concatenated(["second"]) + store.put([], release_keys=["first"]) + store.close() + + +def test_store_terminal_put_preserves_tail_first_eviction_order(): + store = _make_store(max_bytes=8) + keys = ["first", "second", "third"] + store.put( + [BlockObject("first", b"1111"), BlockObject("second", b"2222")], + retain_keys=keys, + ) + + store.put( + [BlockObject("third", b"3333")], + release_keys=reversed(keys), + ) + + assert store.get_concatenated(keys[:2]) == b"11112222" + with pytest.raises(BlockObjectStoreError, match="does not exist"): + store.get_concatenated(["third"]) + store.close() + + +def test_store_terminal_put_orders_survivors_for_later_eviction(): + store = _make_store(max_bytes=12) + keys = ["first", "second", "third"] + store.put( + [BlockObject("first", b"1111"), BlockObject("second", b"2222")], + retain_keys=keys, + ) + + store.put( + [BlockObject("third", b"3333")], + release_keys=reversed(keys), + ) + store.put([BlockObject("fourth", b"4444")]) + + assert store.get_concatenated(["first", "second", "fourth"]) == b"111122224444" + with pytest.raises(BlockObjectStoreError, match="does not exist"): + store.get_concatenated(["third"]) + store.close() + + +def test_store_terminal_put_keeps_a_shared_incoming_key(): + store = _make_store(max_bytes=8) + keys = ["first", "second", "third"] + store.put( + [BlockObject("first", b"1111"), BlockObject("second", b"2222")], + retain_keys=(*keys, "third"), + ) + + store.put( + [BlockObject("third", b"3333")], + release_keys=reversed(keys), + ) + + assert store.get_concatenated(["first", "third"]) == b"11113333" + with pytest.raises(BlockObjectStoreError, match="does not exist"): + store.get_concatenated(["second"]) + store.put([], release_keys=["third"]) + store.close() + + +def test_store_rejects_access_after_close(): + store = _make_store(max_bytes=8) + store.close() + + with pytest.raises(RuntimeError, match="closed"): + store.put([BlockObject("first", b"1111")]) + with pytest.raises(RuntimeError, match="closed"): + store.get_concatenated(["first"]) + + +def test_store_reuses_evicted_slot_without_moving_live_objects(): + store = _make_store(max_bytes=12) + store.put( + [ + BlockObject("first", b"1111"), + BlockObject("second", b"2222"), + BlockObject("third", b"3333"), + ] + ) + assert store.get_concatenated(["second"]) == b"2222" + store.put([BlockObject("fourth", b"4444")]) + + assert store.get_concatenated(["second", "third", "fourth"]) == b"222233334444" + store.close() + + +def _scheduler_request( + request_id: str, + block_hashes: list[bytes], + *, + num_tokens: int = 10, + num_output_tokens: int = 1, + prompt_start: int = 0, +): + return _SchedulerRequest( + request_id=request_id, + block_hashes=block_hashes, + num_tokens=num_tokens, + num_output_tokens=num_output_tokens, + num_computed_tokens=num_tokens, + sampling_params=SimpleNamespace(routed_experts_prompt_start=prompt_start), + ) + + +def _step_output( + request_ids: list[str], token_starts: list[int], token_counts: list[int] +) -> SchedulerOutput: + output = SchedulerOutput.make_empty() + output.scheduled_cached_reqs = CachedRequestData( + req_ids=request_ids, + resumed_req_ids=set(), + new_token_ids=[[] for _ in request_ids], + all_token_ids={}, + new_block_ids=[None for _ in request_ids], + num_computed_tokens=token_starts, + num_output_tokens=[0 for _ in request_ids], + ) + output.num_scheduled_tokens = dict(zip(request_ids, token_counts, strict=True)) + output.total_num_scheduled_tokens = sum(token_counts) + return output + + +def test_scheduler_connector_builds_worker_metadata_and_forwards_output(): + connector = _make_connector() + request = _scheduler_request("request", [b"a" * 32], num_tokens=5) + request.num_computed_tokens = 0 + scheduler_output = _step_output([request.request_id], [0], [4]) + + metadata = connector.build_connector_meta( + scheduler_output, {request.request_id: request} + ) + + assert metadata.generation == 0 + assert metadata.requests == {request.request_id: 4} + assert list(metadata.block_hashes[request.request_id]) == [b"a" * 32] + + routing = np.arange(4 * 3 * 2, dtype=np.uint8).reshape(4, 3, 2) + output = {"request": AuxRequestOutput(0, routing)} + request.num_computed_tokens = 4 + np.testing.assert_array_equal(connector.take_output(request, output), routing) + + +def test_scheduler_starts_worker_output_at_requested_prompt_token(): + connector = _make_connector() + request = _scheduler_request( + "request", + [b"a" * 32], + num_tokens=4, + num_output_tokens=0, + prompt_start=2, + ) + + metadata = connector.build_connector_meta( + _step_output([request.request_id], [0], [4]), + {request.request_id: request}, + ) + + assert metadata.requests == {request.request_id: 2} + + +def test_scheduler_connector_preserves_request_finish_order(): + connector = _make_connector() + first = _scheduler_request("first", [b"a" * 32]) + second = _scheduler_request("second", [b"b" * 32]) + connector.build_connector_meta( + _step_output([first.request_id, second.request_id], [0, 0], [0, 0]), + {first.request_id: first, second.request_id: second}, + ) + connector.request_finished(second) + connector.request_finished(first) + + metadata = connector.build_connector_meta(_step_output([], [], []), {}) + + assert metadata.finished_requests == (second.request_id, first.request_id) + assert not metadata.block_hashes + + +def test_worker_preemption_terminal_keeps_inflight_hashes_for_publish(): + worker = _make_worker(1) + block_hashes = [b"a" * 32, b"b" * 32] + logical = np.arange(2 * _BLOCK_SIZE * 3 * 2, dtype=np.uint8).reshape( + 2 * _BLOCK_SIZE, 3, 2 + ) + first = _metadata( + 0, + [ + _request_metadata( + "request", + 0, + _BLOCK_SIZE, + 0, + block_hashes[:1], + ) + ], + {}, + ) + _process_output( + worker, + first, + logical[:_BLOCK_SIZE], + ["request"], + np.array([0]), + ) + inflight = _metadata( + 0, + [ + _request_metadata( + "request", + _BLOCK_SIZE, + _BLOCK_SIZE, + 0, + block_hashes[1:], + ) + ], + {}, + ) + _process_output( + worker, + inflight, + logical[_BLOCK_SIZE:], + ["request"], + np.array([0]), + np.zeros(1, dtype=np.int32), + token_starts=(_BLOCK_SIZE,), + num_tokens=(_BLOCK_SIZE,), + ) + _begin_step( + worker, + _metadata( + 0, + [], + {"request": []}, + ), + ) + + assert worker._store is not None + np.testing.assert_array_equal( + materialize_routed_experts( + worker._store, + routed_experts_keys(block_hashes[:1], "0"), + shape_per_token=_SHAPE, + dtype=_DTYPE, + ), + logical[:_BLOCK_SIZE], + ) + np.testing.assert_array_equal( + materialize_routed_experts( + worker._store, + routed_experts_keys(block_hashes[1:], "0"), + shape_per_token=_SHAPE, + dtype=_DTYPE, + ), + logical[_BLOCK_SIZE:], + ) + worker.close() + + +def test_worker_releases_terminal_pins_before_same_step_publish(): + worker = _make_worker(2, max_store_blocks=1) + victim_hash = b"a" * 32 + producer_hash = b"b" * 32 + victim_rows = np.zeros((_BLOCK_SIZE, *_SHAPE), dtype=_DTYPE) + producer_rows = np.ones((_BLOCK_SIZE, *_SHAPE), dtype=_DTYPE) + running = _metadata( + 0, + [ + _request_metadata("victim", 0, _BLOCK_SIZE, 0, [victim_hash]), + _request_metadata("producer", 0, _BLOCK_SIZE, 0, []), + ], + {}, + ) + _process_output( + worker, + running, + np.concatenate((victim_rows, producer_rows)), + ["victim", "producer"], + np.zeros(2, dtype=np.int32), + np.zeros(2, dtype=np.int32), + ) + + _begin_step( + worker, + _metadata( + 0, + [], + {"victim": []}, + {"producer": [producer_hash]}, + ), + ) + + assert worker._store is not None + np.testing.assert_array_equal( + materialize_routed_experts( + worker._store, + routed_experts_keys([producer_hash], "0"), + shape_per_token=_SHAPE, + dtype=_DTYPE, + ), + producer_rows, + ) + with pytest.raises(BlockObjectStoreError, match="does not exist"): + worker._store.get_concatenated(routed_experts_keys([victim_hash], "0")) + worker.close() + + +def test_terminal_request_publishes_hash_discovered_after_last_schedule(): + connector = _make_connector() + worker = _make_worker(1) + block_hashes: list[bytes] = [] + request = _scheduler_request("request", block_hashes, num_tokens=5) + request.num_computed_tokens = 0 + metadata = connector.build_connector_meta( + _step_output([request.request_id], [0], [4]), + {request.request_id: request}, + ) + logical = np.arange(_BLOCK_SIZE * 3 * 2, dtype=np.uint8).reshape(_BLOCK_SIZE, 3, 2) + + _process_output( + worker, + metadata, + logical, + [request.request_id], + np.array([0]), + token_starts=(0,), + num_tokens=(_BLOCK_SIZE,), + ) + assert worker._requests[request.request_id].pending_blocks + + block_hash = b"a" * 32 + block_hashes.append(block_hash) + request.num_computed_tokens = _BLOCK_SIZE + connector.request_finished(request) + cleanup = connector.build_connector_meta(_step_output([], [], []), {}) + worker.begin_step(cleanup) + + assert worker._store is not None + np.testing.assert_array_equal( + materialize_routed_experts( + worker._store, + routed_experts_keys([block_hash], "0"), + shape_per_token=_SHAPE, + dtype=_DTYPE, + ), + logical, + ) + worker.close() + + +def test_scheduler_connector_recreates_preempted_request_state(): + connector = _make_connector() + request = _scheduler_request("request", [b"a" * 32], num_tokens=5) + connector.build_connector_meta( + _step_output([request.request_id], [0], [4]), + {request.request_id: request}, + ) + connector.request_finished(request) + cleanup = connector.build_connector_meta( + _step_output([], [], []), + {request.request_id: request}, + ) + assert request.request_id in cleanup.finished_requests + assert request.request_id not in cleanup.block_hashes + + resumed = connector.build_connector_meta( + _step_output([request.request_id], [0], [4]), + {request.request_id: request}, + ) + assert list(resumed.block_hashes[request.request_id]) == [b"a" * 32] + + +def test_scheduler_consumes_ordered_stale_aux_outputs(): + connector = _make_connector() + request = _scheduler_request("request", [], num_tokens=6) + connector.build_connector_meta( + _step_output([request.request_id], [0], [4]), + {request.request_id: request}, + ) + first_rows = np.arange(4 * 3 * 2, dtype=np.uint8).reshape(4, 3, 2) + second_rows = np.arange(4 * 3 * 2, dtype=np.uint8).reshape(4, 3, 2) + 40 + first = {"request": AuxRequestOutput(0, first_rows)} + second = {"request": AuxRequestOutput(4, second_rows)} + connector.request_finished(request) + + request.num_tokens = 5 + np.testing.assert_array_equal(connector.take_output(request, first), first_rows) + request.num_tokens = 6 + np.testing.assert_array_equal( + connector.take_output(request, second), second_rows[:1] + ) + + +def test_scheduler_rejects_stale_output_without_aux_outputs(): + connector = _make_connector() + request = _scheduler_request("request", [], num_tokens=1) + connector.build_connector_meta( + _step_output([request.request_id], [0], [1]), + {request.request_id: request}, + ) + + with pytest.raises( + AssertionError, match="auxiliary output worker output is missing" + ): + connector.take_output( + request, + {}, + ) + + +def test_scheduler_rejects_missing_accepted_aux_output_rows(): + connector = _make_connector() + request = _scheduler_request("request", [], num_tokens=2) + connector.build_connector_meta( + _step_output([request.request_id], [0], [1]), + {request.request_id: request}, + ) + output = { + "request": AuxRequestOutput( + 0, + np.empty((0, *_SHAPE), dtype=_DTYPE), + ) + } + + with pytest.raises(AssertionError, match="invalid token range"): + connector.take_output(request, output) + + +def test_scheduler_rejects_aux_output_past_finished_request(): + connector = _make_connector() + request = _scheduler_request("request", [], num_tokens=1) + request.finished = True + connector.build_connector_meta( + _step_output([request.request_id], [0], [1]), + {request.request_id: request}, + ) + output = { + "request": AuxRequestOutput( + 1, + np.empty((0, *_SHAPE), dtype=_DTYPE), + ) + } + + with pytest.raises( + AssertionError, match="finished auxiliary output output has no accepted" + ): + connector.take_output(request, output) + + +@pytest.mark.parametrize("chunk_size", [2, 4]) +@pytest.mark.parametrize("max_tokens", [1, 2]) +def test_prompt_end_offset_through_worker_and_scheduler(chunk_size, max_tokens): + """Omit prompt R3 without losing the empty first output or later decode rows.""" + connector = _make_connector() + worker = _make_worker(1) + request = _scheduler_request( + "request", [b"a" * 32], num_tokens=4, num_output_tokens=0, prompt_start=4 + ) + rows = np.arange(5 * 3 * 2, dtype=_DTYPE).reshape(5, *_SHAPE) + try: + for start in range(0, 4, chunk_size): + metadata = connector.build_connector_meta( + _step_output([request.request_id], [start], [chunk_size]), + {request.request_id: request}, + ) + output = _process_output( + worker, + metadata, + rows[start : start + chunk_size], + [request.request_id], + np.array([0]), + num_sampled=np.array([int(start + chunk_size == 4)]), + token_starts=(start,), + num_tokens=(chunk_size,), + ) + if start + chunk_size < 4: + assert output == {} + request.num_tokens = 5 + request.num_output_tokens = 1 + request.finished = max_tokens == 1 + result = connector.take_output(request, output) + np.testing.assert_array_equal(result, rows[:0]) + + if max_tokens == 2: + metadata = connector.build_connector_meta( + _step_output([request.request_id], [4], [1]), + {request.request_id: request}, + ) + output = _process_output( + worker, + metadata, + rows[4:], + [request.request_id], + np.array([0]), + token_starts=(4,), + num_tokens=(1,), + ) + request.num_tokens = 6 + request.finished = True + np.testing.assert_array_equal( + connector.take_output(request, output), rows[4:] + ) + finally: + worker.close() + + +def test_scheduler_connector_sends_each_block_hash_once(): + connector = _make_connector() + request = _scheduler_request("request", [b"a" * 32, b"b" * 32], num_tokens=12) + scheduler_output = _step_output([request.request_id], [0], [4]) + + first = connector.build_connector_meta( + scheduler_output, {request.request_id: request} + ) + request.block_hashes.append(b"c" * 32) + second = connector.build_connector_meta( + _step_output([request.request_id], [4], [4]), + {request.request_id: request}, + ) + third = connector.build_connector_meta( + _step_output([request.request_id], [8], [4]), + {request.request_id: request}, + ) + + assert list(first.block_hashes[request.request_id]) == [b"a" * 32, b"b" * 32] + assert list(second.block_hashes[request.request_id]) == [b"c" * 32] + assert request.request_id not in third.block_hashes + + +def test_scheduler_connector_sends_unscheduled_hash_update(): + connector = _make_connector() + request = _scheduler_request("request", [b"a" * 32], num_tokens=4) + connector.build_connector_meta( + _step_output([request.request_id], [0], [4]), + {request.request_id: request}, + ) + + request.block_hashes.append(b"b" * 32) + request.num_tokens = 9 + request.num_computed_tokens = 8 + metadata = connector.build_connector_meta( + _step_output([], [], []), + {request.request_id: request}, + ) + + assert not metadata.requests + assert list(metadata.block_hashes[request.request_id]) == [b"b" * 32] + + +def test_scheduler_connector_reset_derives_emit_start_from_request(): + connector = _make_connector() + request = _scheduler_request("request", [b"a" * 32], num_tokens=5) + request.num_computed_tokens = 0 + before_reset = connector.build_connector_meta( + _step_output([request.request_id], [0], [4]), + {request.request_id: request}, + ) + assert list(before_reset.block_hashes[request.request_id]) == [b"a" * 32] + + connector.reset() + empty = connector.build_connector_meta( + _step_output([], [], []), + {request.request_id: request}, + ) + assert not empty.block_hashes + request.num_computed_tokens = 4 + scheduler_output = _step_output([request.request_id], [4], [1]) + metadata = connector.build_connector_meta( + scheduler_output, {request.request_id: request} + ) + + assert metadata.generation == 1 + assert metadata.requests[request.request_id] == 4 + assert list(metadata.block_hashes[request.request_id]) == [b"a" * 32] diff --git a/tests/entrypoints/openai/test_return_routed_experts.py b/tests/entrypoints/openai/test_return_routed_experts.py index a64e80c398..46eeed1795 100644 --- a/tests/entrypoints/openai/test_return_routed_experts.py +++ b/tests/entrypoints/openai/test_return_routed_experts.py @@ -12,15 +12,13 @@ from ...utils import RemoteOpenAIServer MODEL_NAME = "TitanML/tiny-mixtral" # tiny-mixtral config: 8 local experts, top-2 routing, 2 hidden layers. -# The published config has sliding_window=4096, which produces -# SlidingWindowSpec kv-cache groups; RoutedExpertsManager requires a -# FullAttentionSpec group, so we override sliding_window=null below. +# Its published sliding window is incompatible with AuxOutput retention. NUM_LOCAL_EXPERTS = 8 NUM_EXPERTS_PER_TOK = 2 NUM_HIDDEN_LAYERS = 2 -def assert_valid_routed_experts(encoded: str | None) -> None: +def decode_routed_experts(encoded: str | None) -> np.ndarray: assert encoded is not None routed_experts = np.load(io.BytesIO(base64.b64decode(encoded))) assert routed_experts.ndim == 3 @@ -30,6 +28,7 @@ def assert_valid_routed_experts(encoded: str | None) -> None: assert topk == NUM_EXPERTS_PER_TOK assert (routed_experts >= 0).all() assert (routed_experts < NUM_LOCAL_EXPERTS).all() + return routed_experts @pytest.fixture(scope="module") @@ -44,7 +43,11 @@ def server(): "--hf-overrides", '{"sliding_window": null}', ] - with RemoteOpenAIServer(MODEL_NAME, args) as remote_server: + with RemoteOpenAIServer( + MODEL_NAME, + args, + env_dict={"VLLM_USE_V2_MODEL_RUNNER": "1"}, + ) as remote_server: yield remote_server @@ -63,4 +66,8 @@ async def test_routed_experts(server): choice = result.model_dump()["choices"][0] assert choice["token_ids"] is not None - assert_valid_routed_experts(choice["routed_experts"]) + assert result.usage is not None + routed_experts = decode_routed_experts(choice["routed_experts"]) + assert len(routed_experts) == ( + result.usage.prompt_tokens + len(choice["token_ids"]) - 1 + ) diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_return_routed_experts.py b/tests/entrypoints/scale_out/token_in_token_out/test_return_routed_experts.py index e7527a071b..9359006562 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_return_routed_experts.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_return_routed_experts.py @@ -15,9 +15,7 @@ MODEL_NAME = "TitanML/tiny-mixtral" GEN_ENDPOINT = "/inference/v1/generate" # tiny-mixtral config: 8 local experts, top-2 routing, 2 hidden layers. -# The published config has sliding_window=4096, which produces -# SlidingWindowSpec kv-cache groups; RoutedExpertsManager requires a -# FullAttentionSpec group, so we override sliding_window=null below. +# Its published sliding window is incompatible with AuxOutput retention. NUM_LOCAL_EXPERTS = 8 NUM_EXPERTS_PER_TOK = 2 NUM_HIDDEN_LAYERS = 2 @@ -36,7 +34,13 @@ def server(): '{"sliding_window": null}', "--enable-scale-out", ] - with RemoteOpenAIServer(MODEL_NAME, args) as remote_server: + with RemoteOpenAIServer( + MODEL_NAME, + args, + env_dict={ + "VLLM_USE_V2_MODEL_RUNNER": "1", + }, + ) as remote_server: yield remote_server @@ -56,9 +60,10 @@ async def client(server: RemoteOpenAIServer): @pytest.mark.asyncio async def test_generate_routed_experts(client): """Test that /inference/v1/generate returns routed_experts when enabled.""" + prompt_token_ids = [1, 2, 3] payload = { "model": MODEL_NAME, - "token_ids": [1, 2, 3], + "token_ids": prompt_token_ids, "sampling_params": {"max_tokens": 10, "temperature": 0.0}, "stream": False, } @@ -80,3 +85,4 @@ async def test_generate_routed_experts(client): assert topk == NUM_EXPERTS_PER_TOK assert (routed_experts >= 0).all() assert (routed_experts < NUM_LOCAL_EXPERTS).all() + assert num_tokens == len(prompt_token_ids) + len(choice["token_ids"]) - 1 diff --git a/tests/model_executor/test_routed_experts_capture.py b/tests/model_executor/test_routed_experts_capture.py index b146a235c1..ff683ae44c 100644 --- a/tests/model_executor/test_routed_experts_capture.py +++ b/tests/model_executor/test_routed_experts_capture.py @@ -7,26 +7,19 @@ from unittest.mock import Mock, patch import pytest import torch -from vllm.config import ModelConfig, VllmConfig +from vllm.config import VllmConfig from vllm.config.compilation import CompilationMode from vllm.distributed.eplb.eplb_state import EplbLayerState from vllm.model_executor.layers.fused_moe.config import RoutingMethodType from vllm.model_executor.layers.fused_moe.routed_experts_capturer import ( RoutedExpertsCapturer, - RoutedExpertsManager, bind_routed_experts_capturer, - get_routed_experts_attn_gid, ) from vllm.model_executor.layers.fused_moe.router.base_router import BaseRouter -from vllm.transformers_utils.model_arch_config_convertor import ( - ModelArchConfigConvertorBase, -) from vllm.v1.kv_cache_interface import ( FullAttentionSpec, - KVCacheConfig, KVCacheGroupSpec, MLAAttentionSpec, - SlidingWindowMLASpec, UniformTypeKVCacheSpecs, ) @@ -42,6 +35,7 @@ def _capturer_with_buffer( num_experts_per_tok: int = 2, dp_rank: int = 0, tp_size: int = 1, + dtype: torch.dtype = torch.int32, ) -> RoutedExpertsCapturer: # Bypass __init__ so the test can use a CPU buffer and skip the # VllmConfig dependency. The CUDA device-tensor allocation in the @@ -49,10 +43,11 @@ def _capturer_with_buffer( c = RoutedExpertsCapturer.__new__(RoutedExpertsCapturer) c.dp_rank = dp_rank c.tp_size = tp_size + c.output_dtype = torch.uint8 c.device_buffer = torch.full( (max_tokens, num_layers, num_experts_per_tok), -1, - dtype=torch.int32, + dtype=dtype, ) return c @@ -88,74 +83,36 @@ def _make_modular_routed_experts(): ) -def _make_model_config(hf_config): - num_experts_per_token = ModelArchConfigConvertorBase( - hf_config, hf_config - ).get_num_experts_per_token() - model_config = SimpleNamespace( - hf_text_config=hf_config, - model_arch_config=SimpleNamespace( - num_experts_per_token=num_experts_per_token, +def _full_attention_kv_group( + spec_type: type[FullAttentionSpec] = FullAttentionSpec, +) -> KVCacheGroupSpec: + full_attention = spec_type( + block_size=16, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ) + return KVCacheGroupSpec( + ["layer"], + UniformTypeKVCacheSpecs( + block_size=16, + kv_cache_specs={"layer": full_attention}, ), ) - model_config.get_num_experts = lambda: hf_config.num_experts - model_config.get_num_experts_per_tok = lambda: ( - ModelConfig.get_num_experts_per_tok(model_config) - ) - model_config.get_total_num_hidden_layers = lambda: hf_config.num_hidden_layers - return model_config -def test_routed_experts_manager_uses_gemma4_top_k_experts(): - hf_config = SimpleNamespace( - num_experts=8, - top_k_experts=2, - num_hidden_layers=3, - ) - vllm_config = SimpleNamespace(model_config=_make_model_config(hf_config)) - kv_cache_spec = FullAttentionSpec( - block_size=4, - num_kv_heads=1, - head_size=1, - dtype=torch.float32, - ) - kv_cache_config = KVCacheConfig( - num_blocks=2, - kv_cache_tensors=[], - kv_cache_groups=[KVCacheGroupSpec(["layer"], kv_cache_spec)], - ) +@pytest.mark.parametrize("eplb_enabled", [False, True]) +def test_base_router_capture_pre_eplb_mapping(eplb_enabled): + eplb_state = None + if eplb_enabled: + eplb_state = EplbLayerState() + eplb_state.expert_load_view = torch.zeros(32, dtype=torch.int64) + eplb_state.logical_to_physical_map = torch.arange(32).view(32, 1) + eplb_state.logical_replica_count = torch.ones(32, dtype=torch.int64) + eplb_state.should_record_tensor = torch.ones((), dtype=torch.bool) + eplb_state.num_unpadded_tokens_tensors = [torch.tensor(0, dtype=torch.int32)] + router = _make_router(eplb_state) - manager = RoutedExpertsManager(vllm_config, kv_cache_config) - - assert manager.routed_experts_by_slot.shape == (8, 3, 2) - - -def test_routed_experts_manager_uses_kimi_k3_experts_per_token(): - hf_config = SimpleNamespace( - num_experts=8, - num_experts_per_token=2, - num_hidden_layers=3, - ) - vllm_config = SimpleNamespace(model_config=_make_model_config(hf_config)) - kv_cache_spec = FullAttentionSpec( - block_size=4, - num_kv_heads=1, - head_size=1, - dtype=torch.float32, - ) - kv_cache_config = KVCacheConfig( - num_blocks=2, - kv_cache_tensors=[], - kv_cache_groups=[KVCacheGroupSpec(["layer"], kv_cache_spec)], - ) - - manager = RoutedExpertsManager(vllm_config, kv_cache_config) - - assert manager.routed_experts_by_slot.shape == (8, 3, 2) - - -def test_base_router_capture_pre_eplb_mapping(): - router = _make_router() captured = [] def capture_fn(ids): @@ -173,34 +130,7 @@ def test_base_router_capture_pre_eplb_mapping(): assert torch.equal(topk_ids, torch.tensor([[11, 12], [13, 14]])) -def test_base_router_capture_with_eplb_enabled(): - eplb_state = EplbLayerState() - eplb_state.expert_load_view = torch.zeros(32, dtype=torch.int64) - eplb_state.logical_to_physical_map = torch.arange(32).view(32, 1) - eplb_state.logical_replica_count = torch.ones(32, dtype=torch.int64) - eplb_state.should_record_tensor = torch.ones((), dtype=torch.bool) - eplb_state.num_unpadded_tokens_tensors = [torch.tensor(0, dtype=torch.int32)] - router = _make_router(eplb_state=eplb_state) - - captured = [] - - def capture_fn(ids): - captured.append(ids.clone()) - - router.set_capture_fn(capture_fn) - _, topk_ids = router.select_experts( - hidden_states=torch.empty(1), - router_logits=torch.empty(1), - ) - - assert len(captured) == 1 - # Capture should see logical ids pre-EPLB mapping. - assert torch.equal(captured[0], torch.tensor([[1, 2], [3, 4]])) - # Our DummyRouter mapping adds +10. - assert torch.equal(topk_ids, torch.tensor([[11, 12], [13, 14]])) - - -def test_public_binding_only_visits_target_model(monkeypatch): +def test_public_binding_binds_target_model_router(monkeypatch): class DummyFusedMoE: def __init__(self, layer_id): self.layer_id = layer_id @@ -208,7 +138,6 @@ def test_public_binding_only_visits_target_model(monkeypatch): self._quant_method = _make_modular_routed_experts().quant_method target_module = DummyFusedMoE(layer_id=7) - draft_module = DummyFusedMoE(layer_id=0) import vllm.model_executor.layers.fused_moe.layer as fused_moe_layer @@ -221,12 +150,30 @@ def test_public_binding_only_visits_target_model(monkeypatch): ) assert target_module.router.capture_fn is not None - assert draft_module.router.capture_fn is None topk_ids = torch.tensor([[5, 6]]) target_module.router.capture_fn(topk_ids) assert calls == [(7, topk_ids)] +def test_public_binding_supports_direct_capture_source(): + class DirectCaptureSource(torch.nn.Module): + def __init__(self): + super().__init__() + self.layer_id = 3 + self.capture_fn = None + + source = DirectCaptureSource() + calls = [] + capturer = types.SimpleNamespace(capture=lambda *args: calls.append(args)) + + bind_routed_experts_capturer(source, capturer) + + assert source.capture_fn is not None + topk_ids = torch.tensor([[1, 2]]) + source.capture_fn(topk_ids) + assert calls == [(3, topk_ids)] + + def test_public_binding_rejects_monolithic_without_replay_support(monkeypatch): class DummyFusedMoE: def __init__(self): @@ -277,6 +224,51 @@ def test_routed_experts_capturer_single_dp_no_metadata(): assert capturer.device_buffer[3, 0, 0].item() == -1 +@pytest.mark.parametrize( + ("num_experts", "dtype_name", "torch_dtype"), + [(256, "uint8", torch.uint8), (257, "uint16", torch.uint16)], +) +def test_routed_experts_capturer_exposes_output_profile( + monkeypatch, num_experts, dtype_name, torch_dtype +): + import vllm.model_executor.layers.fused_moe.routed_experts_capturer as module + + monkeypatch.setattr(module, "current_platform", SimpleNamespace(device_type="cpu")) + config = SimpleNamespace( + model_config=SimpleNamespace( + get_total_num_hidden_layers=lambda: 3, + get_num_experts=lambda: num_experts, + get_num_experts_per_tok=lambda: 2, + hf_text_config=SimpleNamespace(model_type="test"), + ), + parallel_config=SimpleNamespace(data_parallel_rank=0, tensor_parallel_size=1), + ) + + capturer = RoutedExpertsCapturer(8, config) + + assert capturer.shape_per_token == (3, 2) + assert capturer.output_dtype_name == dtype_name + assert capturer.output_dtype == torch_dtype + assert capturer.device_buffer.shape == (8, 3, 2) + + +@pytest.mark.parametrize("output_dtype", [torch.uint8, torch.uint16]) +def test_routed_experts_capturer_narrows_snapshot(output_dtype): + capturer = _capturer_with_buffer(dtype=torch.int32) + capturer.output_dtype = output_dtype + topk = torch.tensor([[1, 2], [254, 255]], dtype=torch.int64) + ctx = SimpleNamespace(dp_metadata=None) + with patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx): + capturer.capture(layer_id=0, topk_ids=topk) + + output = capturer.snapshot_routing_data(2) + assert capturer.device_buffer.dtype == torch.int32 + assert capturer.device_buffer[:2, 0, :].tolist() == topk.tolist() + assert output.dtype == output_dtype + assert output[:, 0, :].tolist() == topk.tolist() + assert output.data_ptr() != capturer.device_buffer.data_ptr() + + def test_routed_experts_capturer_dp_naive_concatenated_all_ranks(): """N == sum(num_tokens_dp): slice this rank's segment from concatenated topk.""" capturer = _capturer_with_buffer(dp_rank=1) @@ -307,6 +299,47 @@ def test_routed_experts_capturer_dp_modular_local_tokens(): assert torch.equal(capturer.device_buffer[:3, 0, :], topk) +def test_routed_experts_capturer_dp_ep_gathered_shards(): + capturer = _capturer_with_buffer(dp_rank=1, tp_size=2) + num_tokens_dp = torch.tensor([3, 2], dtype=torch.int32) + ctx = SimpleNamespace( + dp_metadata=SimpleNamespace( + num_tokens_across_dp_cpu=num_tokens_dp, + local_sizes=[2, 2, 1, 1], + ) + ) + topk = torch.tensor( + [[0, 1], [2, 3], [4, 5], [-1, -1], [10, 11], [12, 13]], + dtype=torch.int32, + ) + + with patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx): + capturer.capture(layer_id=0, topk_ids=topk) + + assert torch.equal(capturer.device_buffer[:2, 0], topk[4:]) + + +def test_routed_experts_capturer_sp_modular_gathers_tp_shards(): + capturer = _capturer_with_buffer(dp_rank=1, tp_size=2) + num_tokens_dp = torch.tensor([2, 3], dtype=torch.int32) + ctx = SimpleNamespace( + dp_metadata=SimpleNamespace(num_tokens_across_dp_cpu=num_tokens_dp) + ) + local_shard = torch.tensor([[10, 11], [12, 13]], dtype=torch.int32) + gathered = torch.tensor([[10, 11], [12, 13], [14, 15], [-1, -1]], dtype=torch.int32) + tp_group = Mock() + tp_group.all_gather.return_value = gathered + + with ( + patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx), + patch(f"{_REC_MODULE}.get_tp_group", return_value=tp_group), + ): + capturer.capture(layer_id=0, topk_ids=local_shard) + + tp_group.all_gather.assert_called_once_with(local_shard, dim=0) + assert torch.equal(capturer.device_buffer[:3, 0], gathered[:3]) + + def test_routed_experts_capturer_dp_unexpected_batch_raises(): """Mismatch between topk batch dim and DP layout: fail fast.""" capturer = _capturer_with_buffer(dp_rank=0) @@ -324,128 +357,119 @@ def test_routed_experts_capturer_dp_unexpected_batch_raises(): assert capturer.device_buffer[0, 0, 0].item() == -1 -def test_routed_experts_attention_group_is_shared_and_fail_closed(): - """Both sides key routing data by this gid, so it must skip non-full-attention - groups rather than defaulting to 0, and fail closed when none exists.""" - common = dict(num_kv_heads=1, head_size=1, dtype=torch.float32) +def test_get_aux_output_connector_passes_config(monkeypatch): + import vllm.distributed.aux_output_connector.worker as aux_output_worker + + connector = Mock() + constructor = Mock(return_value=connector) + monkeypatch.setattr(aux_output_worker, "AuxOutputWorkerConnector", constructor) config = SimpleNamespace( - kv_cache_groups=[ - KVCacheGroupSpec( - ["swa_layer"], - SlidingWindowMLASpec(block_size=4, sliding_window=8, **common), - ), - KVCacheGroupSpec(["full_layer"], FullAttentionSpec(block_size=4, **common)), - ] + scheduler_config=SimpleNamespace(max_num_batched_tokens=32) ) - assert get_routed_experts_attn_gid(config) == 1 + model = Mock() + kv_cache_config = Mock() - with pytest.raises(ValueError, match="requires a full-attention KV cache group"): - get_routed_experts_attn_gid(SimpleNamespace(kv_cache_groups=[])) + result = aux_output_worker.get_aux_output_connector(model, config, kv_cache_config) - -def test_routed_experts_attention_group_unwraps_uniform_type_specs(): - """DeepSeek-V4-shaped groups wrap their specs in ``UniformTypeKVCacheSpecs``. - - The wrapper is not a ``FullAttentionSpec``, so a bare isinstance check finds - no group and fails closed on every worker. Unwrap semantics themselves are - covered by ``test_is_full_attention_spec_*`` in tests/v1/core. - """ - common = dict(num_kv_heads=1, head_size=1, dtype=torch.float32) - swa_spec = SlidingWindowMLASpec(block_size=4, sliding_window=8, **common) - mla_spec = MLAAttentionSpec(block_size=4, **common) - config = SimpleNamespace( - kv_cache_groups=[ - KVCacheGroupSpec( - ["swa_layer"], - UniformTypeKVCacheSpecs( - block_size=4, kv_cache_specs={"swa_layer": swa_spec} - ), - ), - KVCacheGroupSpec( - ["mla_layer"], - UniformTypeKVCacheSpecs( - block_size=4, kv_cache_specs={"mla_layer": mla_spec} - ), - ), - ] + constructor.assert_called_once_with( + model=model, + kv_cache_config=kv_cache_config, + vllm_config=config, ) - - assert get_routed_experts_attn_gid(config) == 1 - - swa_only = SimpleNamespace(kv_cache_groups=[config.kv_cache_groups[0]]) - with pytest.raises(ValueError, match="requires a full-attention KV cache group"): - get_routed_experts_attn_gid(swa_only) + assert result is connector -@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") -def test_mrv2_async_output_returns_existing_routed_experts_field(): - from vllm.v1.outputs import ModelRunnerOutput, RoutedExpertsTensors - from vllm.v1.worker.gpu.async_utils import AsyncOutput - from vllm.v1.worker.gpu.sample.output import SamplerOutput - - routed_experts = RoutedExpertsTensors( - routing_data=torch.arange(6, dtype=torch.int32, device="cuda").reshape(3, 1, 2), - slot_mapping=torch.tensor([11, 12, 13], device="cuda"), - ) - num_sampled = torch.tensor([1], dtype=torch.int32, device="cuda") - sampler_output = SamplerOutput( - sampled_token_ids=torch.tensor([[1]], device="cuda"), - logprobs_tensors=None, - num_nans=None, - num_sampled=num_sampled, - num_rejected=torch.tensor([0], dtype=torch.int32, device="cuda"), - ) - output = AsyncOutput( - model_runner_output=ModelRunnerOutput(req_ids=["req"], req_id_to_index={}), - sampler_output=sampler_output, - num_sampled_tokens=num_sampled, - main_stream=torch.cuda.current_stream(), - copy_stream=torch.cuda.Stream(), - routed_experts=routed_experts, - ).get_output() - - assert output.routed_experts is not None - assert output.routed_experts.routing_data[:, 0, 0].tolist() == [0, 2, 4] - assert output.routed_experts.slot_mapping.tolist() == [11, 12, 13] - - -@pytest.mark.parametrize("rank", [0, 1]) -def test_all_tp_ranks_initialize_capture(monkeypatch, rank): - pytest.importorskip("vllm.vllm_flash_attn", exc_type=ImportError) - import vllm.v1.worker.gpu.model_runner as model_runner +def test_aux_output_worker_connector_binds_capture_on_non_output_rank(monkeypatch): + import vllm.distributed.aux_output_connector.worker as aux_output_worker capturer = Mock() constructor = Mock(return_value=capturer) bind = Mock() - monkeypatch.setattr(model_runner, "RoutedExpertsCapturer", constructor) - monkeypatch.setattr(model_runner, "bind_routed_experts_capturer", bind) + monkeypatch.setattr(aux_output_worker, "RoutedExpertsCapturer", constructor) + monkeypatch.setattr(aux_output_worker, "bind_routed_experts_capturer", bind) + monkeypatch.setattr( + aux_output_worker, + "get_tp_group", + lambda: SimpleNamespace(is_first_rank=False, world_size=1), + ) - runner = model_runner.GPUModelRunner.__new__(model_runner.GPUModelRunner) - runner.max_num_tokens = 32 - runner.vllm_config = SimpleNamespace(parallel_config=SimpleNamespace(rank=rank)) - runner.kv_cache_config = SimpleNamespace() - runner.model = Mock() - - runner.init_routed_experts_capturer() + config = SimpleNamespace( + aux_output_config=SimpleNamespace(enable_return_routed_experts=True), + kv_transfer_config=None, + max_concurrent_batches=2, + scheduler_config=SimpleNamespace(max_num_batched_tokens=32), + ) + model = Mock() + connector = aux_output_worker.AuxOutputWorkerConnector( + vllm_config=config, + model=model, + kv_cache_config=SimpleNamespace(kv_cache_groups=[_full_attention_kv_group()]), + ) constructor.assert_called_once_with( max_num_batched_tokens=32, - vllm_config=runner.vllm_config, - kv_cache_config=runner.kv_cache_config, + vllm_config=config, ) - bind.assert_called_once_with(runner.model, capturer) - assert runner.routed_experts_capturer is capturer + bind.assert_called_once_with(model, capturer) + connector.begin_step(Mock()) + assert connector.prepare_output(Mock()) is None + capturer.snapshot_routing_data.assert_not_called() + + +def test_aux_output_worker_connector_default_capacity(monkeypatch): + import vllm.distributed.aux_output_connector.worker as aux_output_worker + + tp_group = SimpleNamespace(is_first_rank=True, world_size=1) + store_constructor = Mock() + background_store_constructor = Mock(side_effect=lambda store, **_: store) + capturer = SimpleNamespace(shape_per_token=(2,), output_dtype_name="int32") + monkeypatch.setattr(aux_output_worker, "get_tp_group", lambda: tp_group) + monkeypatch.setattr( + aux_output_worker, "RoutedExpertsCapturer", Mock(return_value=capturer) + ) + monkeypatch.setattr(aux_output_worker, "bind_routed_experts_capturer", Mock()) + monkeypatch.setattr( + aux_output_worker, + "resolve_kv_cache_block_sizes", + lambda *_: (32, 16), + ) + monkeypatch.setattr(aux_output_worker, "BlockObjectStore", store_constructor) + monkeypatch.setattr( + aux_output_worker, "BackgroundBlockObjectStore", background_store_constructor + ) + monkeypatch.setattr(aux_output_worker, "RoutedExpertsBuffer", Mock()) + + config = SimpleNamespace( + aux_output_config=SimpleNamespace(max_bytes=None), + kv_transfer_config=None, + cache_config=SimpleNamespace(enable_prefix_caching=True), + scheduler_config=SimpleNamespace(max_num_seqs=8, max_num_batched_tokens=32), + max_concurrent_batches=2, + ) + kwargs = dict( + vllm_config=config, + model=Mock(), + kv_cache_config=SimpleNamespace( + num_blocks=10, + kv_cache_groups=[_full_attention_kv_group(MLAAttentionSpec)], + ), + ) + + aux_output_worker.AuxOutputWorkerConnector(**kwargs) + assert store_constructor.call_args.kwargs["max_bytes"] == 2560 + assert store_constructor.call_args.kwargs["object_nbytes"] == 128 + assert background_store_constructor.call_args.kwargs["max_pending_batches"] == 16 def test_v2_model_runner_accepts_routed_experts(monkeypatch): monkeypatch.setattr("importlib.metadata.entry_points", lambda **_: ()) config = SimpleNamespace( model_config=SimpleNamespace( - enable_return_routed_experts=True, use_mla=False, logits_processors=None, enable_prompt_embeds=False, ), + aux_output_config=SimpleNamespace(enable_return_routed_experts=True), speculative_config=None, parallel_config=SimpleNamespace( prefill_context_parallel_size=1, diff --git a/tests/v1/core/test_async_scheduler.py b/tests/v1/core/test_async_scheduler.py index 7e27a4517e..27107aa6b0 100644 --- a/tests/v1/core/test_async_scheduler.py +++ b/tests/v1/core/test_async_scheduler.py @@ -354,8 +354,7 @@ def test_abort_request_when_structured_output_fsm_cannot_advance(): scheduler.finished_req_ids_dict = None scheduler.grammar_compile_error_reqs = set() scheduler.vllm_config = Mock() - scheduler.vllm_config.model_config.enable_return_routed_experts = False - scheduler.enable_return_routed_experts = False + scheduler.aux_output_connector = None scheduler.return_sampling_mask = False scheduler.recompute_kv_load_failures = False scheduler.defer_block_free = False diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index 4951014d1b..a58bcab7d5 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -19,6 +19,10 @@ from vllm.config import ( SpeculativeConfig, VllmConfig, ) +from vllm.distributed.aux_output_connector.connector import ( + AuxOutputSchedulerConnector, + AuxRequestOutput, +) from vllm.distributed.ec_transfer.ec_connector.metrics import ECConnectorStats from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import ( HiSparseConnector, @@ -55,7 +59,6 @@ from vllm.v1.outputs import ( ECConnectorOutput, KVConnectorOutput, ModelRunnerOutput, - RoutedExpertsLists, SamplingMaskLists, make_empty_encoder_model_runner_output, ) @@ -136,21 +139,18 @@ def test_add_requests(): assert len(scheduler.waiting) == i + 1 -def test_routed_experts_prompt_start_at_prompt_end(): +@pytest.mark.parametrize("max_tokens", [1, 2]) +def test_routed_experts_prompt_start_at_prompt_end(max_tokens): """A prompt-end offset should return empty routing data, not crash.""" scheduler = create_scheduler() - (request,) = create_requests(num_requests=1) + (request,) = create_requests(num_requests=1, max_tokens=max_tokens) request.sampling_params.routed_experts_prompt_start = request.num_prompt_tokens + scheduler.aux_output_connector = AuxOutputSchedulerConnector() scheduler.add_request(request) scheduler_output = scheduler.schedule() - num_tokens = scheduler_output.num_scheduled_tokens[request.request_id] - - manager = Mock() - manager.routed_experts_by_slot = np.empty((0, 1, 1), dtype=np.uint8) - manager.get.return_value = np.empty((0, 1, 1), dtype=np.uint8) - scheduler.enable_return_routed_experts = True - scheduler.routed_experts_mgr = manager - scheduler._re_block_ids = {request.request_id: []} + assert scheduler_output.aux_output_connector_metadata.requests == { + request.request_id: request.num_prompt_tokens + } outputs = scheduler.update_from_output( scheduler_output, @@ -161,21 +161,23 @@ def test_routed_experts_prompt_start_at_prompt_end(): logprobs=None, prompt_logprobs_dict={}, pooler_output=[], - routed_experts=RoutedExpertsLists( - routing_data=np.zeros((num_tokens, 1, 1), dtype=np.uint8), - slot_mapping=np.arange(num_tokens), - ), + aux_output_connector_output={ + request.request_id: AuxRequestOutput( + request.num_prompt_tokens, + np.empty((0, 1, 1), dtype=np.uint8), + ) + }, ), ) - manager.get.assert_called_once_with( - [], request.num_prompt_tokens, token_start=request.num_prompt_tokens - ) - assert outputs[request.client_index].outputs[0].routed_experts.shape == (0, 1, 1) + output = outputs[request.client_index].outputs[0] + assert output.new_token_ids == [0] + assert output.routed_experts.shape == (0, 1, 1) def test_finish_request(): scheduler = create_scheduler() + scheduler.aux_output_connector = Mock() requests = create_requests(num_requests=10) for request in requests: scheduler.add_request(request) @@ -184,6 +186,7 @@ def test_finish_request(): scheduler.finish_requests(request.request_id, RequestStatus.FINISHED_ABORTED) assert request.request_id not in scheduler.requests assert len(scheduler.waiting) == 9 - i + scheduler.aux_output_connector.request_finished.assert_called_with(request) def test_get_num_unfinished_requests(): @@ -1373,6 +1376,31 @@ def test_preemption_re_records_prefix_cache_query(): assert stats.preempted_requests == 1 +def test_preemption_processes_stale_aux_output(): + scheduler = create_scheduler(enable_prefix_caching=True) + request = create_requests(num_requests=1)[0] + scheduler.add_request(request) + scheduler_output = scheduler.schedule() + scheduler.running.remove(request) + scheduler.aux_output_connector = Mock() + scheduler._preempt_request(request, 0.0) + scheduler.aux_output_connector.request_finished.assert_called_once_with(request) + + scheduler.update_from_output( + scheduler_output, + ModelRunnerOutput( + req_ids=[request.request_id], + req_id_to_index={request.request_id: 0}, + sampled_token_ids=[[1000]], + logprobs=None, + prompt_logprobs_dict={}, + pooler_output=[], + ), + ) + + scheduler.aux_output_connector.take_output.assert_called_once_with(request, None) + + def test_prefix_cache_stats_not_recorded_when_caching_disabled(): """With prefix caching off there is no local lookup, so admitting a request records no phantom miss.""" @@ -1461,6 +1489,7 @@ def test_prefix_cache_stats_counted_once_for_retried_then_scheduled_request(): def test_scheduler_reset_prefix_cache(): scheduler = create_scheduler(enable_prefix_caching=True) + scheduler.aux_output_connector = Mock() requests = create_requests(num_requests=10) for request in requests: scheduler.add_request(request) @@ -1477,10 +1506,20 @@ def test_scheduler_reset_prefix_cache(): # Reset prefix cache should fail since there are still running requests # and they are taking KV cache assert not scheduler.reset_prefix_cache() + scheduler.aux_output_connector.reset.assert_not_called() + + with pytest.raises(RuntimeError, match=r"pause\(mode='keep'\)"): + scheduler.reset_prefix_cache(reset_running_requests=True) + + # pause(mode="keep") also waits for scheduled model outputs to drain. + scheduler.set_pause_state(PauseState.PAUSED_ALL) + with pytest.raises(RuntimeError, match="model output is in flight"): + scheduler.reset_prefix_cache(reset_running_requests=True) + for request in requests: + request.num_in_flight_tokens = 0 - # Reset prefix cache with reset_running_requests=True. All running requests - # Should be pushed back to the waiting queue and kv cache should be freed assert scheduler.reset_prefix_cache(reset_running_requests=True) + scheduler.aux_output_connector.reset.assert_called_once_with() # Verify requests moved from running to waiting assert len(scheduler.waiting) == len(requests) @@ -1490,6 +1529,16 @@ def test_scheduler_reset_prefix_cache(): assert scheduler.waiting[i] == request +@pytest.mark.parametrize("reset_successful", [False, True]) +def test_aux_output_reset_follows_kv_reset_result(reset_successful: bool): + scheduler = create_scheduler(enable_prefix_caching=True) + scheduler.aux_output_connector = Mock() + scheduler.kv_cache_manager.reset_prefix_cache = Mock(return_value=reset_successful) + + assert scheduler.reset_prefix_cache() is reset_successful + assert scheduler.aux_output_connector.reset.call_count == int(reset_successful) + + def test_kv_cache_release_after_keep_pause_preserves_requests(): """A completed keep pause permits release and recomputation of live requests.""" scheduler = create_scheduler(max_num_seqs=1, enable_prefix_caching=True) @@ -3791,10 +3840,9 @@ def test_abort_request_when_structured_output_fsm_cannot_advance(): scheduler.kv_event_publisher = Mock() scheduler.finished_req_ids = set() scheduler.finished_req_ids_dict = None + scheduler.aux_output_connector = None scheduler.grammar_compile_error_reqs = set() scheduler.vllm_config = Mock() - scheduler.vllm_config.model_config.enable_return_routed_experts = False - scheduler.enable_return_routed_experts = False scheduler.return_sampling_mask = False scheduler.recompute_kv_load_failures = False scheduler.defer_block_free = False diff --git a/tests/v1/e2e/general/test_mamba_prefix_cache.py b/tests/v1/e2e/general/test_mamba_prefix_cache.py index e388b73468..5b6826a4b3 100644 --- a/tests/v1/e2e/general/test_mamba_prefix_cache.py +++ b/tests/v1/e2e/general/test_mamba_prefix_cache.py @@ -1122,6 +1122,7 @@ def _run_mamba_prefix_cache_mrv2( logprobs_tensors=None, num_nans=None, num_sampled=num_sampled, + num_rejected=num_rejected, ) return sampler_output, num_sampled, num_rejected diff --git a/tests/v1/engine/test_output_processor.py b/tests/v1/engine/test_output_processor.py index 69128f55b7..485187f222 100644 --- a/tests/v1/engine/test_output_processor.py +++ b/tests/v1/engine/test_output_processor.py @@ -3,8 +3,9 @@ import math import time -from unittest.mock import MagicMock +from unittest.mock import MagicMock, Mock +import numpy as np import pytest from tests.v1.engine.utils import ( @@ -1307,6 +1308,7 @@ def test_lora_request_tracking(log_stats: bool, dummy_test_vectors): async def test_request_output_collector(): NUM_REQS = 3 TEXT = "a" + routed_experts = np.arange(12, dtype=np.uint8).reshape(2, 3, 2) def make_outputs() -> list[RequestOutput]: return [ @@ -1322,6 +1324,9 @@ async def test_request_output_collector(): token_ids=[idx], cumulative_logprob=(idx + 1 * 1.0), logprobs=[{"a": idx, "b": idx}], + routed_experts=( + routed_experts if idx == NUM_REQS - 1 else None + ), finish_reason="length" if (idx == NUM_REQS - 1) else None, ) ], @@ -1374,6 +1379,7 @@ async def test_request_output_collector(): assert output.finished assert output.outputs[0].finish_reason == "length" + np.testing.assert_array_equal(output.outputs[0].routed_experts, routed_experts) # Text, token_ids, and logprobs should get merged. assert output.outputs[0].text == TEXT * num_to_put for tok_0, tok_1 in zip(output.outputs[0].token_ids, list(range(num_to_put))): @@ -1385,6 +1391,46 @@ async def test_request_output_collector(): assert output.outputs[0].cumulative_logprob == cumulative_logprob_expected +def test_routed_experts_are_accumulated_until_finish(): + state = RequestState( + request_id="request-int", + external_req_id="request", + parent_req=None, + request_index=0, + lora_request=None, + output_kind=RequestOutputKind.DELTA, + prompt="prompt", + prompt_token_ids=[1, 2, 3, 4], + prompt_embeds=None, + logprobs_processor=Mock( + logprobs=None, + cumulative_logprob=0.0, + pop_prompt_logprobs=Mock(return_value=None), + ), + detokenizer=Mock(get_next_output_text=Mock(return_value="")), + max_tokens_param=2, + arrival_time=0.0, + queue=None, + log_stats=False, + stream_interval=1, + ) + prompt_chunk = np.arange(24, dtype=np.uint8).reshape(4, 3, 2) + decode_chunk = np.arange(12, dtype=np.uint8).reshape(2, 3, 2) + 24 + state.routed_experts_chunks.append(prompt_chunk) + + partial = state.make_request_output([5], None, None, None) + assert partial is not None + assert partial.outputs[0].routed_experts is None + + state.routed_experts_chunks.append(decode_chunk) + finished = state.make_request_output([6], None, FinishReason.LENGTH, None) + assert finished is not None + np.testing.assert_array_equal( + finished.outputs[0].routed_experts, + np.concatenate((prompt_chunk, decode_chunk)), + ) + + @pytest.mark.asyncio async def test_cumulative_output_collector_n(): """Test collector correctly handles multiple outputs by index.""" diff --git a/tests/v1/executor/test_ray_utils.py b/tests/v1/executor/test_ray_utils.py index 0fa2c129f1..2f945c5a3f 100644 --- a/tests/v1/executor/test_ray_utils.py +++ b/tests/v1/executor/test_ray_utils.py @@ -3,12 +3,14 @@ import numpy as np +from vllm.distributed.aux_output_connector.connector import ( + AuxRequestOutput, +) from vllm.v1.executor.ray_utils import detach_zero_copy_from_model_runner_output from vllm.v1.outputs import ( LogprobsLists, LogprobsTensors, ModelRunnerOutput, - RoutedExpertsLists, ) @@ -59,26 +61,19 @@ def test_detach_zero_copy_from_model_runner_output_copies_only_numpy_views(): assert output.prompt_logprobs_dict["req-0"] is prompt_logprobs -def test_detach_zero_copy_routed_experts_without_logprobs(): +def test_detach_zero_copy_aux_output_without_logprobs(): + rows = _make_readonly(np.arange(12, dtype=np.uint8).reshape(2, 3, 2)) output = ModelRunnerOutput( req_ids=["req-0"], req_id_to_index={"req-0": 0}, - routed_experts=RoutedExpertsLists( - routing_data=_make_readonly(np.arange(12, dtype=np.int32).reshape(2, 3, 2)), - slot_mapping=_make_readonly(np.array([7, 8], dtype=np.int64)), - ), + aux_output_connector_output={"req-0": AuxRequestOutput(0, rows)}, ) - original = output.routed_experts - assert output.logprobs is None detach_zero_copy_from_model_runner_output(output) - detached = output.routed_experts - assert detached is not None - assert detached is not original - assert detached.routing_data is not original.routing_data - assert detached.slot_mapping is not original.slot_mapping - assert detached.routing_data.flags.writeable - assert detached.slot_mapping.flags.writeable - np.testing.assert_array_equal(detached.routing_data, original.routing_data) - np.testing.assert_array_equal(detached.slot_mapping, original.slot_mapping) + aux_output = output.aux_output_connector_output + assert aux_output is not None + detached = aux_output["req-0"].rows + assert detached is not rows + assert detached.flags.writeable + np.testing.assert_array_equal(detached, rows) diff --git a/vllm/config/__init__.py b/vllm/config/__init__.py index ff2ba0493d..940aea7552 100644 --- a/vllm/config/__init__.py +++ b/vllm/config/__init__.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from vllm.config.attention import AttentionConfig, HiSparseConfig +from vllm.config.aux_output import AuxOutputConfig from vllm.config.cache import CacheConfig from vllm.config.compilation import ( CompilationConfig, @@ -66,6 +67,8 @@ from vllm.config.weight_transfer import WeightTransferConfig # __all__ should only contain classes and functions. # Types and globals should be imported from their respective modules. __all__ = [ + # From vllm.config.aux_output + "AuxOutputConfig", # From vllm.config.attention "AttentionConfig", "HiSparseConfig", diff --git a/vllm/config/aux_output.py b/vllm/config/aux_output.py new file mode 100644 index 0000000000..f70f57e70f --- /dev/null +++ b/vllm/config/aux_output.py @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Configuration for execution auxiliary outputs.""" + +from pydantic import Field + +from vllm.config.utils import config + + +@config +class AuxOutputConfig: + """Configuration for auxiliary-output delivery.""" + + enable_return_routed_experts: bool = False + """Capture and return routed-experts auxiliary outputs.""" + + max_bytes: int | None = Field(default=None, gt=0) + """LRU capacity, or ``None`` to derive it from the KV cache capacity.""" + + @property + def enabled(self) -> bool: + """Whether any execution auxiliary output is enabled.""" + return self.enable_return_routed_experts + + def compute_hash(self) -> str: + """Hash AuxOutput settings that alter the model forward graph.""" + from vllm.config.utils import hash_factors + + return hash_factors( + {"enable_return_routed_experts": self.enable_return_routed_experts} + ) diff --git a/vllm/config/model.py b/vllm/config/model.py index 9ee35069da..145814a268 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -247,8 +247,6 @@ class ModelConfig: NOTE: This disables both `torch.compile` and CUDA graphs, and is equivalent to setting `-cc.mode=none -cc.cudagraph_mode=none`.""" - enable_return_routed_experts: bool = False - """Whether to return routed experts.""" return_sampling_mask: bool = False """Whether to return the post-processing token support for each sample.""" max_logprobs: int = Field(default=20, ge=-1) diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 1ff675a8f0..bf11255e8e 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -28,6 +28,7 @@ from vllm.utils import random_uuid from vllm.utils.hashing import safe_hash from .attention import AttentionConfig, HiSparseConfig +from .aux_output import AuxOutputConfig from .cache import CacheConfig from .compilation import CompilationConfig, CompilationMode, CUDAGraphMode from .device import DeviceConfig @@ -376,6 +377,8 @@ class VllmConfig: """Model weight offloading configuration.""" attention_config: AttentionConfig = Field(default_factory=AttentionConfig) """Attention configuration.""" + aux_output_config: AuxOutputConfig = Field(default_factory=AuxOutputConfig) + """Execution auxiliary output configuration.""" engram_config: EngramConfig | None = None """N-gram embedding storage and sharding settings.""" mamba_config: MambaConfig = Field(default_factory=MambaConfig) @@ -554,6 +557,7 @@ class VllmConfig: vllm_factors.append(self.ec_transfer_config.compute_hash()) else: vllm_factors.append("None") + vllm_factors.append(self.aux_output_config.compute_hash()) if self.additional_config: if isinstance(additional_config := self.additional_config, dict): additional_config_hash = safe_hash( @@ -1084,6 +1088,53 @@ class VllmConfig: # This is the same for all backends self.kv_transfer_config.kv_role = "kv_both" + def _verify_aux_output_compatibility(self) -> None: + """Reject configurations unsupported by enabled auxiliary outputs.""" + if not self.aux_output_config.enabled: + return + if not self.use_v2_model_runner: + raise ValueError( + "AuxOutput Connector requires Model Runner V2; set " + "VLLM_USE_V2_MODEL_RUNNER=1." + ) + if self.model_config.runner_type != "generate": + raise ValueError("AuxOutput Connector only supports generate runners.") + if not self.model_config.is_moe: + raise ValueError("AuxOutput Connector only supports MoE models.") + if not self.cache_config.enable_prefix_caching: + raise ValueError("AuxOutput Connector requires prefix caching.") + if ( + self.speculative_config is not None + and self.speculative_config.enable_adaptive_verification + ): + raise ValueError( + "--enable-return-routed-experts is incompatible with " + "adaptive speculative verification." + ) + if self.parallel_config.pipeline_parallel_size > 1: + raise ValueError( + "--enable-return-routed-experts is incompatible with " + "pipeline parallelism (PP > 1)." + ) + if ( + self.parallel_config.decode_context_parallel_size > 1 + or self.parallel_config.prefill_context_parallel_size > 1 + ): + raise ValueError( + "--enable-return-routed-experts is incompatible with " + "context parallelism (DCP/PCP > 1)." + ) + + kv_transfer_config = self.kv_transfer_config + if ( + kv_transfer_config is not None + and kv_transfer_config.is_kv_transfer_instance + ): + raise ValueError( + "--enable-return-routed-experts is incompatible with KV " + "connectors (PD disaggregation and KV cache offload)." + ) + def _verify_kv_transfer_compat(self) -> None: """Reject configurations that silently corrupt KV transfers.""" if ( @@ -1294,38 +1345,6 @@ class VllmConfig: self.parallel_config.is_moe_model = self.model_config.is_moe - if ( - self.model_config is not None - and self.model_config.enable_return_routed_experts - ): - if self.parallel_config.pipeline_parallel_size > 1: - raise ValueError( - "--enable-return-routed-experts is incompatible with " - "pipeline parallelism (PP > 1)." - ) - if ( - self.parallel_config.decode_context_parallel_size > 1 - or self.parallel_config.prefill_context_parallel_size > 1 - ): - raise ValueError( - "--enable-return-routed-experts is incompatible with context " - "parallelism (DCP > 1 or PCP > 1)." - ) - - # Incompatible with any KV connector — covers both PD disaggregation - # (kv_producer/kv_consumer: routing captured on P can't reach D) and - # single-instance KV offload/sharing (kv_both: slot_mapping semantics - # change when KV blocks live outside local GPU memory, breaking the - # slot-indexed routed_experts buffer). - if ( - self.kv_transfer_config is not None - and self.kv_transfer_config.is_kv_transfer_instance - ): - raise ValueError( - "--enable-return-routed-experts is incompatible with KV " - "connectors (PD disaggregation, KV cache offload)." - ) - if ( self.model_config is not None and self.model_config.multimodal_config is not None @@ -2032,6 +2051,7 @@ class VllmConfig: # Resolve kv_offloading-derived connector name into kv_transfer_config # before the HMA check below, which inspects the connector class. self._post_init_kv_transfer_config() + self._verify_aux_output_compatibility() if self.is_mm_encoder_only and self.cache_config.enable_prefix_caching: # Such an instance publishes encoder embeddings and runs no language @@ -2689,7 +2709,7 @@ class VllmConfig: f"quantization={self.model_config.quantization}, " f"quantization_config={self.model_config.quantization_config}, " # noqa f"enforce_eager={self.model_config.enforce_eager}, " - f"enable_return_routed_experts={self.model_config.enable_return_routed_experts}, " # noqa + f"aux_output_config={self.aux_output_config!r}, " f"kv_cache_dtype={self.cache_config.cache_dtype}, " f"device_config={self.device_config.device}, " f"structured_outputs_config={self.structured_outputs_config!r}, " diff --git a/vllm/distributed/aux_output_connector/__init__.py b/vllm/distributed/aux_output_connector/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/vllm/distributed/aux_output_connector/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm/distributed/aux_output_connector/connector.py b/vllm/distributed/aux_output_connector/connector.py new file mode 100644 index 0000000000..bd8cbf4d18 --- /dev/null +++ b/vllm/distributed/aux_output_connector/connector.py @@ -0,0 +1,157 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Scheduler-side control plane for execution auxiliary outputs.""" + +from __future__ import annotations + +from collections.abc import Iterator, Sequence +from dataclasses import dataclass +from typing import TYPE_CHECKING + +import numpy as np + +if TYPE_CHECKING: + from vllm.v1.core.sched.output import SchedulerOutput + from vllm.v1.request import Request + + +@dataclass +class PackedBlockHashes: + """Contiguous, self-contained block hashes for scheduler-worker IPC.""" + + data: bytes + item_size: int + + def __iter__(self) -> Iterator[bytes]: + for start in range(0, len(self.data), self.item_size): + yield self.data[start : start + self.item_size] + + +@dataclass +class AuxOutputConnectorMetadata: + generation: int + requests: dict[str, int] + block_hashes: dict[str, PackedBlockHashes] + finished_requests: tuple[str, ...] + + +@dataclass +class AuxRequestOutput: + token_start: int + rows: np.ndarray + + +class AuxOutputSchedulerConnector: + """Build worker metadata without owning auxiliary output payloads or stores.""" + + def __init__(self) -> None: + # Number of hashes already sent to the worker for each active request. + self._sent_hash_counts: dict[str, int] = {} + # Terminal events are delivered with the next connector metadata. + self._finished_requests: dict[str, PackedBlockHashes | None] = {} + self._generation = 0 + + def build_connector_meta( + self, + scheduler_output: SchedulerOutput, + requests: dict[str, Request], + ) -> AuxOutputConnectorMetadata: + """Build one step's incremental worker metadata.""" + scheduled_requests: dict[str, int] = {} + block_hashes_by_request: dict[str, PackedBlockHashes] = {} + for request_id in scheduler_output.num_scheduled_tokens: + num_sent = self._sent_hash_counts.setdefault(request_id, 0) + request = requests[request_id] + packed = self._pack_new_hashes(request.block_hashes, num_sent) + if packed is not None: + block_hashes_by_request[request_id] = packed + self._sent_hash_counts[request_id] = len(request.block_hashes) + assert request.sampling_params is not None + scheduled_requests[request_id] = max( + request.sampling_params.routed_experts_prompt_start, + 0 if request.num_output_tokens == 0 else request.num_tokens - 1, + ) + # A settled token can complete a hash block after the next async schedule + # was built. Send a hash-only update if the request was not rescheduled. + if len(self._sent_hash_counts) != len(scheduled_requests): + for request_id, num_sent in self._sent_hash_counts.items(): + if request_id in scheduled_requests: + continue + request = requests[request_id] + packed = self._pack_new_hashes(request.block_hashes, num_sent) + if packed is None: + continue + block_hashes_by_request[request_id] = packed + self._sent_hash_counts[request_id] = len(request.block_hashes) + # Sending transfers ownership of these one-shot events. + finished_requests = tuple(self._finished_requests) + block_hashes_by_request.update( + (request_id, block_hashes) + for request_id, block_hashes in self._finished_requests.items() + if block_hashes is not None + ) + self._finished_requests = {} + return AuxOutputConnectorMetadata( + self._generation, + scheduled_requests, + block_hashes_by_request, + finished_requests, + ) + + def take_output( + self, + request: Request, + output: dict[str, AuxRequestOutput] | None, + ) -> np.ndarray | None: + """Return the accepted R3 rows for one scheduled request.""" + request_id = request.request_id + assert output is not None and request_id in output, ( + f"auxiliary output worker output is missing {request_id}" + ) + request_output = output[request_id] + token_end = request.num_tokens - 1 + local_end = token_end - request_output.token_start + if local_end < 0: + assert not request.is_finished(), ( + "finished auxiliary output output has no accepted token range: " + f"request={request_id}, token_end={token_end}, " + f"output_start={request_output.token_start}, " + "output_end=" + f"{request_output.token_start + len(request_output.rows)}" + ) + return None + assert local_end <= len(request_output.rows), ( + "auxiliary output worker output has an invalid token range: " + f"request={request_id}, token_end={token_end}, " + f"output_start={request_output.token_start}, " + f"output_end={request_output.token_start + len(request_output.rows)}" + ) + return request_output.rows[:local_end] + + def request_finished(self, request: Request) -> None: + """Queue a request's terminal event and final block hashes.""" + request_id = request.request_id + num_sent = self._sent_hash_counts.pop(request_id, None) + if num_sent is None: + return + # The next metadata delivers this terminal event to the worker. + self._finished_requests[request_id] = self._pack_new_hashes( + request.block_hashes, num_sent + ) + + @staticmethod + def _pack_new_hashes( + block_hashes: Sequence[bytes], num_sent: int + ) -> PackedBlockHashes | None: + assert num_sent <= len(block_hashes), "KV block-hash history shrank" + new_hashes = block_hashes[num_sent:] + if not new_hashes: + return None + return PackedBlockHashes(b"".join(new_hashes), len(new_hashes[0])) + + def reset(self) -> None: + """Start a new auxiliary output namespace after a prefix-cache reset.""" + # The worker drops temporary state on generation changes; resend hashes. + self._sent_hash_counts.clear() + self._finished_requests.clear() + self._generation += 1 diff --git a/vllm/distributed/aux_output_connector/routed_experts.py b/vllm/distributed/aux_output_connector/routed_experts.py new file mode 100644 index 0000000000..1c89b97215 --- /dev/null +++ b/vllm/distributed/aux_output_connector/routed_experts.py @@ -0,0 +1,219 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Routed-experts auxiliary output keys, publication, and materialization.""" + +from __future__ import annotations + +from collections.abc import Hashable, Iterable, Sequence +from dataclasses import dataclass +from typing import Any + +import numpy as np + +from vllm.distributed.aux_output_connector.store import ( + BackgroundBlockObjectStore, + BlockObject, + BlockObjectStore, +) + + +@dataclass +class _RequestTail: + slot: int + block_start: int + length: int = 0 + + +class RoutedExpertsBuffer: + """Own incomplete and unkeyed full routed-experts blocks.""" + + def __init__( + self, + dtype: np.dtype[Any], + shape_per_token: tuple[int, ...], + block_size: int, + max_num_seqs: int, + max_num_batched_tokens: int, + max_concurrent_batches: int, + ) -> None: + self.dtype = dtype + self.shape_per_token = shape_per_token + self.block_size = block_size + max_step_blocks = ( + max_num_batched_tokens + block_size - 1 + ) // block_size + max_num_seqs + # In-flight full blocks plus one incomplete tail per active request. + max_blocks = max_concurrent_batches * max_step_blocks + max_num_seqs + self._rows: np.ndarray = np.empty( + (max_blocks, block_size, *shape_per_token), dtype=dtype + ) + self._free_slots = list(range(len(self._rows) - 1, -1, -1)) + self._owned_slots: dict[int, int] = {} + self._requests: dict[Hashable, _RequestTail] = {} + + def _tail(self, request_id: Hashable, block_start: int) -> _RequestTail: + tail = self._requests.get(request_id) + if tail is not None: + assert tail.block_start == block_start, ( + "auxiliary output capture skipped an incomplete block: " + f"request={request_id}, expected={tail.block_start}, " + f"actual={block_start}" + ) + return tail + assert self._free_slots, "auxiliary output block pool is exhausted" + tail = _RequestTail(self._free_slots.pop(), block_start) + self._requests[request_id] = tail + return tail + + def capture( + self, request_id: Hashable, token_start: int, rows: np.ndarray + ) -> list[tuple[int, np.ndarray]]: + """Stage rows and return completed blocks without retaining them.""" + rows = np.asarray(rows) + assert rows.shape[1:] == self.shape_per_token and rows.dtype == self.dtype, ( + "routed-experts capture profile changed" + ) + if token_start < 0: + raise ValueError("auxiliary output token start must be non-negative") + + completed: list[tuple[int, np.ndarray]] = [] + offset = 0 + while offset < len(rows): + position = token_start + offset + block_start = position // self.block_size * self.block_size + local_start = position - block_start + + # Full aligned input blocks do not need tail staging. + if ( + request_id not in self._requests + and local_start == 0 + and len(rows) - offset >= self.block_size + ): + completed.append((block_start, rows[offset : offset + self.block_size])) + offset += self.block_size + continue + + tail = self._tail(request_id, block_start) + assert local_start == tail.length, ( + "auxiliary output capture is not contiguous: " + f"request={request_id}, expected={block_start + tail.length}, " + f"actual={position}" + ) + count = min(self.block_size - local_start, len(rows) - offset) + if count == 1: + self._rows[tail.slot, local_start] = rows[offset] + else: + self._rows[tail.slot, local_start : local_start + count] = rows[ + offset : offset + count + ] + tail.length += count + offset += count + if tail.length == self.block_size: + block = self._rows[tail.slot] + del self._requests[request_id] + self._owned_slots[id(block)] = tail.slot + completed.append((block_start, block)) + + return completed + + def read( + self, request_id: Hashable, token_start: int, token_end: int + ) -> np.ndarray: + tail = self._requests.get(request_id) + assert tail is not None, ( + f"auxiliary output buffer is missing request {request_id}" + ) + local_start = token_start - tail.block_start + local_end = token_end - tail.block_start + assert 0 <= local_start < local_end <= tail.length, ( + "auxiliary output range is unavailable: " + f"request={request_id}, range=[{token_start}, {token_end}), " + f"available=[{tail.block_start}, {tail.block_start + tail.length})" + ) + return self._rows[tail.slot, local_start:local_end].copy() + + def retain_block(self, rows: np.ndarray) -> np.ndarray: + """Retain one unkeyed block after the current capture call.""" + if id(rows) in self._owned_slots: + return rows + assert self._free_slots, "auxiliary output block pool is exhausted" + slot = self._free_slots.pop() + retained = self._rows[slot] + retained[...] = rows + self._owned_slots[id(retained)] = slot + return retained + + def release_block(self, rows: np.ndarray) -> None: + slot = self._owned_slots.pop(id(rows), None) + if slot is not None: + self._free_slots.append(slot) + + def discard(self, request_id: Hashable) -> None: + tail = self._requests.pop(request_id, None) + if tail is not None: + self._free_slots.append(tail.slot) + + def reset(self) -> None: + self._requests.clear() + self._owned_slots.clear() + self._free_slots[:] = range(len(self._rows) - 1, -1, -1) + + +def routed_experts_keys( + block_hashes: Iterable[bytes], aux_output_namespace: str +) -> list[str]: + prefix = f"vllm-artifact/{aux_output_namespace}/" + return [prefix + block_hash.hex() for block_hash in block_hashes] + + +def materialize_routed_experts( + store: BackgroundBlockObjectStore | BlockObjectStore, + aux_output_keys: list[str], + *, + shape_per_token: tuple[int, ...], + dtype: np.dtype[Any], +) -> np.ndarray: + if not aux_output_keys: + raise ValueError("routed-experts auxiliary output key list must not be empty") + payload = store.get_concatenated(aux_output_keys) + return np.frombuffer(payload, dtype=dtype).reshape((-1, *shape_per_token)) + + +def publish_routed_experts( + store: BackgroundBlockObjectStore | BlockObjectStore, + *, + batches: Sequence[tuple[Sequence[str], list[tuple[int, np.ndarray]]]], + block_size: int, + retain_keys: Sequence[str] = (), + release_keys: Sequence[str] = (), +) -> None: + """Publish immutable full R3 blocks.""" + objects = [] + for aux_output_keys, blocks in batches: + for block_start, array in blocks: + if block_start < 0 or block_start % block_size: + raise ValueError( + "auxiliary output block start is not hash-block aligned" + ) + block_index = block_start // block_size + if block_index >= len(aux_output_keys): + raise ValueError( + "auxiliary output block has no corresponding key: " + f"start={block_start}, index={block_index}, " + f"keys={len(aux_output_keys)}, block_size={block_size}" + ) + if len(array) != block_size: + raise ValueError( + "auxiliary output block length does not match hash block size" + ) + objects.append( + BlockObject( + key=aux_output_keys[block_index], + payload=array.tobytes(order="C"), + ) + ) + store.put( + objects, + retain_keys=retain_keys, + release_keys=release_keys, + ) diff --git a/vllm/distributed/aux_output_connector/store.py b/vllm/distributed/aux_output_connector/store.py new file mode 100644 index 0000000000..2b364288a5 --- /dev/null +++ b/vllm/distributed/aux_output_connector/store.py @@ -0,0 +1,234 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""In-process auxiliary-output storage.""" + +from __future__ import annotations + +import mmap +import queue +import threading +from collections import OrderedDict +from collections.abc import Iterable +from dataclasses import dataclass + + +@dataclass(frozen=True) +class BlockObject: + """One immutable auxiliary output object.""" + + key: str + payload: bytes + + +class BlockObjectStoreError(RuntimeError): + """AuxOutput storage or retrieval failed.""" + + +class BackgroundBlockObjectStore: + """Serialize store mutations on a background thread.""" + + def __init__(self, store: BlockObjectStore, *, max_pending_batches: int) -> None: + self._store = store + self._queue: queue.Queue[ + tuple[list[BlockObject], tuple[str, ...], tuple[str, ...]] | None + ] = queue.Queue(maxsize=max_pending_batches) + self._error: BaseException | None = None + self._closed = False + self._thread = threading.Thread( + target=self._run, + daemon=True, + name="vllm-aux-output-writer", + ) + self._thread.start() + + def _run(self) -> None: + while True: + update = self._queue.get() + try: + if update is None: + return + if self._error is None: + objects, retain_keys, release_keys = update + self._store.put( + objects, + retain_keys=retain_keys, + release_keys=release_keys, + ) + except BaseException as error: + self._error = error + finally: + self._queue.task_done() + + def _raise_if_failed(self) -> None: + if self._error is not None: + raise BlockObjectStoreError( + "auxiliary output publication failed" + ) from self._error + + def put( + self, + objects: list[BlockObject], + *, + retain_keys: Iterable[str] = (), + release_keys: Iterable[str] = (), + ) -> None: + retains = tuple(retain_keys) + releases = tuple(release_keys) + if not objects and not retains and not releases: + return + if self._closed: + raise RuntimeError("auxiliary output store is closed") + self._queue.put((objects, retains, releases)) + self._raise_if_failed() + + def get_concatenated(self, keys: list[str]) -> bytes: + self._queue.join() + self._raise_if_failed() + return self._store.get_concatenated(keys) + + def close(self) -> None: + if self._closed: + return + self._closed = True + self._queue.join() + self._queue.put(None) + self._thread.join() + self._store.close() + self._raise_if_failed() + + +class BlockObjectStore: + """Single-owner bounded store that fails closed after an eviction.""" + + _UNALLOCATED_SLOT = -1 + + def __init__( + self, + *, + max_bytes: int, + object_nbytes: int, + ) -> None: + if object_nbytes <= 0: + raise ValueError("auxiliary output object size must be positive") + if max_bytes < object_nbytes: + raise ValueError("auxiliary output store must fit at least one object") + self.object_nbytes = object_nbytes + self.num_slots = max_bytes // object_nbytes + self._lru: OrderedDict[str, int] = OrderedDict() + self._references: dict[str, int] = {} + self._free_slots: list[int] = [] + self._next_slot = 0 + self._arena: mmap.mmap | None = mmap.mmap( + -1, self.num_slots * self.object_nbytes + ) + + def _evict_to_fit(self, protected: set[str]) -> None: + excess = len(self._lru) - self.num_slots + if excess <= 0: + return + victims = [] + for key in self._lru: + if key not in self._references and key not in protected: + victims.append(key) + if len(victims) == excess: + break + if len(victims) != excess: + raise BlockObjectStoreError( + "auxiliary output store cannot retain the requested batch: " + f"limit={self.num_slots} objects" + ) + for victim in victims: + slot = self._lru.pop(victim) + if slot != self._UNALLOCATED_SLOT: + self._free_slots.append(slot) + + def _allocate_slot(self) -> int: + if self._free_slots: + return self._free_slots.pop() + slot = self._next_slot + self._next_slot += 1 + return slot + + def put( + self, + objects: list[BlockObject], + *, + retain_keys: Iterable[str] = (), + release_keys: Iterable[str] = (), + ) -> None: + arena = self._arena + if arena is None: + raise RuntimeError("auxiliary output store is closed") + unique = {obj.key: obj for obj in objects} + self._retain(retain_keys) + terminal_order = self._release(release_keys) + for key in unique: + self._lru.setdefault(key, self._UNALLOCATED_SLOT) + self._lru.move_to_end(key) + for key in terminal_order: + if key in self._lru: + self._lru.move_to_end(key) + try: + self._evict_to_fit(set(unique) - set(terminal_order)) + except BlockObjectStoreError: + for key in unique: + if self._lru.get(key) == self._UNALLOCATED_SLOT: + del self._lru[key] + raise + for object_id, obj in unique.items(): + slot = self._lru.get(object_id) + if slot is None: + continue + if slot != self._UNALLOCATED_SLOT: + continue + slot = self._allocate_slot() + offset = slot * self.object_nbytes + arena[offset : offset + self.object_nbytes] = obj.payload + self._lru[object_id] = slot + + def _retain(self, keys: Iterable[str]) -> None: + for key in keys: + references = self._references.get(key, 0) + self._references[key] = references + 1 + + def _release(self, keys: Iterable[str]) -> list[str]: + terminal_order = [] + for key in keys: + references = self._references[key] - 1 + if references: + self._references[key] = references + continue + del self._references[key] + terminal_order.append(key) + return terminal_order + + def get_concatenated(self, keys: list[str]) -> bytes: + arena_obj = self._arena + if arena_obj is None: + raise RuntimeError("auxiliary output store is closed") + try: + entries = [self._lru[key] for key in keys] + except KeyError as error: + raise BlockObjectStoreError( + "auxiliary output object does not exist; the object may have been " + f"evicted (used={len(self._lru)}, " + f"limit={self.num_slots} objects). Increase " + "aux_output_config.max_bytes when a KV cache hit requires it." + ) from error + arena = memoryview(arena_obj) + try: + payload = b"".join( + arena[slot * self.object_nbytes : (slot + 1) * self.object_nbytes] + for slot in entries + ) + finally: + arena.release() + for key in keys: + self._lru.move_to_end(key) + return payload + + def close(self) -> None: + arena = self._arena + if arena is not None: + self._arena = None + arena.close() diff --git a/vllm/distributed/aux_output_connector/worker.py b/vllm/distributed/aux_output_connector/worker.py new file mode 100644 index 0000000000..7959e15ea8 --- /dev/null +++ b/vllm/distributed/aux_output_connector/worker.py @@ -0,0 +1,412 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Worker-side auxiliary-output data plane.""" + +from __future__ import annotations + +from collections.abc import Iterable, Sequence +from dataclasses import dataclass, field +from threading import Lock +from typing import TYPE_CHECKING, Any + +import numpy as np +import torch + +from vllm.config import VllmConfig +from vllm.distributed.aux_output_connector.connector import ( + AuxOutputConnectorMetadata, + AuxRequestOutput, +) +from vllm.distributed.aux_output_connector.routed_experts import ( + RoutedExpertsBuffer, + materialize_routed_experts, + publish_routed_experts, + routed_experts_keys, +) +from vllm.distributed.aux_output_connector.store import ( + BackgroundBlockObjectStore, + BlockObjectStore, +) +from vllm.distributed.parallel_state import get_tp_group +from vllm.model_executor.layers.fused_moe.routed_experts_capturer import ( + RoutedExpertsCapturer, + bind_routed_experts_capturer, +) +from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes + +if TYPE_CHECKING: + from vllm.v1.kv_cache_interface import KVCacheConfig + from vllm.v1.worker.gpu.input_batch import InputBatch + + +@dataclass +class _WorkerRequestState: + aux_output_keys: list[str] = field(default_factory=list) + pending_blocks: list[tuple[int, np.ndarray]] = field(default_factory=list) + capture_cursor: int | None = None + scheduled_cursor: int = 0 + emit_cursor: int = 0 + pending_outputs: int = 0 + # Terminal event received; teardown waits for in-flight step outputs. + finished: bool = False + + +@dataclass +class PendingAuxOutput: + """Own one step's R3 snapshot until its asynchronous copy is consumed.""" + + connector: AuxOutputWorkerConnector + request_ids: list[str] + token_starts: np.ndarray + query_start_loc: np.ndarray + # GPU snapshot taken on the main stream. + routed_experts_gpu: torch.Tensor + # Filled by enqueue_cpu_copy on the output copy stream. + routed_experts: np.ndarray | None = None + num_sampled: np.ndarray | None = None + num_rejected: np.ndarray | None = None + + def enqueue_cpu_copy( + self, num_sampled: np.ndarray, num_rejected: np.ndarray + ) -> None: + """Enqueue asynchronous D2H copies; call on the output copy stream.""" + self.num_sampled = num_sampled + self.num_rejected = num_rejected + self.routed_experts = self.routed_experts_gpu.to( + "cpu", non_blocking=True + ).numpy() + + def process_output(self) -> dict[str, AuxRequestOutput]: + return self.connector.process_output(self) + + +class AuxOutputWorkerConnector: + """Own capture, request tails, and backend resources on the output worker.""" + + def __init__( + self, + *, + vllm_config: VllmConfig, + model: torch.nn.Module, + kv_cache_config: KVCacheConfig, + ) -> None: + max_num_batched_tokens = vllm_config.scheduler_config.max_num_batched_tokens + capturer = RoutedExpertsCapturer( + max_num_batched_tokens=max_num_batched_tokens, vllm_config=vllm_config + ) + bind_routed_experts_capturer(model, capturer) + self._capturer = capturer + self._store: BackgroundBlockObjectStore | None = None + self._buffer: RoutedExpertsBuffer | None = None + self._requests: dict[str, _WorkerRequestState] = {} + self._generation = 0 + self._step_metadata: AuxOutputConnectorMetadata | None = None + # Steps whose asynchronous copy has not been consumed yet. The engine + # consumes step outputs in order, so at most max_concurrent_batches + # are outstanding; guarded by _lock against the output thread. + self._pending_outputs: list[PendingAuxOutput] = [] + self._lock = Lock() + self._max_concurrent_batches = vllm_config.max_concurrent_batches + # Every TP rank participates in capture collectives, but only the + # executor output rank owns the auxiliary output data plane. + if not get_tp_group().is_first_rank: + return + + shape_per_token = self._capturer.shape_per_token + dtype: np.dtype[Any] = np.dtype(self._capturer.output_dtype_name) + scheduler_block_size, hash_block_size = resolve_kv_cache_block_sizes( + kv_cache_config, vllm_config + ) + hashes_per_kv_block = scheduler_block_size // hash_block_size + block_nbytes = hash_block_size * int(np.prod(shape_per_token)) * dtype.itemsize + max_bytes = vllm_config.aux_output_config.max_bytes + if max_bytes is None: + max_bytes = kv_cache_config.num_blocks * hashes_per_kv_block * block_nbytes + self._store = BackgroundBlockObjectStore( + BlockObjectStore(max_bytes=max_bytes, object_nbytes=block_nbytes), + max_pending_batches=2 * vllm_config.scheduler_config.max_num_seqs, + ) + self._buffer = RoutedExpertsBuffer( + dtype, + shape_per_token, + hash_block_size, + vllm_config.scheduler_config.max_num_seqs, + max_num_batched_tokens, + vllm_config.max_concurrent_batches, + ) + + def prepare_output(self, input_batch: InputBatch) -> PendingAuxOutput | None: + """Snapshot one step's R3 tensor for asynchronous CPU transfer.""" + if self._buffer is None or self._step_metadata is None: + return None + + request_ids = list(input_batch.req_ids) + query_start_loc = input_batch.query_start_loc_np[: len(request_ids) + 1] + num_rows = int(query_start_loc[-1]) + pending_output = PendingAuxOutput( + connector=self, + request_ids=request_ids, + token_starts=input_batch.num_computed_tokens_np, + query_start_loc=query_start_loc, + routed_experts_gpu=self._capturer.snapshot_routing_data(num_rows), + ) + with self._lock: + assert len(self._pending_outputs) < self._max_concurrent_batches, ( + "auxiliary output step outputs are not consumed in order" + ) + self._pending_outputs.append(pending_output) + for request_id in pending_output.request_ids: + self._requests[request_id].pending_outputs += 1 + return pending_output + + def process_output(self, pending: PendingAuxOutput) -> dict[str, AuxRequestOutput]: + """Commit one consumed R3 snapshot and build request outputs. + + Request teardown deferred by begin_step is completed here once no + in-flight step covers the request. + """ + with self._lock: + teardown = [] + release_keys: list[str] = [] + try: + outputs = self._commit_output(pending) + finally: + self._pending_outputs.remove(pending) + for request_id in pending.request_ids: + state = self._requests[request_id] + state.pending_outputs -= 1 + if state.pending_outputs == 0 and state.finished: + teardown.append(request_id) + release_keys.extend(reversed(state.aux_output_keys)) + if teardown: + self._publish_blocks([], release_keys=release_keys) + self._teardown(teardown) + return outputs + + def _commit_output(self, pending: PendingAuxOutput) -> dict[str, AuxRequestOutput]: + buffer = self._buffer + store = self._store + assert buffer is not None and store is not None + block_size = buffer.block_size + + routed_experts = pending.routed_experts + num_sampled = pending.num_sampled + num_rejected = pending.num_rejected + assert ( + routed_experts is not None + and num_sampled is not None + and num_rejected is not None + ), "auxiliary output CPU copy was not enqueued" + + # Publish the whole batch before materializing any consumer output. + materialize_outputs: list[tuple[str, int, int]] = [] + block_batches = [] + outputs: dict[str, AuxRequestOutput] = {} + + # Use the ModelRunner's actual batch boundaries rather than rebuilding them. + for request_id, token_start, start, end, sampled, rejected in zip( + pending.request_ids, + pending.token_starts, + pending.query_start_loc[:-1], + pending.query_start_loc[1:], + num_sampled, + num_rejected, + strict=True, + ): + request_num_tokens = end - start + assert request_num_tokens > 0, ( + "auxiliary output request token count must be positive" + ) + state = self._requests[request_id] + + # Capture precedes speculative acceptance, so discard the rejected + # suffix. Batch boundaries still span the full executed range. + rejected = int(rejected) + assert 0 <= rejected <= request_num_tokens, ( + "auxiliary output rejected-token count is invalid" + ) + rows = routed_experts[start : end - rejected] + + capture_start = token_start + capture_cursor = state.capture_cursor + if capture_cursor is None: + capture_cursor = capture_start + + assert capture_start >= capture_cursor, ( + "auxiliary output capture moved backwards" + ) + if capture_start > capture_cursor: + # Reattach after an optimistically scheduled suffix was rejected. + assert capture_cursor < state.scheduled_cursor, ( + "auxiliary output capture has an unbacked token gap" + ) + capture_start = capture_cursor + + emit_start = state.emit_cursor + # Complete blocks without keys remain pending until a hash update. + completed = buffer.capture(request_id, capture_start, rows) + token_end = capture_start + len(rows) + state.capture_cursor = token_end + state.scheduled_cursor = token_start + request_num_tokens + block_batches.append((state, completed)) + + if sampled > 0 and emit_start <= token_end: + if emit_start >= capture_start: + outputs[request_id] = AuxRequestOutput( + emit_start, rows[emit_start - capture_start :] + ) + state.emit_cursor = token_end + else: + materialize_outputs.append((request_id, emit_start, token_end)) + + # A consumer may reuse a block produced earlier in the same batch. + self._publish_blocks(block_batches) + + for request_id, emit_start, token_end in materialize_outputs: + state = self._requests[request_id] + stored_end = ( + min(token_end // block_size, len(state.aux_output_keys)) * block_size + ) + if emit_start < stored_end: + first_block = emit_start // block_size + stored = materialize_routed_experts( + store, + state.aux_output_keys[first_block : stored_end // block_size], + shape_per_token=buffer.shape_per_token, + dtype=buffer.dtype, + ) + local_start = emit_start % block_size + rows = stored[local_start : local_start + stored_end - emit_start] + if stored_end < token_end: + rows = np.concatenate( + (rows, buffer.read(request_id, stored_end, token_end)) + ) + else: + rows = buffer.read(request_id, emit_start, token_end) + outputs[request_id] = AuxRequestOutput(emit_start, rows) + state.emit_cursor = token_end + return outputs + + def _publish_blocks( + self, + batches: list[tuple[_WorkerRequestState, list[tuple[int, np.ndarray]]]], + retain_keys: Sequence[str] = (), + release_keys: Sequence[str] = (), + ) -> None: + store = self._store + buffer = self._buffer + assert store is not None and buffer is not None + ready_batches = [] + for state, completed in batches: + blocks = state.pending_blocks + completed + keyed_end = len(state.aux_output_keys) * buffer.block_size + ready = [(start, rows) for start, rows in blocks if start < keyed_end] + state.pending_blocks = [ + (start, buffer.retain_block(rows)) + for start, rows in blocks + if start >= keyed_end + ] + if ready: + ready_batches.append((state.aux_output_keys, ready)) + if ready_batches or retain_keys or release_keys: + publish_routed_experts( + store, + batches=ready_batches, + block_size=buffer.block_size, + retain_keys=retain_keys, + release_keys=release_keys, + ) + for _, blocks in ready_batches: + for _, rows in blocks: + buffer.release_block(rows) + + def _teardown(self, request_ids: Iterable[str]) -> None: + """Drop finished requests' temporary state after publishing key releases.""" + buffer = self._buffer + assert buffer is not None + for request_id in request_ids: + state = self._requests.pop(request_id) + for _, rows in state.pending_blocks: + buffer.release_block(rows) + buffer.discard(request_id) + + def begin_step(self, metadata: AuxOutputConnectorMetadata | None) -> None: + """Apply one scheduler step's request and block-hash updates. + + While an unconsumed step covers a finished request, its teardown is + deferred to that step's process_output. + """ + self._step_metadata = metadata + if self._buffer is None or metadata is None: + return + assert not metadata.requests.keys() & metadata.finished_requests, ( + "auxiliary output request cannot run and finish in one step" + ) + assert metadata.generation >= self._generation, ( + "auxiliary output metadata generation moved backwards" + ) + with self._lock: + release_keys: list[str] = [] + if metadata.generation > self._generation: + # A generation change follows a prefix-cache reset, which the + # scheduler only performs once all model output is consumed. + assert not self._pending_outputs, ( + "auxiliary output generation changed with output in flight" + ) + release_keys.extend( + key + for state in self._requests.values() + for key in reversed(state.aux_output_keys) + ) + self._buffer.reset() + self._requests.clear() + self._generation = metadata.generation + for request_id, emit_start in metadata.requests.items(): + state = self._requests.setdefault( + request_id, _WorkerRequestState(emit_cursor=emit_start) + ) + if state.pending_outputs == 0: + # In-flight steps leave the worker cursor behind the + # scheduler's optimistic view; only check settled requests. + assert emit_start <= state.emit_cursor, ( + "auxiliary output Scheduler emit cursor moved ahead" + ) + block_batches: list[ + tuple[_WorkerRequestState, list[tuple[int, np.ndarray]]] + ] = [] + retained_keys: list[str] = [] + for request_id, block_hashes in metadata.block_hashes.items(): + state = self._requests[request_id] + keys = routed_experts_keys(block_hashes, str(self._generation)) + state.aux_output_keys.extend(keys) + retained_keys.extend(keys) + block_batches.append((state, [])) + finished_now: list[str] = [] + for request_id in metadata.finished_requests: + state = self._requests[request_id] + state.finished = True + if state.pending_outputs: + continue + finished_now.append(request_id) + release_keys.extend(reversed(state.aux_output_keys)) + self._publish_blocks(block_batches, retained_keys, release_keys) + self._teardown(finished_now) + + def close(self) -> None: + with self._lock: + self._pending_outputs.clear() + if self._store is not None: + try: + self._store.close() + finally: + self._store = None + + +def get_aux_output_connector( + model: torch.nn.Module, vllm_config: VllmConfig, kv_cache_config: KVCacheConfig +) -> AuxOutputWorkerConnector: + return AuxOutputWorkerConnector( + model=model, + kv_cache_config=kv_cache_config, + vllm_config=vllm_config, + ) diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 0d33caf625..9fad681d7d 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -35,6 +35,7 @@ from typing_extensions import TypeIs import vllm.envs as envs from vllm.config import ( AttentionConfig, + AuxOutputConfig, CacheConfig, CompilationConfig, ConfigType, @@ -447,7 +448,9 @@ class EngineArgs: """Arguments for vLLM engine.""" model: str = ModelConfig.model - enable_return_routed_experts: bool = ModelConfig.enable_return_routed_experts + # Public compatibility argument. Canonical runtime state lives in + # AuxOutputConfig. + enable_return_routed_experts: bool = False return_sampling_mask: bool = ModelConfig.return_sampling_mask model_weights: str = ModelConfig.model_weights served_model_name: str | list[str] | None = ModelConfig.served_model_name @@ -674,6 +677,7 @@ class EngineArgs: structured_outputs_config: StructuredOutputsConfig = get_field( VllmConfig, "structured_outputs_config" ) + aux_output_config: AuxOutputConfig = get_field(VllmConfig, "aux_output_config") reasoning_parser: str = StructuredOutputsConfig.reasoning_parser reasoning_parser_plugin: str | None = None @@ -816,6 +820,10 @@ class EngineArgs: self.compilation_config = CompilationConfig(**self.compilation_config) if isinstance(self.attention_config, dict): self.attention_config = AttentionConfig(**self.attention_config) + if isinstance(self.aux_output_config, dict): + self.aux_output_config = AuxOutputConfig(**self.aux_output_config) + if self.enable_return_routed_experts: + self.aux_output_config.enable_return_routed_experts = True if isinstance(self.engram_config, dict): self.engram_config = EngramConfig(**self.engram_config) if isinstance(self.mamba_config, dict): @@ -884,6 +892,7 @@ class EngineArgs: """Shared CLI arguments for vLLM engine.""" # Model arguments model_kwargs = get_kwargs(ModelConfig) + aux_output_kwargs = get_kwargs(AuxOutputConfig) model_group = parser.add_argument_group( title="ModelConfig", description=ModelConfig.__doc__, @@ -923,7 +932,7 @@ class EngineArgs: model_group.add_argument("--enforce-eager", **model_kwargs["enforce_eager"]) model_group.add_argument( "--enable-return-routed-experts", - **model_kwargs["enable_return_routed_experts"], + **aux_output_kwargs["enable_return_routed_experts"], ) model_group.add_argument( "--return-sampling-mask", @@ -1749,6 +1758,9 @@ class EngineArgs: vllm_group.add_argument( "--structured-outputs-config", **vllm_kwargs["structured_outputs_config"] ) + vllm_group.add_argument( + "--aux-output-config", **vllm_kwargs["aux_output_config"] + ) vllm_group.add_argument("--profiler-config", **vllm_kwargs["profiler_config"]) vllm_group.add_argument( "--optimization-level", **vllm_kwargs["optimization_level"] @@ -1856,7 +1868,6 @@ class EngineArgs: quantization_config=self.quantization_config, allow_deprecated_quantization=self.allow_deprecated_quantization, enforce_eager=self.enforce_eager, - enable_return_routed_experts=self.enable_return_routed_experts, return_sampling_mask=self.return_sampling_mask, max_logprobs=self.max_logprobs, logprobs_mode=self.logprobs_mode, @@ -2669,6 +2680,7 @@ class EngineArgs: load_config=load_config, offload_config=offload_config, attention_config=attention_config, + aux_output_config=self.aux_output_config, engram_config=self.engram_config, mamba_config=mamba_config, kernel_config=kernel_config, diff --git a/vllm/model_executor/layers/fused_moe/routed_experts_capturer.py b/vllm/model_executor/layers/fused_moe/routed_experts_capturer.py index 2358089602..c49d614549 100644 --- a/vllm/model_executor/layers/fused_moe/routed_experts_capturer.py +++ b/vllm/model_executor/layers/fused_moe/routed_experts_capturer.py @@ -10,15 +10,12 @@ from collections.abc import Callable from functools import partial from typing import Protocol, runtime_checkable -import numpy as np import torch from vllm.config import VllmConfig from vllm.distributed.parallel_state import get_tp_group from vllm.forward_context import get_forward_context from vllm.platforms import current_platform -from vllm.v1.kv_cache_interface import KVCacheConfig, is_full_attention_spec -from vllm.v1.outputs import RoutedExpertsTensors logger = logging.getLogger(__name__) @@ -46,37 +43,29 @@ def _get_routed_experts_shape(vllm_config: VllmConfig) -> tuple[int, int, int]: class RoutedExpertsCapturer: """Worker-side capturer for routed experts, lives on GPU. - Layer-level hooks call :meth:`capture` from inside the forward pass - with the per-layer ``topk_ids`` tensor. The tensor is sliced to the - tokens owned by this DP rank and written into a preallocated device - buffer. At the end of the step, :class:`GPUModelRunner` reads the - device buffer, issues a D2H copy into a pinned CPU buffer, and hands - the result to the scheduler via :class:`RoutedExpertsLists`. + Layer-level hooks call :meth:`capture` inside the forward pass. Routing + rows owned by this DP rank are written into a preallocated device buffer. - The device / pinned-CPU transit buffers use ``torch.int32`` (not a - narrow ``uint8``/``uint16`` sized by ``num_experts``). This keeps the - SP all-gather path free of dtype casts, matches the router's native - ``topk_ids`` indices dtype more closely, and costs only a few MB per - worker (``max_num_batched_tokens * num_layers * top_k * 4`` bytes). - The scheduler-side slot buffer - (``RoutedExpertsManager.routed_experts_by_slot``) still uses the - narrow dtype -- numpy fancy-index assignment in ``store_batch`` - narrows the data on the way in. + The device buffer uses ``int32``. Stable snapshots use the narrowest dtype + that can represent every logical expert ID. Invariants: - One instance per worker; shape is fixed at init and covers the worst-case step (``max_num_batched_tokens`` tokens). - Every routed layer overwrites the current step's token rows. - - ``device_buffer.dtype`` is ``torch.int32``. """ def __init__( self, max_num_batched_tokens: int, vllm_config: VllmConfig, - kv_cache_config: KVCacheConfig, ) -> None: - num_layers, _, num_experts_per_tok = _get_routed_experts_shape(vllm_config) + num_layers, num_experts, num_experts_per_tok = _get_routed_experts_shape( + vllm_config + ) + self.shape_per_token = (num_layers, num_experts_per_tok) + self.output_dtype_name = "uint8" if num_experts <= 256 else "uint16" + self.output_dtype = getattr(torch, self.output_dtype_name) logger.info( "RoutedExpertsCapturer: allocating buffer with " "max_tokens=%d, num_layers=%d, num_experts_per_tok=%d " @@ -87,23 +76,12 @@ class RoutedExpertsCapturer: vllm_config.model_config.hf_text_config.model_type, ) self.device_buffer = torch.zeros( - ( - max_num_batched_tokens, - num_layers, - num_experts_per_tok, - ), - # Use int32 for the device / host transit buffers: it - # matches the router's native topk_ids dtype, is universally - # supported by NCCL (uint8/uint16 are version-dependent), - # and the extra bytes are small (few MB per worker). The - # big scheduler-side slot buffer stays narrow. + (max_num_batched_tokens, *self.shape_per_token), dtype=torch.int32, device=current_platform.device_type, ) self.dp_rank = vllm_config.parallel_config.data_parallel_rank self.tp_size = vllm_config.parallel_config.tensor_parallel_size - # KV cache group whose slot layout the routing data is keyed by. - self.attn_gid = get_routed_experts_attn_gid(kv_cache_config) def capture(self, layer_id: int, topk_ids: torch.Tensor) -> None: """Capture expert routing decisions for a specific layer. @@ -139,31 +117,21 @@ class RoutedExpertsCapturer: """ ctx = get_forward_context() - if ctx.dp_metadata is None: # single dp - start_loc = 0 - end_loc = topk_ids.shape[0] - token_num_per_dp = topk_ids.shape[0] - else: # multi dp + if ctx.dp_metadata is None: + local_topk_ids = topk_ids + else: num_tokens_dp = ctx.dp_metadata.num_tokens_across_dp_cpu - token_num_per_dp = int(num_tokens_dp[self.dp_rank].item()) + num_local_tokens = int(num_tokens_dp[self.dp_rank].item()) total = int(num_tokens_dp.sum().item()) n = topk_ids.shape[0] shard_sizes = getattr(ctx.dp_metadata, "local_sizes", None) gathered_size = sum(shard_sizes) if shard_sizes is not None else -1 if n == total: - # Naive dispatch: all DP ranks' tokens concatenated - # before routing. This rank owns tokens - # [end_loc - token_num_per_dp, end_loc). - cumsum = torch.cumsum(num_tokens_dp, dim=0) - end_loc = int(cumsum[self.dp_rank].item()) - start_loc = end_loc - token_num_per_dp - elif n == token_num_per_dp: - # Modular-kernel path: DP combine happens inside - # quant_method.apply; select_experts only sees this - # rank's tokens, take the whole tensor. - start_loc = 0 - end_loc = token_num_per_dp + end = int(num_tokens_dp[: self.dp_rank + 1].sum().item()) + local_topk_ids = topk_ids[end - num_local_tokens : end] + elif n == num_local_tokens: + local_topk_ids = topk_ids elif shard_sizes is not None and n == gathered_size: # Naive DP+EP dispatch gathers the sequence-parallel shards in # flattened DP-then-TP order. ``local_sizes`` is the exact @@ -173,12 +141,11 @@ class RoutedExpertsCapturer: assert len(shard_sizes) % num_dp_ranks == 0 shards_per_dp_rank = len(shard_sizes) // num_dp_ranks first_shard = self.dp_rank * shards_per_dp_rank - start_loc = sum(shard_sizes[:first_shard]) - end_loc = start_loc + token_num_per_dp + start = sum(shard_sizes[:first_shard]) + local_topk_ids = topk_ids[start : start + num_local_tokens] elif ( self.tp_size > 1 - and n != token_num_per_dp - and n == (token_num_per_dp + self.tp_size - 1) // self.tp_size + and n == (num_local_tokens + self.tp_size - 1) // self.tp_size ): # SP + modular-kernel path. All-gather across the TP # group along dim=0 to reconstruct the full per-DP-rank @@ -193,60 +160,30 @@ class RoutedExpertsCapturer: # # ``topk_ids`` is already whatever the router produced # (typically int32/int64, both supported by NCCL); the - # downstream ``device_buffer[...] = topk_ids[...]`` - # setitem narrows into int32 automatically. - topk_ids = get_tp_group().all_gather(topk_ids, dim=0) - start_loc = 0 - end_loc = token_num_per_dp + # downstream buffer assignment narrows to the capture dtype. + local_topk_ids = get_tp_group().all_gather(topk_ids, dim=0)[ + :num_local_tokens + ] else: - sp_expected = ( - (token_num_per_dp + self.tp_size - 1) // self.tp_size - if self.tp_size > 0 - else -1 - ) + sp_expected = (num_local_tokens + self.tp_size - 1) // self.tp_size raise AssertionError( "RoutedExpertsCapturer: unexpected topk_ids batch " - f"dim {n} (expected {total}, {token_num_per_dp}, " + f"dim {n} (expected {total}, {num_local_tokens}, " f"{gathered_size}, or {sp_expected} for " f"dp_rank={self.dp_rank}, tp_size={self.tp_size})" ) - if layer_id >= self.device_buffer.shape[1]: + if not 0 <= layer_id < self.device_buffer.shape[1]: raise IndexError( f"routed-experts layer {layer_id} exceeds capture buffer " f"layer count {self.device_buffer.shape[1]}" ) - self.device_buffer[:token_num_per_dp, layer_id, :] = topk_ids[ - start_loc:end_loc, : - ] + self.device_buffer[: len(local_topk_ids), layer_id] = local_topk_ids - def get_device_buffer(self) -> torch.Tensor: - """Return the underlying device buffer so the model runner can - issue the D2H copy. The tensor is shared; callers must either - clone or fully drain it before the next forward pass overwrites it. - """ - return self.device_buffer - - def get_routed_experts( - self, slot_mappings: torch.Tensor, num_tokens: int - ) -> RoutedExpertsTensors: - """Snapshot this step's routing data and its attention slot mapping. - - Both tensors are cloned since the capture buffer and the shared - ``slot_mappings`` are overwritten by the next step, which may race - with an in-flight D2H copy. - - Args: - slot_mappings: Per-KV-cache-group slot mappings for this step, - shape ``(num_kv_cache_groups, max_num_batched_tokens)``. - num_tokens: Total number of tokens scheduled in this step. - - """ - return RoutedExpertsTensors( - routing_data=self.device_buffer[:num_tokens].clone(), - slot_mapping=slot_mappings[self.attn_gid, :num_tokens].clone(), - ) + def snapshot_routing_data(self, num_tokens: int) -> torch.Tensor: + """Return a stable snapshot of the current routing data.""" + return self.device_buffer[:num_tokens].to(self.output_dtype) def bind_routed_experts_capturer( @@ -268,20 +205,12 @@ def bind_routed_experts_capturer( continue if not isinstance(module, MoERunner): continue - layer_id = module.layer_id - - def capture_fn( - topk_ids: torch.Tensor, - layer_id: int = layer_id, - capturer: RoutedExpertsCapturer = capturer, - ) -> None: - capturer.capture(layer_id, topk_ids) - + capture_fn = partial(capturer.capture, module.layer_id) quant_method = module._quant_method - moe_kernel = getattr(quant_method, "moe_kernel", None) - impl = getattr(moe_kernel, "impl", None) - fused_experts = getattr(impl, "fused_experts", None) if quant_method.is_monolithic: + moe_kernel = getattr(quant_method, "moe_kernel", None) + impl = getattr(moe_kernel, "impl", None) + fused_experts = getattr(impl, "fused_experts", None) if not ( isinstance(fused_experts, FusedMoEExpertsMonolithic) and fused_experts.supports_routing_replay_capture() @@ -303,132 +232,3 @@ def bind_routed_experts_capturer( if num_bound == 0: raise ValueError("No supported MoE router found for routed-experts capture.") - - -def get_routed_experts_attn_gid(kv_cache_config: KVCacheConfig) -> int: - """Return the full-attention KV cache group used for routed experts.""" - for gid, group in enumerate(kv_cache_config.kv_cache_groups): - if is_full_attention_spec(group.kv_cache_spec): - return gid - raise ValueError("Routed-experts capture requires a full-attention KV cache group.") - - -class RoutedExpertsManager: - """Scheduler-side slot-indexed buffer for routed experts. - - Lives on CPU in the scheduler process. Each slot corresponds to - ``block_id * block_size + offset_in_block`` where ``block_id`` is - drawn from the physical KV-cache block pool, so routing data is - tied to physical blocks and naturally survives preemption for - prefix-cached blocks (prefix hits re-expose the same slots). - - Data flow per step: - 1. Worker D2Hs its device capture buffer into - :class:`RoutedExpertsLists` and returns it via - :class:`ModelRunnerOutput`. - 2. Scheduler calls :meth:`store_batch` with that step's - ``(routing_data, slot_mapping)`` — a single CPU->CPU - fancy-index assign, ~few MB per step. - 3. On request completion / abort / preemption, the scheduler - calls :meth:`get` with the request's block IDs to recover - the full per-token routing. - - Memory: ``routed_experts_by_slot`` is sized for the whole block - pool (``num_blocks * block_size`` slots). For large block pools - this can reach multiple GB; see the init log for the exact size. - """ - - def __init__( - self, - vllm_config: VllmConfig, - kv_cache_config: KVCacheConfig, - ) -> None: - # Hybrid models also have KV groups whose slot layout differs. - self.attn_gid = get_routed_experts_attn_gid(kv_cache_config) - attn_group = kv_cache_config.kv_cache_groups[self.attn_gid] - self.block_size = attn_group.kv_cache_spec.block_size - - # All kv_cache_groups share the same physical block pool, so - # block IDs span [0, num_blocks) regardless of how many groups - # exist. Sizing to the full pool avoids index-out-of-range - # when different groups happen to land on the same block. - num_layers, num_experts, num_experts_per_tok = _get_routed_experts_shape( - vllm_config - ) - max_num_slots = kv_cache_config.num_blocks * self.block_size - # Expert IDs are 0..num_experts-1; uint8 fits 256 distinct - # values so the boundary is ``<= 256`` (NOT ``< 256``). Keeping - # this narrow matters because the slot buffer is sized for the - # whole block pool and can reach multiple GB. - expert_id_dtype = np.uint8 if num_experts <= 256 else np.uint16 - self.routed_experts_by_slot = np.zeros( - ( - max_num_slots, - num_layers, - num_experts_per_tok, - ), - dtype=expert_id_dtype, - ) - logger.info( - "RoutedExpertsManager CPU buffer: %.2f GB " - "(slots=%d, layers=%d, top_k=%d, dtype=%s)", - self.routed_experts_by_slot.nbytes / 1e9, - max_num_slots, - num_layers, - num_experts_per_tok, - self.routed_experts_by_slot.dtype.name, - ) - - def store_batch(self, data: np.ndarray, slot_mapping: np.ndarray) -> None: - """Persist one step's routed experts into the slot buffer. - - Equivalent to ``slot_buffer[slot_mapping] = data``; numpy fancy - indexing handles repeated / out-of-order indices. Called once - per scheduler step in ``update_from_output``. - """ - self.routed_experts_by_slot[slot_mapping] = data - - def get( - self, - block_ids: list[int], - num_tokens: int, - token_start: int = 0, - ) -> np.ndarray: - """Read routed experts data for a completed / preempted request. - - Reconstructs a per-token slot_mapping from the request's block - IDs and returns the routing slice. Because numpy fancy indexing - returns a **copy** (not a view), the returned ndarray is safe - to hold across subsequent :meth:`store_batch` calls — do not - replace the fancy index with a slice without re-verifying. - - Args: - block_ids: Block IDs from the attention KV-cache group. - num_tokens: Number of tokens that have gone through a forward - pass and therefore have routing data written to their - slots (typically ``request.num_tokens - 1``; the last - sampled token has not been forwarded yet). Slots beyond - ``request.num_computed_tokens`` are zero-initialized. - token_start: Skip the first ``token_start`` tokens from the - result. The slot_mapping is sliced before the fancy-index - read, so only the requested slots are fetched — no large - intermediate array is allocated. Clamped to - ``[0, num_tokens]`` automatically. - - Returns: - Array of shape (num_tokens - token_start, num_layers, - num_experts_per_tok). - - """ - bs = self.block_size - block_ids_array = np.array(block_ids, dtype=np.int32) - block_offsets = np.arange(bs) - # slot = block_id * block_size + offset_in_block; flatten the - # (num_blocks, block_size) grid and trim to num_tokens, then - # skip the first token_start entries so only the requested - # range is fetched in a single fancy-index read. - slot_mapping = ( - block_ids_array.reshape(-1, 1) * bs + block_offsets.reshape(1, -1) - ).flatten()[:num_tokens] - slot_mapping = slot_mapping[token_start:] - return self.routed_experts_by_slot[slot_mapping] diff --git a/vllm/outputs.py b/vllm/outputs.py index 9d46f327d7..e9a08f7c46 100644 --- a/vllm/outputs.py +++ b/vllm/outputs.py @@ -194,6 +194,10 @@ class RequestOutput: completion.cumulative_logprob = ( next_completion.cumulative_logprob ) + # R3 is returned on the terminal output and must survive + # aggregation with earlier chunks that have no R3. + if next_completion.routed_experts is not None: + completion.routed_experts = next_completion.routed_experts completion.finish_reason = next_completion.finish_reason completion.stop_reason = next_completion.stop_reason else: diff --git a/vllm/v1/core/sched/output.py b/vllm/v1/core/sched/output.py index 7501c51b2c..0da64cf27e 100644 --- a/vllm/v1/core/sched/output.py +++ b/vllm/v1/core/sched/output.py @@ -14,6 +14,9 @@ if TYPE_CHECKING: import numpy.typing as npt import torch + from vllm.distributed.aux_output_connector.connector import ( + AuxOutputConnectorMetadata, + ) from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorMetadata from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata from vllm.lora.request import LoRARequest @@ -23,6 +26,7 @@ if TYPE_CHECKING: from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy from vllm.v1.request import Request else: + AuxOutputConnectorMetadata = object ECConnectorMetadata = object KVConnectorMetadata = object KVCacheBlockCopy = object @@ -285,6 +289,9 @@ class SchedulerOutput: # synchronously during this step (load_async=False). has_sync_kv_loads: bool = False + # Execution-auxiliary output control metadata consumed by the worker connector. + aux_output_connector_metadata: AuxOutputConnectorMetadata | None = None + # EC Cache Connector metadata ec_connector_metadata: ECConnectorMetadata | None = None # EC Cache Manager metadata diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index a2638942f8..7d8aee55dc 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -9,6 +9,7 @@ from typing import Any from vllm.compilation.cuda_graph import CUDAGraphStat from vllm.config import KVEventsConfig, VllmConfig +from vllm.distributed.aux_output_connector.connector import AuxOutputSchedulerConnector from vllm.distributed.ec_transfer.ec_connector.base import ( ECConnectorBase, ECConnectorMetadata, @@ -26,9 +27,6 @@ from vllm.distributed.kv_transfer.kv_connector.v1 import ( from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats from vllm.logger import init_logger -from vllm.model_executor.layers.fused_moe.routed_experts_capturer import ( - RoutedExpertsManager, -) from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry from vllm.multimodal.encoder_budget import MultiModalBudget from vllm.multimodal.utils import get_mm_features_in_window @@ -393,26 +391,13 @@ class Scheduler(SchedulerInterface): if self.log_stats and vllm_config.observability_config.enable_mfu_metrics: self.perf_metrics = ModelMetrics(vllm_config) - self.enable_return_routed_experts = ( - vllm_config.model_config.enable_return_routed_experts + self.aux_output_connector = ( + AuxOutputSchedulerConnector() + if vllm_config.aux_output_config.enabled + else None ) self.return_sampling_mask = vllm_config.model_config.return_sampling_mask - if self.enable_return_routed_experts: - assert self.dcp_world_size == 1 and self.pcp_world_size == 1, ( - "enable_return_routed_experts does not support context parallelism " - "(dcp_world_size > 1 or pcp_world_size > 1)" - ) - - self.routed_experts_mgr = RoutedExpertsManager( - vllm_config=vllm_config, - kv_cache_config=kv_cache_config, - ) - # Block-ID snapshot taken at schedule time (before forward), - # so update_from_output can read slot data even if a later - # schedule() frees the blocks (async scheduling race). - self._re_block_ids: dict[str, list[int]] = {} - self._pause_state: PauseState = PauseState.UNPAUSED # In-flight requests still prefilling (prefill chunks + in-progress @@ -1487,6 +1472,13 @@ class Scheduler(SchedulerInterface): meta = self._build_kv_connector_meta(self.connector, scheduler_output) scheduler_output.kv_connector_metadata = meta + if self.aux_output_connector is not None: + scheduler_output.aux_output_connector_metadata = ( + self.aux_output_connector.build_connector_meta( + scheduler_output, self.requests + ) + ) + # Build the connector meta for ECConnector if self.ec_connector is not None: ec_meta: ECConnectorMetadata = self.ec_connector.build_connector_meta( @@ -1541,6 +1533,8 @@ class Scheduler(SchedulerInterface): assert request.status == RequestStatus.RUNNING, ( "Only running requests can be preempted" ) + if self.aux_output_connector is not None: + self.aux_output_connector.request_finished(request) self._free_request_blocks(request) self.encoder_cache_manager.free(request) self._inflight_prefills.discard(request) @@ -1596,22 +1590,6 @@ class Scheduler(SchedulerInterface): if not request.is_prefill_chunk: self._inflight_prefills.discard(request) - # Snapshot block IDs for routed experts before forward starts. - # A concurrent schedule() may preempt requests and free blocks - # before update_from_output runs; the snapshot survives that. - # Use update() to preserve entries from the previous step that - # have not yet been consumed by update_from_output (async - # scheduling may call _update_after_schedule again before the - # prior update_from_output runs). - if self.enable_return_routed_experts: - gid = self.routed_experts_mgr.attn_gid - self._re_block_ids.update( - { - rid: self.kv_cache_manager.get_blocks(rid).get_block_ids()[gid] - for rid in num_scheduled_tokens - } - ) - # Clear the finished and preempted request IDs. # NOTE: We shouldn't just clear() here because it will also affect # the scheduler output. @@ -1962,28 +1940,6 @@ class Scheduler(SchedulerInterface): num_scheduled_tokens, ) ) - - # Persist per-step routed experts into the scheduler-side slot - # buffer (CPU->CPU fancy-index assign; ~few MB per step). - # MUST precede the per-request routing reads below: stopped - # requests may terminate on tokens generated in this very step, - # whose routing was just D2H'd into model_runner_output. - routing_data = None - routing_offsets: dict[str, int] = {} - if model_runner_output.routed_experts is not None: - re = model_runner_output.routed_experts - self.routed_experts_mgr.store_batch(re.routing_data, re.slot_mapping) - routing_data = re.routing_data.astype( - self.routed_experts_mgr.routed_experts_by_slot.dtype, - copy=False, - ) - # Build offset map using model runner's request order - # (input_batch ordering), NOT scheduler dict order. - offset = 0 - for rid in model_runner_output.req_ids: - routing_offsets[rid] = offset - offset += num_scheduled_tokens[rid] - # NOTE(woosuk): As len(num_scheduled_tokens) can be up to 1K or more, # the below loop can be a performance bottleneck. We should do our best # to avoid expensive operations inside the loop. @@ -2076,7 +2032,6 @@ class Scheduler(SchedulerInterface): ec_transfer_params = None prefill_stats = None status_before_stop = request.status - num_output_tokens_before = len(request._output_token_ids) # Check for stop and update request status. if new_token_ids: @@ -2113,48 +2068,13 @@ class Scheduler(SchedulerInterface): stopped = True routed_experts = None - if ( - self.enable_return_routed_experts - and routing_data is not None - and new_token_ids - ): - req_offset = routing_offsets[req_id] - end = req_offset + num_tokens_scheduled - block_ids = self._re_block_ids.pop(req_id, []) - if num_output_tokens_before == 0: - # Prefill completed: read full prompt routing from - # slot buffer using the block-ID snapshot taken at - # schedule time (immune to async preemption). - if ( - request.sampling_params is not None - and request.sampling_params.routed_experts_prompt_start - is not None - ): - prompt_start = ( - request.sampling_params.routed_experts_prompt_start - ) - assert prompt_start <= request.num_prompt_tokens - else: - prompt_start = 0 - routed_experts = self.routed_experts_mgr.get( - block_ids, - request.num_prompt_tokens, - token_start=prompt_start, - ) - else: - if scheduled_spec_token_ids: - # Spec decode: accepted tokens at the START of - # the scheduled range, rejected at the end. - routed_experts = routing_data[ - req_offset : req_offset + len(new_token_ids) - ] - else: - # Normal decode / re-prefill: token(s) at the END. - routed_experts = routing_data[end - len(new_token_ids) : end] - should_emit_output = bool( new_token_ids or pooler_output is not None or stopped ) + if self.aux_output_connector is not None and should_emit_output: + routed_experts = self.aux_output_connector.take_output( + request, model_runner_output.aux_output_connector_output + ) if should_emit_output: prefill_stats = request.take_prefill_stats() if prefill_stats is not None: @@ -2641,6 +2561,8 @@ class Scheduler(SchedulerInterface): ) -> tuple[dict[str, Any] | None, dict[str, Any] | None]: assert request.is_finished() + if self.aux_output_connector is not None: + self.aux_output_connector.request_finished(request) self._inflight_prefills.discard(request) connector_delay_free_blocks, kv_xfer_params = self._connector_finished(request) @@ -2773,6 +2695,16 @@ class Scheduler(SchedulerInterface): Otherwise, this method will only reset the KV prefix cache when there is no running requests taking KV cache. """ + if reset_running_requests and self.aux_output_connector is not None: + if self._pause_state != PauseState.PAUSED_ALL: + raise RuntimeError( + "AuxOutput Connector only supports resetting running requests " + "after pause(mode='keep')." + ) + if any(request.num_in_flight_tokens for request in self.requests.values()): + raise RuntimeError( + "AuxOutput Connector cannot reset while model output is in flight." + ) if reset_running_requests: # For logging. timestamp = time.monotonic() @@ -2803,6 +2735,9 @@ class Scheduler(SchedulerInterface): if reset_connector: reset_successful = self.reset_connector_cache() and reset_successful + if reset_successful and self.aux_output_connector is not None: + self.aux_output_connector.reset() + return reset_successful def reset_connector_cache(self) -> bool: diff --git a/vllm/v1/executor/ray_utils.py b/vllm/v1/executor/ray_utils.py index e609fb6e7e..a5edb67dee 100644 --- a/vllm/v1/executor/ray_utils.py +++ b/vllm/v1/executor/ray_utils.py @@ -233,14 +233,10 @@ def detach_zero_copy_from_model_runner_output(output: "ModelRunnerOutput") -> No token_ids_c, logprobs_c, ranks_c, cu_num_generated_tokens ) - if output.routed_experts is not None: - routing_data, slot_mapping = output.routed_experts - routing_data_c = _copy_if_readonly(routing_data) - slot_mapping_c = _copy_if_readonly(slot_mapping) - if routing_data_c is not routing_data or slot_mapping_c is not slot_mapping: - output.routed_experts = type(output.routed_experts)( - routing_data_c, slot_mapping_c - ) + aux_output = output.aux_output_connector_output + if aux_output is not None: + for request_output in aux_output.values(): + request_output.rows = _copy_if_readonly(request_output.rows) class FutureWrapper(Future): diff --git a/vllm/v1/outputs.py b/vllm/v1/outputs.py index c4535f47f2..c38eafa4a4 100644 --- a/vllm/v1/outputs.py +++ b/vllm/v1/outputs.py @@ -15,6 +15,7 @@ from vllm.utils.torch_utils import PIN_MEMORY from vllm.v1.core.sched.output import SchedulerOutput if TYPE_CHECKING: + from vllm.distributed.aux_output_connector.connector import AuxRequestOutput from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorWorkerMetadata from vllm.distributed.ec_transfer.ec_connector.metrics import ECConnectorStats from vllm.distributed.kv_events import KVConnectorKVEvents @@ -23,6 +24,7 @@ if TYPE_CHECKING: ) from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats else: + AuxRequestOutput = object KVConnectorStats = object KVConnectorWorkerMetadata = object KVConnectorKVEvents = object @@ -188,73 +190,6 @@ class LogprobsTensors(NamedTuple): ) -class RoutedExpertsTensors(NamedTuple): - """Device-side snapshot of routed experts data, pending async D2H. - - Produced by :class:`GPUModelRunner` at the end of each async-scheduled - step. The copy stream waits on the default stream, then issues - non-blocking D2H via :meth:`to_cpu_nonblocking` into a pinned CPU - buffer; :class:`AsyncGPUModelRunnerOutput.get_output` synchronizes - the copy before the scheduler reads it. - - Sliced to ``total_num_scheduled_tokens`` (step-level, across all - requests — NOT per-request). Both ``routing_data`` and - ``slot_mapping`` must be private clones when sourced from shared - capturer / prepare-input buffers, so the next forward pass / - ``_prepare_inputs`` on the default stream does not race with a - D2H still pending on the copy stream. - """ - - # (num_scheduled_tokens, num_layers, num_experts_per_tok) - routing_data: torch.Tensor - # (num_scheduled_tokens,) - slot_mapping: torch.Tensor - - def to_cpu_nonblocking(self) -> "RoutedExpertsTensors": - """Issue non-blocking D2H on the current stream. - - NOTE: ``non_blocking=True`` only delivers true overlap when the - CPU target is pinned. The current fallback here allocates a - new pageable CPU tensor per call, which silently degrades to a - synchronous copy; acceptable because the sync happens on the - dedicated copy stream, not the default stream. - """ - if self.routing_data.device.type == "cpu": - return self - return RoutedExpertsTensors( - self.routing_data.to("cpu", non_blocking=True), - self.slot_mapping.to("cpu", non_blocking=True), - ) - - def tolists(self) -> "RoutedExpertsLists": - """Convert to the numpy-backed form consumed by the scheduler. - - ``.cpu()`` is a no-op when the tensor is already on CPU, so this - is cheap for the post-D2H case; for raw device tensors it will - synchronously block, which is only reached in tests. - """ - return RoutedExpertsLists( - self.routing_data.cpu().numpy(), - self.slot_mapping.cpu().numpy(), - ) - - -class RoutedExpertsLists(NamedTuple): - """CPU-side routed experts, the form :meth:`RoutedExpertsManager.store_batch` - consumes. - - Batched per scheduler step: the leading dim is the number of tokens - scheduled across all requests in this step (``total_num_scheduled_tokens``), - not per-request tokens. ``slot_mapping[i]`` tells the scheduler which - physical KV-cache slot row ``i`` of ``routing_data`` belongs to. - """ - - # (num_scheduled_tokens, num_layers, num_experts_per_tok) - routing_data: np.ndarray - # (num_scheduled_tokens,) - slot_mapping: np.ndarray - - # [num_reqs, ] # The shape of each element depends on the pooler used PoolerOutput: TypeAlias = torch.Tensor | list[torch.Tensor] | list[torch.Tensor | None] @@ -361,16 +296,7 @@ class ModelRunnerOutput: # information related to cudagraph execution cudagraph_stats: CUDAGraphStat | None = None - # Per-step routed experts data captured by the worker. - # ``routing_data`` shape: (num_scheduled_tokens, num_layers, - # num_experts_per_tok); expert IDs as uint8/uint16. - # ``slot_mapping`` shape: (num_scheduled_tokens,); physical KV-cache - # slot for each row of routing_data. - # ``num_scheduled_tokens`` is step-level (total across all requests - # in this step), not per-request. The scheduler persists this into - # its slot buffer via ``slot_buffer[slot_mapping] = routing_data``. - # ``None`` when ``enable_return_routed_experts`` is off. - routed_experts: RoutedExpertsLists | None = None + aux_output_connector_output: dict[str, AuxRequestOutput] | None = None # ``None`` when ``return_sampling_mask`` is off. sampling_masks: SamplingMaskLists | None = None diff --git a/vllm/v1/worker/gpu/async_utils.py b/vllm/v1/worker/gpu/async_utils.py index 0ab0ab3f3c..02592980a8 100644 --- a/vllm/v1/worker/gpu/async_utils.py +++ b/vllm/v1/worker/gpu/async_utils.py @@ -15,12 +15,12 @@ from vllm.v1.outputs import ( LogprobsTensors, ModelRunnerOutput, PoolerOutput, - RoutedExpertsTensors, ) from vllm.v1.worker.gpu.sample.output import SamplerOutput, SamplingMaskTensors from vllm.v1.worker.utils import raise_if_nan_logits if TYPE_CHECKING: + from vllm.distributed.aux_output_connector.worker import PendingAuxOutput from vllm.v1.worker.gpu.input_batch import InputBatch @@ -120,8 +120,8 @@ class AsyncOutput(AsyncModelRunnerOutput): num_sampled_tokens: torch.Tensor, main_stream: torch.cuda.Stream, copy_stream: torch.cuda.Stream, - check_ep_fault: bool = False, - routed_experts: RoutedExpertsTensors | None = None, + check_ep_fault: bool, + pending_aux_output: "PendingAuxOutput | None", ): # NOTE(woosuk): We must retain references to the GPU tensors, # as the copy operations are performed on a different CUDA stream than @@ -129,7 +129,7 @@ class AsyncOutput(AsyncModelRunnerOutput): self.model_runner_output = model_runner_output self.sampler_output = sampler_output self.num_sampled_tokens = num_sampled_tokens - self.routed_experts = routed_experts + self.pending_aux_output = pending_aux_output # Blocking (sleep) event to avoid busy-polling the CUDA driver lock. self.copy_event = torch.cuda.Event(blocking=True) self._has_fault: torch.Tensor | None = None @@ -152,13 +152,15 @@ class AsyncOutput(AsyncModelRunnerOutput): self.sampling_mask_tensors = ( sampler_output.sampling_mask_tensors.to_cpu_nonblocking() ) - self.routed_experts_cpu: RoutedExpertsTensors | None = None - if routed_experts is not None: - self.routed_experts_cpu = routed_experts.to_cpu_nonblocking() self.prompt_logprobs_dict = { k: v.to_cpu_nonblocking() if v is not None else None for k, v in self.model_runner_output.prompt_logprobs_dict.items() } + if self.pending_aux_output is not None: + self.pending_aux_output.enqueue_cpu_copy( + num_sampled=self.num_sampled_tokens_np, + num_rejected=async_copy_to_np(sampler_output.num_rejected), + ) if check_ep_fault: has_fault = get_ep_all2all_manager().query_fault() self._has_fault = has_fault.to("cpu", non_blocking=True) @@ -192,8 +194,10 @@ class AsyncOutput(AsyncModelRunnerOutput): if self.logprobs_tensors is not None: self.model_runner_output.logprobs = self.logprobs_tensors.tolists() self.model_runner_output.prompt_logprobs_dict = self.prompt_logprobs_dict - if self.routed_experts_cpu is not None: - self.model_runner_output.routed_experts = self.routed_experts_cpu.tolists() + if self.pending_aux_output is not None: + self.model_runner_output.aux_output_connector_output = ( + self.pending_aux_output.process_output() + ) if self._has_fault is not None and self._has_fault.item(): mask = get_ep_all2all_manager().query_active_mask() diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 8c736394cc..1ba40c5fda 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -32,6 +32,10 @@ from vllm.compilation.counter import compilation_counter from vllm.compilation.cuda_graph import CUDAGraphStat from vllm.config import VllmConfig from vllm.config.compilation import CUDAGraphMode +from vllm.distributed.aux_output_connector.worker import ( + AuxOutputWorkerConnector, + get_aux_output_connector, +) from vllm.distributed.parallel_state import ( get_dcp_group, get_pp_group, @@ -39,10 +43,6 @@ from vllm.distributed.parallel_state import ( from vllm.forward_context import BatchDescriptor, set_forward_context from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe.all2all_utils import get_ep_all2all_manager -from vllm.model_executor.layers.fused_moe.routed_experts_capturer import ( - RoutedExpertsCapturer, - bind_routed_experts_capturer, -) from vllm.model_executor.layers.mamba.ops.ssu_dispatch import ( initialize_mamba_ssu_backend, ) @@ -73,7 +73,6 @@ from vllm.v1.outputs import ( DraftTokenIds, ECConnectorOutput, ModelRunnerOutput, - RoutedExpertsTensors, ) from vllm.v1.watermarking import create_watermarker from vllm.v1.watermarking.gpu_sampler import GPUWatermarkSampler @@ -348,7 +347,8 @@ class GPUModelRunner(LoRAModelRunnerMixin): # Expert parallelism load balancer. self.eplb = EPLBController(self.parallel_config, self.device) - self.routed_experts_capturer: RoutedExpertsCapturer | None = None + # The AuxOutput Connector owns R3 capture, copying, and storage. + self.aux_output_connector: AuxOutputWorkerConnector | None = None set_offloader(create_offloader(self.vllm_config.offload_config)) @@ -356,15 +356,6 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.max_model_len = max_model_len self.req_states.max_model_len = max_model_len - def init_routed_experts_capturer(self) -> None: - """Initialize target-model capture on every participating worker.""" - self.routed_experts_capturer = RoutedExpertsCapturer( - max_num_batched_tokens=self.max_num_tokens, - vllm_config=self.vllm_config, - kv_cache_config=self.kv_cache_config, - ) - bind_routed_experts_capturer(self.model, self.routed_experts_capturer) - def get_supported_tasks(self) -> tuple[SupportedTask, ...]: tasks: list[SupportedTask] = [] if self.model_config.runner_type == "generate": @@ -756,6 +747,12 @@ class GPUModelRunner(LoRAModelRunnerMixin): else: self.kv_connector = get_kv_connector(self.vllm_config, kv_caches_dict) + # AuxOutput connector requires resolved kv_cache_config. + if self.vllm_config.aux_output_config.enabled: + self.aux_output_connector = get_aux_output_connector( + self.model, self.vllm_config, kv_cache_config + ) + def _init_kv_zero_meta(self) -> None: """Build KV-block zeroing metadata; invoked from gpu_worker.""" self.kv_block_zeroer = KVBlockZeroer( @@ -1648,6 +1645,11 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.add_requests(scheduler_output) self.update_requests(scheduler_output) self.block_tables.apply_staged_writes() + if self.aux_output_connector is not None: + # Register this step before the GPU forward. + self.aux_output_connector.begin_step( + scheduler_output.aux_output_connector_metadata + ) if scheduler_output.total_num_scheduled_tokens == 0: # No need to run the model. empty_output = self.kv_connector.no_forward(scheduler_output) @@ -1969,11 +1971,6 @@ class GPUModelRunner(LoRAModelRunnerMixin): aux_hidden_states = None output_intermediate_tensors = model_output - routed_experts = None - if not dummy_run and (capturer := self.routed_experts_capturer) is not None: - assert slot_mappings is not None - routed_experts = capturer.get_routed_experts(slot_mappings, num_toks) - finished_req_ids = scheduler_output.finished_req_ids self.execute_model_state = ExecuteModelState( input_batch=input_batch, @@ -1984,7 +1981,6 @@ class GPUModelRunner(LoRAModelRunnerMixin): dp_sync=dp_sync, finished_req_ids=finished_req_ids, ec_connector_output=ec_connector_output, - routed_experts=routed_experts, cudagraph_stats=cudagraph_stats, ) @@ -2014,7 +2010,6 @@ class GPUModelRunner(LoRAModelRunnerMixin): dp_sync = self.execute_model_state.dp_sync finished_req_ids = self.execute_model_state.finished_req_ids ec_connector_output = self.execute_model_state.ec_connector_output - routed_experts = self.execute_model_state.routed_experts cudagraph_stats = self.execute_model_state.cudagraph_stats self.execute_model_state = None @@ -2083,6 +2078,10 @@ class GPUModelRunner(LoRAModelRunnerMixin): prompt_logprobs_dict=prompt_logprobs_dict, # type: ignore[arg-type] cudagraph_stats=cudagraph_stats, ) + pending_aux_output = None + if self.aux_output_connector is not None: + pending_aux_output = self.aux_output_connector.prepare_output(input_batch) + # Start async output copy here so that it can overlap with speculator proposal. async_output = AsyncOutput( model_runner_output=model_runner_output, @@ -2091,7 +2090,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): main_stream=self.main_stream, copy_stream=self.output_copy_stream, check_ep_fault=self.check_ep_fault, - routed_experts=routed_experts, + pending_aux_output=pending_aux_output, ) mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None @@ -2244,6 +2243,8 @@ class GPUModelRunner(LoRAModelRunnerMixin): """Release GPU tensors (model weights, KV caches, workspace) so that memory is reclaimable when running in the same process.""" torch.accelerator.synchronize() + if self.aux_output_connector is not None: + self.aux_output_connector.close() self.cudagraph_manager = None self.fast_prefill = None if hasattr(self, "kv_caches"): @@ -2316,7 +2317,6 @@ class ExecuteModelState(NamedTuple): dp_sync: DPSyncState | None finished_req_ids: set[str] ec_connector_output: ECConnectorOutput | None - routed_experts: RoutedExpertsTensors | None cudagraph_stats: CUDAGraphStat | None diff --git a/vllm/v1/worker/gpu/sample/batch_shard.py b/vllm/v1/worker/gpu/sample/batch_shard.py index b2803de9da..3d41c49720 100644 --- a/vllm/v1/worker/gpu/sample/batch_shard.py +++ b/vllm/v1/worker/gpu/sample/batch_shard.py @@ -617,8 +617,6 @@ def gather_sampler_output( device=device, ) if local_output is not None: - assert local_output.num_sampled is not None - assert local_output.num_rejected is not None assert not gather_num_nans or local_output.num_nans is not None num_src_cols = min( local_output.sampled_token_ids.shape[1], max_num_logits_per_req diff --git a/vllm/v1/worker/gpu/sample/output.py b/vllm/v1/worker/gpu/sample/output.py index ff3675db13..aedff5ce18 100644 --- a/vllm/v1/worker/gpu/sample/output.py +++ b/vllm/v1/worker/gpu/sample/output.py @@ -17,8 +17,8 @@ class SamplerOutput: sampled_token_ids: torch.Tensor logprobs_tensors: LogprobsTensors | None num_nans: torch.Tensor | None - num_sampled: torch.Tensor | None - num_rejected: torch.Tensor | None = None + num_sampled: torch.Tensor + num_rejected: torch.Tensor sampling_mask_tensors: SamplingMaskTensors | None = None diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 7d933917bc..bdabe398f6 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -61,10 +61,6 @@ from vllm.lora.layers import BaseLayerWithLoRA, LoRAMapping, LoRAMappingType from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase from vllm.model_executor.layers.fused_moe.all2all_utils import get_ep_all2all_manager -from vllm.model_executor.layers.fused_moe.routed_experts_capturer import ( - RoutedExpertsCapturer, - bind_routed_experts_capturer, -) from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFuncsByType from vllm.model_executor.layers.mamba.ops.ssu_dispatch import ( initialize_mamba_ssu_backend, @@ -176,8 +172,6 @@ from vllm.v1.outputs import ( LogprobsTensors, ModelRunnerOutput, PoolerOutput, - RoutedExpertsLists, - RoutedExpertsTensors, SamplerOutput, make_empty_encoder_model_runner_output, ) @@ -293,7 +287,6 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput): invalid_req_indices: list[int], async_output_copy_stream: torch.cuda.Stream, vocab_size: int, - routed_experts: RoutedExpertsTensors | None = None, check_ep_fault: bool = False, num_nans: torch.Tensor | None = None, ): @@ -309,7 +302,6 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput): self._sampled_token_ids = sampled_token_ids self.vocab_size = vocab_size self._logprobs_tensors = logprobs_tensors - self._routed_experts = routed_experts self._num_nans = num_nans self._has_fault: torch.Tensor | None = None @@ -325,11 +317,6 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput): if self._logprobs_tensors else None ) - self._routed_experts_cpu = ( - self._routed_experts.to_cpu_nonblocking() - if self._routed_experts is not None - else None - ) self._num_nans_cpu = ( self._num_nans.to("cpu", non_blocking=True) if self._num_nans is not None @@ -370,10 +357,6 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput): output.sampled_token_ids = valid_sampled_token_ids output.logprobs = logprobs_lists - if self._routed_experts_cpu is not None: - output.routed_experts = self._routed_experts_cpu.tolists() - del self._routed_experts - if self._num_nans_cpu is not None: output.num_nans_in_logits = nans_to_dict( self._num_nans_cpu.tolist(), output.req_id_to_index @@ -537,9 +520,6 @@ class GPUModelRunner( # These will be overridden in load_model() self.is_multimodal_pruning_enabled = False self.requires_sequential_video_encoding = False - # Set to True after init_routed_experts_capturer() completes. - # Prevents routed experts code from running during profiling/dummy run. - self.routed_experts_initialized = False self.max_model_len = model_config.max_model_len # Always set to false after the first forward pass @@ -2360,18 +2340,6 @@ class GPUModelRunner( block_table_gid_0 = _get_block_table(0) slot_mapping_gid_0 = slot_mappings[0] - if self.routed_experts_initialized: - # Copy this step's attention slot_mapping into our private - # device buffer. The shared ``slot_mappings[attn_gid]`` is - # owned by the attention block table and will be overwritten - # by the next ``_prepare_inputs``; we need a stable snapshot - # because the async D2H may still be in flight on the copy - # stream when the next step runs. - slot_mapping_attn = slot_mappings[self.routed_experts_capturer.attn_gid] - self.routed_experts_slot_mapping_device[:num_tokens].copy_( - slot_mapping_attn[:num_tokens] - ) - num_computed_tokens_cpu = self.input_batch.num_computed_tokens_cpu_tensor[ :num_reqs_padded ] @@ -3745,20 +3713,6 @@ class GPUModelRunner( invalid_req_indices = [] logprobs_lists = None if not self.use_async_scheduling: - # Sync scheduling: issue routed experts D2H into the pinned - # CPU buffer BEFORE ``_to_list`` below. ``_to_list`` does - # ``event.synchronize()`` on the async copy stream which - # waits for every D2H queued on the default stream since - # the last sync, so this enqueue is naturally covered - # without requiring its own synchronize. - if self.routed_experts_initialized: - buf = self.routed_experts_capturer.get_device_buffer() - total = scheduler_output.total_num_scheduled_tokens - self.routed_experts_cpu[:total].copy_(buf[:total], non_blocking=True) - self.routed_experts_slot_mapping_cpu[:total].copy_( - self.routed_experts_slot_mapping_device[:total], - non_blocking=True, - ) with gpu_sync_allowed(): # Get the valid generated tokens. max_gen_len = sampled_token_ids.shape[-1] @@ -4802,39 +4756,14 @@ class GPUModelRunner( else None, num_nans_in_logits=num_nans_in_logits, cudagraph_stats=cudagraph_stats, - routed_experts=None, ) if not self.use_async_scheduling: - if self.routed_experts_initialized: - # Sync path: D2H was issued in ``_bookkeeping_sync`` and - # synchronized by ``_to_list``'s event.synchronize(), so - # the pinned buffers are ready to be wrapped as numpy. - total = scheduler_output.total_num_scheduled_tokens - output.routed_experts = RoutedExpertsLists( - routing_data=self.routed_experts_cpu[:total].numpy(), - slot_mapping=self.routed_experts_slot_mapping_cpu[:total].numpy(), - ) return output with record_function_or_nullcontext( "gpu_model_runner: AsyncGPUModelRunnerOutput" ): - # Async path: produce a device-side snapshot that the async - # copy stream can D2H later. Both tensors must be private - # clones because: - # - ``routing_data`` source is the shared capturer buffer, - # which the next forward overwrites on the default stream. - # - ``slot_mapping`` source is our own - # ``routed_experts_slot_mapping_device``, which the - # next ``_prepare_inputs`` overwrites on the default - # stream while the D2H is still pending on the copy - # stream. - # Without clones, the copy stream would read torn data. - routed_experts_snapshot = self.get_routed_experts( - scheduler_output.total_num_scheduled_tokens - ) - async_output = AsyncGPUModelRunnerOutput( model_runner_output=output, sampled_token_ids=sampler_output.sampled_token_ids, @@ -4842,7 +4771,6 @@ class GPUModelRunner( invalid_req_indices=invalid_req_indices, async_output_copy_stream=self._get_or_create_async_output_copy_stream(), vocab_size=self.input_batch.vocab_size, - routed_experts=routed_experts_snapshot, check_ep_fault=self.check_ep_fault, num_nans=num_nans_device, ) @@ -7468,61 +7396,6 @@ class GPUModelRunner( kv_transfer_group.register_kv_caches(kv_caches) kv_transfer_group.set_host_xfer_buffer_ops(copy_kv_blocks) - def get_routed_experts( - self, - num_tokens: int, - ) -> RoutedExpertsTensors | None: - if not self.routed_experts_initialized: - return None - - device_buffer = self.routed_experts_capturer.get_device_buffer() - return RoutedExpertsTensors( - routing_data=device_buffer[:num_tokens].clone(), - slot_mapping=self.routed_experts_slot_mapping_device[:num_tokens].clone(), - ) - - def init_routed_experts_capturer(self): - logger.info( - "Initializing routed experts capturer, enable_return_routed_experts: %s", - self.model_config.enable_return_routed_experts, - ) - self.routed_experts_capturer = RoutedExpertsCapturer( - max_num_batched_tokens=self.scheduler_config.max_num_batched_tokens, - vllm_config=self.vllm_config, - kv_cache_config=self.kv_cache_config, - ) - bind_routed_experts_capturer(self.model, self.routed_experts_capturer) - - # Pinned CPU buffer for non-blocking D2H of ``routing_data`` on - # the sync scheduling path. Shape / dtype mirror the device - # capturer exactly so ``copy_`` is a straight memcpy. - self.routed_experts_cpu = torch.empty( - self.routed_experts_capturer.device_buffer.shape, - dtype=self.routed_experts_capturer.device_buffer.dtype, - device="cpu", - pin_memory=PIN_MEMORY, - ) - # ``slot_mapping`` dtype is fixed to int64 by - # ``block_table.slot_mapping``; we mirror that here. - max_tokens = self.scheduler_config.max_num_batched_tokens - self.routed_experts_slot_mapping_cpu = torch.empty( - (max_tokens,), - dtype=torch.int64, - device="cpu", - pin_memory=PIN_MEMORY, - ) - # Private device buffer so the shared ``block_table.slot_mapping`` - # can be overwritten by the next ``_prepare_inputs`` while the - # D2H is still pending on the copy stream. Written in - # ``_prepare_inputs``, read in ``_bookkeeping_sync`` (sync path) - # or cloned into a snapshot (async path). - self.routed_experts_slot_mapping_device = torch.empty( - (max_tokens,), - dtype=torch.int64, - device=self.device, - ) - self.routed_experts_initialized = True - def may_add_encoder_only_layers_to_kv_cache_config(self) -> None: """Add encoder-only layers to the KV cache config.""" block_size = self.vllm_config.cache_config.block_size diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index d914bd2dea..b6bc122a28 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -761,9 +761,6 @@ class Worker(WorkerBase): ), ) - if self.model_config.enable_return_routed_experts: - self.model_runner.init_routed_experts_capturer() - # Build KV-zero metadata outside the CuMem pool so the bookkeeping # GPU tensors (seg_addrs, block-id buffers) use the standard PyTorch # allocator and are not discarded during sleep/wake cycles.