From 755513d9a24fb639a8985172e645c7fe5a263154 Mon Sep 17 00:00:00 2001 From: Francesco Fusco Date: Mon, 3 Aug 2026 22:24:22 +0200 Subject: [PATCH] [Hybrid] Stage the postprocess inputs with a single loop over the request list (#48120) Signed-off-by: Francesco Fusco Co-authored-by: Claude --- tests/v1/worker/test_mamba_utils.py | 118 ++++++++++++++++++++++++++++ vllm/v1/worker/mamba_utils.py | 109 ++++++++----------------- 2 files changed, 149 insertions(+), 78 deletions(-) diff --git a/tests/v1/worker/test_mamba_utils.py b/tests/v1/worker/test_mamba_utils.py index 4bbab10b48..a338b934b5 100644 --- a/tests/v1/worker/test_mamba_utils.py +++ b/tests/v1/worker/test_mamba_utils.py @@ -21,6 +21,7 @@ from vllm.v1.worker.mamba_utils import ( collect_mamba_copy_meta, do_mamba_copy_block, preprocess_mamba, + stage_postprocess_inputs_to_gpu, ) MambaStateCopyFunc = Callable[..., Any] @@ -371,6 +372,123 @@ def _make_gpu_ctx( ) +# ----------------------------------------------------------------------------- +# stage_postprocess_inputs_to_gpu: single-pass staging into pinned views +# ----------------------------------------------------------------------------- + + +def _make_staging_ctx(max_num_reqs: int, device: torch.device) -> MagicMock: + """Build a MambaSpecDecodeGPUContext stand-in exposing only the four + per-request staging buffers touched by stage_postprocess_inputs_to_gpu.""" + ctx = MagicMock() + ctx.mamba_state_idx_buf = _MockCpuGpuBuffer(max_num_reqs, torch.int32, device) + ctx.num_scheduled_tokens_buf = _MockCpuGpuBuffer(max_num_reqs, torch.int32, device) + ctx.num_computed_tokens_buf = _MockCpuGpuBuffer(max_num_reqs, torch.int32, device) + ctx.num_draft_tokens_buf = _MockCpuGpuBuffer(max_num_reqs, torch.int32, device) + return ctx + + +def test_stage_postprocess_inputs_to_gpu_fills_pinned_views(): + """stage_postprocess_inputs_to_gpu writes each request's state_idx and + scheduled/computed/draft counts into slot `i` for `req_ids[i]`, leaves + slots past `num_reqs` untouched, and mirrors the values to the GPU buffers.""" + device = torch.device("cpu") + max_num_reqs = 8 + ctx = _make_staging_ctx(max_num_reqs, device) + + # Any negative int32 works as a sentinel: all staged values (state_idx, + # scheduled/computed/draft token counts) are non-negative, so a negative + # entry surviving in slots [num_reqs:] proves the loop didn't overrun. + sentinel = np.int32(-1) + bufs = ( + ctx.mamba_state_idx_buf, + ctx.num_scheduled_tokens_buf, + ctx.num_computed_tokens_buf, + ctx.num_draft_tokens_buf, + ) + for buf in bufs: + buf.np[:] = sentinel + + req_ids = ["req_a", "req_b", "req_c"] + num_reqs = len(req_ids) + scheduler_output = _make_postprocess_scheduler_output( + req_ids=req_ids, + num_scheduled_tokens={"req_a": 5, "req_b": 12, "req_c": 3}, + # req_b intentionally absent -> defaults to 0 drafts. + scheduled_spec_decode_tokens={ + "req_a": [1, 2], + "req_c": [1, 2, 3, 4], + }, + ) + requests = _make_requests( + req_ids=req_ids, + num_computed_tokens=[10, 20, 30], + block_ids_per_req=[[0], [0], [0]], + ) + mamba_state_idx = {"req_a": 100, "req_b": 200, "req_c": 300} + # A trailing entry past num_reqs must not be read. + req_ids_padded = req_ids + ["not_scheduled"] + + stage_postprocess_inputs_to_gpu( + ctx, + scheduler_output, + req_ids_padded, + num_reqs, + requests, + mamba_state_idx, + ) + + np.testing.assert_array_equal( + ctx.mamba_state_idx_buf.np[:num_reqs], [100, 200, 300] + ) + np.testing.assert_array_equal( + ctx.num_scheduled_tokens_buf.np[:num_reqs], [5, 12, 3] + ) + np.testing.assert_array_equal( + ctx.num_computed_tokens_buf.np[:num_reqs], [10, 20, 30] + ) + np.testing.assert_array_equal(ctx.num_draft_tokens_buf.np[:num_reqs], [2, 0, 4]) + for buf in bufs: + assert (buf.np[num_reqs:] == sentinel).all() + + assert torch.equal( + ctx.mamba_state_idx_buf.gpu[:num_reqs], + torch.tensor([100, 200, 300], dtype=torch.int32), + ) + assert torch.equal( + ctx.num_draft_tokens_buf.gpu[:num_reqs], + torch.tensor([2, 0, 4], dtype=torch.int32), + ) + + +def test_stage_postprocess_inputs_to_gpu_asserts_on_missing_state_idx(): + """If preprocess_mamba didn't populate mamba_state_idx for a req in the + batch, staging must fail loudly rather than silently writing a stale index.""" + device = torch.device("cpu") + ctx = _make_staging_ctx(max_num_reqs=4, device=device) + scheduler_output = _make_postprocess_scheduler_output( + req_ids=["req_a"], + num_scheduled_tokens={"req_a": 1}, + ) + requests = _make_requests(["req_a"], [0], [[0]]) + + # mamba_state_idx has an entry for a *different* request but not for + # req_a (the one being staged). This is the realistic failure mode: + # preprocess_mamba ran, populated some reqs, but missed this one. + # The safety assert in stage_postprocess_inputs_to_gpu must fire on + # req_a's missing entry rather than silently writing None. + mamba_state_idx: dict[str, int] = {"other_req": 42} + with pytest.raises(AssertionError, match="mamba_state_idx missing entry"): + stage_postprocess_inputs_to_gpu( + ctx, + scheduler_output, + ["req_a"], + 1, + requests, + mamba_state_idx, + ) + + def _run_gpu_postprocess( gpu_ctx: MambaSpecDecodeGPUContext, *, diff --git a/vllm/v1/worker/mamba_utils.py b/vllm/v1/worker/mamba_utils.py index fd2075bf8a..131432fffb 100644 --- a/vllm/v1/worker/mamba_utils.py +++ b/vllm/v1/worker/mamba_utils.py @@ -1238,66 +1238,6 @@ def postprocess_mamba_align_gpu( ) -def stage_postprocess_metadata_to_gpu( - scheduler_output: SchedulerOutput, - req_ids: list[str], - num_reqs: int, - requests: dict[str, CachedRequestState], - num_scheduled_tokens_buf: CpuGpuBuffer, - num_computed_tokens_buf: CpuGpuBuffer, - num_draft_tokens_buf: CpuGpuBuffer, -) -> None: - """Stage per-request postprocess metadata into GPU buffers (non-blocking). - - Walks ``req_ids[:num_reqs]`` in batch order and writes each request's - scheduled/computed/draft token counts into the matching pinned numpy - views, then issues three non-blocking H→D copies. These values don't - change between ``_prepare_inputs`` and ``_update_states_after_model_execute``. - The fused postprocess kernel indexes the resulting GPU tensors - by ``req_idx``. - """ - scheduled_spec_tokens = scheduler_output.scheduled_spec_decode_tokens - num_scheduled = scheduler_output.num_scheduled_tokens - scheduled_np = num_scheduled_tokens_buf.np - computed_np = num_computed_tokens_buf.np - draft_np = num_draft_tokens_buf.np - for i in range(num_reqs): - req_id = req_ids[i] - scheduled_np[i] = num_scheduled[req_id] - computed_np[i] = requests[req_id].num_computed_tokens - draft_np[i] = len(scheduled_spec_tokens.get(req_id, [])) - num_scheduled_tokens_buf.copy_to_gpu(num_reqs) - num_computed_tokens_buf.copy_to_gpu(num_reqs) - num_draft_tokens_buf.copy_to_gpu(num_reqs) - - -def stage_mamba_state_idx_to_gpu( - mamba_state_idx: dict[str, int], - req_ids: list[str], - num_reqs: int, - gpu_buf: CpuGpuBuffer, -) -> None: - """Materialize ``mamba_state_idx`` into ``gpu_buf`` and copy to GPU. - - Walks ``req_ids[:num_reqs]`` in batch order, writing each request's block - index into the buffer's pinned numpy view, then issues a non-blocking H→D - copy. The fused kernel indexes the resulting GPU tensor by ``req_idx``. - - Invariant: ``preprocess_mamba`` must have run first for the same batch so - that every ``req_ids[i]`` has an entry in ``mamba_state_idx``. - """ - np_view = gpu_buf.np - for i in range(num_reqs): - req_id = req_ids[i] - state_idx = mamba_state_idx.get(req_id) - assert state_idx is not None, ( - f"mamba_state_idx missing entry for {req_id!r}; " - "preprocess_mamba must run before stage_mamba_state_idx_to_gpu" - ) - np_view[i] = state_idx - gpu_buf.copy_to_gpu(num_reqs) - - def stage_postprocess_inputs_to_gpu( ctx: MambaSpecDecodeGPUContext, scheduler_output: SchedulerOutput, @@ -1308,27 +1248,40 @@ def stage_postprocess_inputs_to_gpu( ) -> None: """Stage all per-request inputs the fused mamba postprocess kernel reads. - Bundles ``stage_mamba_state_idx_to_gpu`` and - ``stage_postprocess_metadata_to_gpu`` into a single call so the runner - has one entry point for postprocess staging. Buffers live on ``ctx`` + Walks ``req_ids[:num_reqs]`` once, writing each request's mamba block + index and scheduled/computed/draft token counts into the matching pinned + numpy views, then issues four non-blocking H→D copies. The fused kernel + indexes the resulting GPU tensors by ``req_idx``. Buffers live on ``ctx`` and only exist when the postprocess kernel is enabled. + + Invariant: ``preprocess_mamba`` must have run first for the same batch so + that every ``req_ids[i]`` has an entry in ``mamba_state_idx``. """ assert ctx.mamba_state_idx_buf is not None assert ctx.num_scheduled_tokens_buf is not None assert ctx.num_computed_tokens_buf is not None assert ctx.num_draft_tokens_buf is not None - stage_mamba_state_idx_to_gpu( - mamba_state_idx, - req_ids, - num_reqs, - ctx.mamba_state_idx_buf, - ) - stage_postprocess_metadata_to_gpu( - scheduler_output, - req_ids, - num_reqs, - requests, - ctx.num_scheduled_tokens_buf, - ctx.num_computed_tokens_buf, - ctx.num_draft_tokens_buf, - ) + + scheduled_spec_tokens = scheduler_output.scheduled_spec_decode_tokens + num_scheduled = scheduler_output.num_scheduled_tokens + state_idx_np = ctx.mamba_state_idx_buf.np + scheduled_np = ctx.num_scheduled_tokens_buf.np + computed_np = ctx.num_computed_tokens_buf.np + draft_np = ctx.num_draft_tokens_buf.np + + for i in range(num_reqs): + req_id = req_ids[i] + state_idx = mamba_state_idx.get(req_id) + assert state_idx is not None, ( + f"mamba_state_idx missing entry for {req_id!r}; " + "preprocess_mamba must run before stage_postprocess_inputs_to_gpu" + ) + state_idx_np[i] = state_idx + scheduled_np[i] = num_scheduled[req_id] + computed_np[i] = requests[req_id].num_computed_tokens + draft_np[i] = len(scheduled_spec_tokens.get(req_id, [])) + + ctx.mamba_state_idx_buf.copy_to_gpu(num_reqs) + ctx.num_scheduled_tokens_buf.copy_to_gpu(num_reqs) + ctx.num_computed_tokens_buf.copy_to_gpu(num_reqs) + ctx.num_draft_tokens_buf.copy_to_gpu(num_reqs)