[Hybrid] Stage the postprocess inputs with a single loop over the request list (#48120)

Signed-off-by: Francesco Fusco <ffu@zurich.ibm.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Francesco Fusco
2026-08-03 20:24:22 +00:00
committed by GitHub
co-authored by Claude
parent 1c0d207915
commit 755513d9a2
2 changed files with 149 additions and 78 deletions
+118
View File
@@ -21,6 +21,7 @@ from vllm.v1.worker.mamba_utils import (
collect_mamba_copy_meta,
do_mamba_copy_block,
preprocess_mamba,
stage_postprocess_inputs_to_gpu,
)
MambaStateCopyFunc = Callable[..., Any]
@@ -371,6 +372,123 @@ def _make_gpu_ctx(
)
# -----------------------------------------------------------------------------
# stage_postprocess_inputs_to_gpu: single-pass staging into pinned views
# -----------------------------------------------------------------------------
def _make_staging_ctx(max_num_reqs: int, device: torch.device) -> MagicMock:
"""Build a MambaSpecDecodeGPUContext stand-in exposing only the four
per-request staging buffers touched by stage_postprocess_inputs_to_gpu."""
ctx = MagicMock()
ctx.mamba_state_idx_buf = _MockCpuGpuBuffer(max_num_reqs, torch.int32, device)
ctx.num_scheduled_tokens_buf = _MockCpuGpuBuffer(max_num_reqs, torch.int32, device)
ctx.num_computed_tokens_buf = _MockCpuGpuBuffer(max_num_reqs, torch.int32, device)
ctx.num_draft_tokens_buf = _MockCpuGpuBuffer(max_num_reqs, torch.int32, device)
return ctx
def test_stage_postprocess_inputs_to_gpu_fills_pinned_views():
"""stage_postprocess_inputs_to_gpu writes each request's state_idx and
scheduled/computed/draft counts into slot `i` for `req_ids[i]`, leaves
slots past `num_reqs` untouched, and mirrors the values to the GPU buffers."""
device = torch.device("cpu")
max_num_reqs = 8
ctx = _make_staging_ctx(max_num_reqs, device)
# Any negative int32 works as a sentinel: all staged values (state_idx,
# scheduled/computed/draft token counts) are non-negative, so a negative
# entry surviving in slots [num_reqs:] proves the loop didn't overrun.
sentinel = np.int32(-1)
bufs = (
ctx.mamba_state_idx_buf,
ctx.num_scheduled_tokens_buf,
ctx.num_computed_tokens_buf,
ctx.num_draft_tokens_buf,
)
for buf in bufs:
buf.np[:] = sentinel
req_ids = ["req_a", "req_b", "req_c"]
num_reqs = len(req_ids)
scheduler_output = _make_postprocess_scheduler_output(
req_ids=req_ids,
num_scheduled_tokens={"req_a": 5, "req_b": 12, "req_c": 3},
# req_b intentionally absent -> defaults to 0 drafts.
scheduled_spec_decode_tokens={
"req_a": [1, 2],
"req_c": [1, 2, 3, 4],
},
)
requests = _make_requests(
req_ids=req_ids,
num_computed_tokens=[10, 20, 30],
block_ids_per_req=[[0], [0], [0]],
)
mamba_state_idx = {"req_a": 100, "req_b": 200, "req_c": 300}
# A trailing entry past num_reqs must not be read.
req_ids_padded = req_ids + ["not_scheduled"]
stage_postprocess_inputs_to_gpu(
ctx,
scheduler_output,
req_ids_padded,
num_reqs,
requests,
mamba_state_idx,
)
np.testing.assert_array_equal(
ctx.mamba_state_idx_buf.np[:num_reqs], [100, 200, 300]
)
np.testing.assert_array_equal(
ctx.num_scheduled_tokens_buf.np[:num_reqs], [5, 12, 3]
)
np.testing.assert_array_equal(
ctx.num_computed_tokens_buf.np[:num_reqs], [10, 20, 30]
)
np.testing.assert_array_equal(ctx.num_draft_tokens_buf.np[:num_reqs], [2, 0, 4])
for buf in bufs:
assert (buf.np[num_reqs:] == sentinel).all()
assert torch.equal(
ctx.mamba_state_idx_buf.gpu[:num_reqs],
torch.tensor([100, 200, 300], dtype=torch.int32),
)
assert torch.equal(
ctx.num_draft_tokens_buf.gpu[:num_reqs],
torch.tensor([2, 0, 4], dtype=torch.int32),
)
def test_stage_postprocess_inputs_to_gpu_asserts_on_missing_state_idx():
"""If preprocess_mamba didn't populate mamba_state_idx for a req in the
batch, staging must fail loudly rather than silently writing a stale index."""
device = torch.device("cpu")
ctx = _make_staging_ctx(max_num_reqs=4, device=device)
scheduler_output = _make_postprocess_scheduler_output(
req_ids=["req_a"],
num_scheduled_tokens={"req_a": 1},
)
requests = _make_requests(["req_a"], [0], [[0]])
# mamba_state_idx has an entry for a *different* request but not for
# req_a (the one being staged). This is the realistic failure mode:
# preprocess_mamba ran, populated some reqs, but missed this one.
# The safety assert in stage_postprocess_inputs_to_gpu must fire on
# req_a's missing entry rather than silently writing None.
mamba_state_idx: dict[str, int] = {"other_req": 42}
with pytest.raises(AssertionError, match="mamba_state_idx missing entry"):
stage_postprocess_inputs_to_gpu(
ctx,
scheduler_output,
["req_a"],
1,
requests,
mamba_state_idx,
)
def _run_gpu_postprocess(
gpu_ctx: MambaSpecDecodeGPUContext,
*,
+31 -78
View File
@@ -1238,66 +1238,6 @@ def postprocess_mamba_align_gpu(
)
def stage_postprocess_metadata_to_gpu(
scheduler_output: SchedulerOutput,
req_ids: list[str],
num_reqs: int,
requests: dict[str, CachedRequestState],
num_scheduled_tokens_buf: CpuGpuBuffer,
num_computed_tokens_buf: CpuGpuBuffer,
num_draft_tokens_buf: CpuGpuBuffer,
) -> None:
"""Stage per-request postprocess metadata into GPU buffers (non-blocking).
Walks ``req_ids[:num_reqs]`` in batch order and writes each request's
scheduled/computed/draft token counts into the matching pinned numpy
views, then issues three non-blocking H→D copies. These values don't
change between ``_prepare_inputs`` and ``_update_states_after_model_execute``.
The fused postprocess kernel indexes the resulting GPU tensors
by ``req_idx``.
"""
scheduled_spec_tokens = scheduler_output.scheduled_spec_decode_tokens
num_scheduled = scheduler_output.num_scheduled_tokens
scheduled_np = num_scheduled_tokens_buf.np
computed_np = num_computed_tokens_buf.np
draft_np = num_draft_tokens_buf.np
for i in range(num_reqs):
req_id = req_ids[i]
scheduled_np[i] = num_scheduled[req_id]
computed_np[i] = requests[req_id].num_computed_tokens
draft_np[i] = len(scheduled_spec_tokens.get(req_id, []))
num_scheduled_tokens_buf.copy_to_gpu(num_reqs)
num_computed_tokens_buf.copy_to_gpu(num_reqs)
num_draft_tokens_buf.copy_to_gpu(num_reqs)
def stage_mamba_state_idx_to_gpu(
mamba_state_idx: dict[str, int],
req_ids: list[str],
num_reqs: int,
gpu_buf: CpuGpuBuffer,
) -> None:
"""Materialize ``mamba_state_idx`` into ``gpu_buf`` and copy to GPU.
Walks ``req_ids[:num_reqs]`` in batch order, writing each request's block
index into the buffer's pinned numpy view, then issues a non-blocking H→D
copy. The fused kernel indexes the resulting GPU tensor by ``req_idx``.
Invariant: ``preprocess_mamba`` must have run first for the same batch so
that every ``req_ids[i]`` has an entry in ``mamba_state_idx``.
"""
np_view = gpu_buf.np
for i in range(num_reqs):
req_id = req_ids[i]
state_idx = mamba_state_idx.get(req_id)
assert state_idx is not None, (
f"mamba_state_idx missing entry for {req_id!r}; "
"preprocess_mamba must run before stage_mamba_state_idx_to_gpu"
)
np_view[i] = state_idx
gpu_buf.copy_to_gpu(num_reqs)
def stage_postprocess_inputs_to_gpu(
ctx: MambaSpecDecodeGPUContext,
scheduler_output: SchedulerOutput,
@@ -1308,27 +1248,40 @@ def stage_postprocess_inputs_to_gpu(
) -> None:
"""Stage all per-request inputs the fused mamba postprocess kernel reads.
Bundles ``stage_mamba_state_idx_to_gpu`` and
``stage_postprocess_metadata_to_gpu`` into a single call so the runner
has one entry point for postprocess staging. Buffers live on ``ctx``
Walks ``req_ids[:num_reqs]`` once, writing each request's mamba block
index and scheduled/computed/draft token counts into the matching pinned
numpy views, then issues four non-blocking H→D copies. The fused kernel
indexes the resulting GPU tensors by ``req_idx``. Buffers live on ``ctx``
and only exist when the postprocess kernel is enabled.
Invariant: ``preprocess_mamba`` must have run first for the same batch so
that every ``req_ids[i]`` has an entry in ``mamba_state_idx``.
"""
assert ctx.mamba_state_idx_buf is not None
assert ctx.num_scheduled_tokens_buf is not None
assert ctx.num_computed_tokens_buf is not None
assert ctx.num_draft_tokens_buf is not None
stage_mamba_state_idx_to_gpu(
mamba_state_idx,
req_ids,
num_reqs,
ctx.mamba_state_idx_buf,
)
stage_postprocess_metadata_to_gpu(
scheduler_output,
req_ids,
num_reqs,
requests,
ctx.num_scheduled_tokens_buf,
ctx.num_computed_tokens_buf,
ctx.num_draft_tokens_buf,
)
scheduled_spec_tokens = scheduler_output.scheduled_spec_decode_tokens
num_scheduled = scheduler_output.num_scheduled_tokens
state_idx_np = ctx.mamba_state_idx_buf.np
scheduled_np = ctx.num_scheduled_tokens_buf.np
computed_np = ctx.num_computed_tokens_buf.np
draft_np = ctx.num_draft_tokens_buf.np
for i in range(num_reqs):
req_id = req_ids[i]
state_idx = mamba_state_idx.get(req_id)
assert state_idx is not None, (
f"mamba_state_idx missing entry for {req_id!r}; "
"preprocess_mamba must run before stage_postprocess_inputs_to_gpu"
)
state_idx_np[i] = state_idx
scheduled_np[i] = num_scheduled[req_id]
computed_np[i] = requests[req_id].num_computed_tokens
draft_np[i] = len(scheduled_spec_tokens.get(req_id, []))
ctx.mamba_state_idx_buf.copy_to_gpu(num_reqs)
ctx.num_scheduled_tokens_buf.copy_to_gpu(num_reqs)
ctx.num_computed_tokens_buf.copy_to_gpu(num_reqs)
ctx.num_draft_tokens_buf.copy_to_gpu(num_reqs)