[Hybrid] Stage the postprocess inputs with a single loop over the request list (#48120)
Signed-off-by: Francesco Fusco <ffu@zurich.ibm.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude
parent
1c0d207915
commit
755513d9a2
@@ -21,6 +21,7 @@ from vllm.v1.worker.mamba_utils import (
|
||||
collect_mamba_copy_meta,
|
||||
do_mamba_copy_block,
|
||||
preprocess_mamba,
|
||||
stage_postprocess_inputs_to_gpu,
|
||||
)
|
||||
|
||||
MambaStateCopyFunc = Callable[..., Any]
|
||||
@@ -371,6 +372,123 @@ def _make_gpu_ctx(
|
||||
)
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# stage_postprocess_inputs_to_gpu: single-pass staging into pinned views
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_staging_ctx(max_num_reqs: int, device: torch.device) -> MagicMock:
|
||||
"""Build a MambaSpecDecodeGPUContext stand-in exposing only the four
|
||||
per-request staging buffers touched by stage_postprocess_inputs_to_gpu."""
|
||||
ctx = MagicMock()
|
||||
ctx.mamba_state_idx_buf = _MockCpuGpuBuffer(max_num_reqs, torch.int32, device)
|
||||
ctx.num_scheduled_tokens_buf = _MockCpuGpuBuffer(max_num_reqs, torch.int32, device)
|
||||
ctx.num_computed_tokens_buf = _MockCpuGpuBuffer(max_num_reqs, torch.int32, device)
|
||||
ctx.num_draft_tokens_buf = _MockCpuGpuBuffer(max_num_reqs, torch.int32, device)
|
||||
return ctx
|
||||
|
||||
|
||||
def test_stage_postprocess_inputs_to_gpu_fills_pinned_views():
|
||||
"""stage_postprocess_inputs_to_gpu writes each request's state_idx and
|
||||
scheduled/computed/draft counts into slot `i` for `req_ids[i]`, leaves
|
||||
slots past `num_reqs` untouched, and mirrors the values to the GPU buffers."""
|
||||
device = torch.device("cpu")
|
||||
max_num_reqs = 8
|
||||
ctx = _make_staging_ctx(max_num_reqs, device)
|
||||
|
||||
# Any negative int32 works as a sentinel: all staged values (state_idx,
|
||||
# scheduled/computed/draft token counts) are non-negative, so a negative
|
||||
# entry surviving in slots [num_reqs:] proves the loop didn't overrun.
|
||||
sentinel = np.int32(-1)
|
||||
bufs = (
|
||||
ctx.mamba_state_idx_buf,
|
||||
ctx.num_scheduled_tokens_buf,
|
||||
ctx.num_computed_tokens_buf,
|
||||
ctx.num_draft_tokens_buf,
|
||||
)
|
||||
for buf in bufs:
|
||||
buf.np[:] = sentinel
|
||||
|
||||
req_ids = ["req_a", "req_b", "req_c"]
|
||||
num_reqs = len(req_ids)
|
||||
scheduler_output = _make_postprocess_scheduler_output(
|
||||
req_ids=req_ids,
|
||||
num_scheduled_tokens={"req_a": 5, "req_b": 12, "req_c": 3},
|
||||
# req_b intentionally absent -> defaults to 0 drafts.
|
||||
scheduled_spec_decode_tokens={
|
||||
"req_a": [1, 2],
|
||||
"req_c": [1, 2, 3, 4],
|
||||
},
|
||||
)
|
||||
requests = _make_requests(
|
||||
req_ids=req_ids,
|
||||
num_computed_tokens=[10, 20, 30],
|
||||
block_ids_per_req=[[0], [0], [0]],
|
||||
)
|
||||
mamba_state_idx = {"req_a": 100, "req_b": 200, "req_c": 300}
|
||||
# A trailing entry past num_reqs must not be read.
|
||||
req_ids_padded = req_ids + ["not_scheduled"]
|
||||
|
||||
stage_postprocess_inputs_to_gpu(
|
||||
ctx,
|
||||
scheduler_output,
|
||||
req_ids_padded,
|
||||
num_reqs,
|
||||
requests,
|
||||
mamba_state_idx,
|
||||
)
|
||||
|
||||
np.testing.assert_array_equal(
|
||||
ctx.mamba_state_idx_buf.np[:num_reqs], [100, 200, 300]
|
||||
)
|
||||
np.testing.assert_array_equal(
|
||||
ctx.num_scheduled_tokens_buf.np[:num_reqs], [5, 12, 3]
|
||||
)
|
||||
np.testing.assert_array_equal(
|
||||
ctx.num_computed_tokens_buf.np[:num_reqs], [10, 20, 30]
|
||||
)
|
||||
np.testing.assert_array_equal(ctx.num_draft_tokens_buf.np[:num_reqs], [2, 0, 4])
|
||||
for buf in bufs:
|
||||
assert (buf.np[num_reqs:] == sentinel).all()
|
||||
|
||||
assert torch.equal(
|
||||
ctx.mamba_state_idx_buf.gpu[:num_reqs],
|
||||
torch.tensor([100, 200, 300], dtype=torch.int32),
|
||||
)
|
||||
assert torch.equal(
|
||||
ctx.num_draft_tokens_buf.gpu[:num_reqs],
|
||||
torch.tensor([2, 0, 4], dtype=torch.int32),
|
||||
)
|
||||
|
||||
|
||||
def test_stage_postprocess_inputs_to_gpu_asserts_on_missing_state_idx():
|
||||
"""If preprocess_mamba didn't populate mamba_state_idx for a req in the
|
||||
batch, staging must fail loudly rather than silently writing a stale index."""
|
||||
device = torch.device("cpu")
|
||||
ctx = _make_staging_ctx(max_num_reqs=4, device=device)
|
||||
scheduler_output = _make_postprocess_scheduler_output(
|
||||
req_ids=["req_a"],
|
||||
num_scheduled_tokens={"req_a": 1},
|
||||
)
|
||||
requests = _make_requests(["req_a"], [0], [[0]])
|
||||
|
||||
# mamba_state_idx has an entry for a *different* request but not for
|
||||
# req_a (the one being staged). This is the realistic failure mode:
|
||||
# preprocess_mamba ran, populated some reqs, but missed this one.
|
||||
# The safety assert in stage_postprocess_inputs_to_gpu must fire on
|
||||
# req_a's missing entry rather than silently writing None.
|
||||
mamba_state_idx: dict[str, int] = {"other_req": 42}
|
||||
with pytest.raises(AssertionError, match="mamba_state_idx missing entry"):
|
||||
stage_postprocess_inputs_to_gpu(
|
||||
ctx,
|
||||
scheduler_output,
|
||||
["req_a"],
|
||||
1,
|
||||
requests,
|
||||
mamba_state_idx,
|
||||
)
|
||||
|
||||
|
||||
def _run_gpu_postprocess(
|
||||
gpu_ctx: MambaSpecDecodeGPUContext,
|
||||
*,
|
||||
|
||||
@@ -1238,66 +1238,6 @@ def postprocess_mamba_align_gpu(
|
||||
)
|
||||
|
||||
|
||||
def stage_postprocess_metadata_to_gpu(
|
||||
scheduler_output: SchedulerOutput,
|
||||
req_ids: list[str],
|
||||
num_reqs: int,
|
||||
requests: dict[str, CachedRequestState],
|
||||
num_scheduled_tokens_buf: CpuGpuBuffer,
|
||||
num_computed_tokens_buf: CpuGpuBuffer,
|
||||
num_draft_tokens_buf: CpuGpuBuffer,
|
||||
) -> None:
|
||||
"""Stage per-request postprocess metadata into GPU buffers (non-blocking).
|
||||
|
||||
Walks ``req_ids[:num_reqs]`` in batch order and writes each request's
|
||||
scheduled/computed/draft token counts into the matching pinned numpy
|
||||
views, then issues three non-blocking H→D copies. These values don't
|
||||
change between ``_prepare_inputs`` and ``_update_states_after_model_execute``.
|
||||
The fused postprocess kernel indexes the resulting GPU tensors
|
||||
by ``req_idx``.
|
||||
"""
|
||||
scheduled_spec_tokens = scheduler_output.scheduled_spec_decode_tokens
|
||||
num_scheduled = scheduler_output.num_scheduled_tokens
|
||||
scheduled_np = num_scheduled_tokens_buf.np
|
||||
computed_np = num_computed_tokens_buf.np
|
||||
draft_np = num_draft_tokens_buf.np
|
||||
for i in range(num_reqs):
|
||||
req_id = req_ids[i]
|
||||
scheduled_np[i] = num_scheduled[req_id]
|
||||
computed_np[i] = requests[req_id].num_computed_tokens
|
||||
draft_np[i] = len(scheduled_spec_tokens.get(req_id, []))
|
||||
num_scheduled_tokens_buf.copy_to_gpu(num_reqs)
|
||||
num_computed_tokens_buf.copy_to_gpu(num_reqs)
|
||||
num_draft_tokens_buf.copy_to_gpu(num_reqs)
|
||||
|
||||
|
||||
def stage_mamba_state_idx_to_gpu(
|
||||
mamba_state_idx: dict[str, int],
|
||||
req_ids: list[str],
|
||||
num_reqs: int,
|
||||
gpu_buf: CpuGpuBuffer,
|
||||
) -> None:
|
||||
"""Materialize ``mamba_state_idx`` into ``gpu_buf`` and copy to GPU.
|
||||
|
||||
Walks ``req_ids[:num_reqs]`` in batch order, writing each request's block
|
||||
index into the buffer's pinned numpy view, then issues a non-blocking H→D
|
||||
copy. The fused kernel indexes the resulting GPU tensor by ``req_idx``.
|
||||
|
||||
Invariant: ``preprocess_mamba`` must have run first for the same batch so
|
||||
that every ``req_ids[i]`` has an entry in ``mamba_state_idx``.
|
||||
"""
|
||||
np_view = gpu_buf.np
|
||||
for i in range(num_reqs):
|
||||
req_id = req_ids[i]
|
||||
state_idx = mamba_state_idx.get(req_id)
|
||||
assert state_idx is not None, (
|
||||
f"mamba_state_idx missing entry for {req_id!r}; "
|
||||
"preprocess_mamba must run before stage_mamba_state_idx_to_gpu"
|
||||
)
|
||||
np_view[i] = state_idx
|
||||
gpu_buf.copy_to_gpu(num_reqs)
|
||||
|
||||
|
||||
def stage_postprocess_inputs_to_gpu(
|
||||
ctx: MambaSpecDecodeGPUContext,
|
||||
scheduler_output: SchedulerOutput,
|
||||
@@ -1308,27 +1248,40 @@ def stage_postprocess_inputs_to_gpu(
|
||||
) -> None:
|
||||
"""Stage all per-request inputs the fused mamba postprocess kernel reads.
|
||||
|
||||
Bundles ``stage_mamba_state_idx_to_gpu`` and
|
||||
``stage_postprocess_metadata_to_gpu`` into a single call so the runner
|
||||
has one entry point for postprocess staging. Buffers live on ``ctx``
|
||||
Walks ``req_ids[:num_reqs]`` once, writing each request's mamba block
|
||||
index and scheduled/computed/draft token counts into the matching pinned
|
||||
numpy views, then issues four non-blocking H→D copies. The fused kernel
|
||||
indexes the resulting GPU tensors by ``req_idx``. Buffers live on ``ctx``
|
||||
and only exist when the postprocess kernel is enabled.
|
||||
|
||||
Invariant: ``preprocess_mamba`` must have run first for the same batch so
|
||||
that every ``req_ids[i]`` has an entry in ``mamba_state_idx``.
|
||||
"""
|
||||
assert ctx.mamba_state_idx_buf is not None
|
||||
assert ctx.num_scheduled_tokens_buf is not None
|
||||
assert ctx.num_computed_tokens_buf is not None
|
||||
assert ctx.num_draft_tokens_buf is not None
|
||||
stage_mamba_state_idx_to_gpu(
|
||||
mamba_state_idx,
|
||||
req_ids,
|
||||
num_reqs,
|
||||
ctx.mamba_state_idx_buf,
|
||||
)
|
||||
stage_postprocess_metadata_to_gpu(
|
||||
scheduler_output,
|
||||
req_ids,
|
||||
num_reqs,
|
||||
requests,
|
||||
ctx.num_scheduled_tokens_buf,
|
||||
ctx.num_computed_tokens_buf,
|
||||
ctx.num_draft_tokens_buf,
|
||||
)
|
||||
|
||||
scheduled_spec_tokens = scheduler_output.scheduled_spec_decode_tokens
|
||||
num_scheduled = scheduler_output.num_scheduled_tokens
|
||||
state_idx_np = ctx.mamba_state_idx_buf.np
|
||||
scheduled_np = ctx.num_scheduled_tokens_buf.np
|
||||
computed_np = ctx.num_computed_tokens_buf.np
|
||||
draft_np = ctx.num_draft_tokens_buf.np
|
||||
|
||||
for i in range(num_reqs):
|
||||
req_id = req_ids[i]
|
||||
state_idx = mamba_state_idx.get(req_id)
|
||||
assert state_idx is not None, (
|
||||
f"mamba_state_idx missing entry for {req_id!r}; "
|
||||
"preprocess_mamba must run before stage_postprocess_inputs_to_gpu"
|
||||
)
|
||||
state_idx_np[i] = state_idx
|
||||
scheduled_np[i] = num_scheduled[req_id]
|
||||
computed_np[i] = requests[req_id].num_computed_tokens
|
||||
draft_np[i] = len(scheduled_spec_tokens.get(req_id, []))
|
||||
|
||||
ctx.mamba_state_idx_buf.copy_to_gpu(num_reqs)
|
||||
ctx.num_scheduled_tokens_buf.copy_to_gpu(num_reqs)
|
||||
ctx.num_computed_tokens_buf.copy_to_gpu(num_reqs)
|
||||
ctx.num_draft_tokens_buf.copy_to_gpu(num_reqs)
|
||||
|
||||
Reference in New Issue
Block a user