[1/2][Model Runner V2] DBO support, eager mode (#50945)

Signed-off-by: specture724 <specture724@gmail.com>
Signed-off-by: Anjie Hou <149605198+specture724@users.noreply.github.com>
Co-authored-by: jiangkuaixue123 <jiangxiaozhou111@163.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
Anjie Hou
2026-09-04 14:17:39 -07:00
committed by GitHub
co-authored by jiangkuaixue123 Lucas Wilkinson
parent 784cac7c42
commit 3f41d102c5
17 changed files with 1346 additions and 111 deletions
@@ -453,6 +453,7 @@ def test_v2_model_runner_accepts_routed_experts(monkeypatch):
distributed_executor_backend=None,
pipeline_parallel_size=1,
enable_dbo=False,
use_ubatching=False,
enable_elastic_ep=False,
),
compilation_config=SimpleNamespace(
+583
View File
@@ -0,0 +1,583 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Dual batch overlap (DBO) in the V2 model runner.
The V2 runner slices the batch *before* building attention metadata, where V1
builds metadata for the whole batch and slices it afterwards. Both must describe
the same microbatches, so the main test here pins V2's slicing against V1's
`split_attn_metadata`. The rest covers the decision to microbatch and the
threaded execution of the microbatches.
"""
import threading
from dataclasses import replace
from types import SimpleNamespace
from typing import Any, cast
from unittest.mock import patch
import numpy as np
import pytest
import torch
from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
from vllm.config import CUDAGraphMode, ModelConfig, ParallelConfig, VllmConfig
from vllm.forward_context import create_forward_context
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.gpu import dp_utils
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.model_states.interface import ModelState
from vllm.v1.worker.gpu.ubatch_utils import (
UBatchRunner,
UBatchState,
_slice_input_batch,
create_ubatch_slices,
slice_model_inputs,
)
from vllm.v1.worker.ubatch_utils import (
UBatchSlice,
maybe_create_ubatch_slices,
split_attn_metadata,
)
from vllm.v1.worker.ubatching import dbo_current_ubatch_id, dbo_yield
MAX_NUM_REQS = 32
MAX_NUM_TOKENS = 128
def _make_buffers() -> InputBuffers:
return InputBuffers(
max_num_reqs=MAX_NUM_REQS,
max_num_tokens=MAX_NUM_TOKENS,
device=torch.device("cpu"),
)
def _make_ubatch_buffers(num_ubatches: int = 2) -> list[tuple[torch.Tensor, ...]]:
"""The per-microbatch (query_start_loc, seq_lens) buffers UBatchRunner owns."""
return [
(
torch.zeros(MAX_NUM_REQS + 1, dtype=torch.int32),
torch.zeros(MAX_NUM_REQS, dtype=torch.int32),
)
for _ in range(num_ubatches)
]
def _make_input_batch(
query_lens: list[int],
seq_lens: list[int],
buffers: InputBuffers,
num_reqs_padded: int | None = None,
num_tokens_padded: int | None = None,
) -> InputBatch:
"""Build an InputBatch shaped like the one `prepare_inputs` produces."""
num_reqs = len(query_lens)
num_tokens = int(sum(query_lens))
num_reqs_padded = num_reqs_padded or num_reqs
num_tokens_padded = num_tokens_padded or num_tokens
# make_dummy writes the shared buffers, so fill them afterwards.
base = InputBatch.make_dummy(num_reqs, num_tokens, buffers)
query_start_loc_np = np.zeros(num_reqs_padded + 1, dtype=np.int32)
np.cumsum(query_lens, out=query_start_loc_np[1 : num_reqs + 1])
# Padded entries repeat the token count, as `prepare_inputs` does.
query_start_loc_np[num_reqs + 1 :] = num_tokens
buffers.query_start_loc[: num_reqs_padded + 1] = torch.from_numpy(
query_start_loc_np
)
buffers.seq_lens[:num_reqs] = torch.tensor(seq_lens, dtype=torch.int32)
buffers.seq_lens[num_reqs:num_reqs_padded] = 0
seq_lens_upper_bound = np.zeros(num_reqs_padded, dtype=np.int32)
seq_lens_upper_bound[:num_reqs] = seq_lens
return replace(
base,
req_ids=[f"req_{i}" for i in range(num_reqs)],
num_reqs=num_reqs,
num_reqs_after_padding=num_reqs_padded,
idx_mapping=torch.arange(num_reqs, dtype=torch.int32),
idx_mapping_np=np.arange(num_reqs, dtype=np.int32),
num_scheduled_tokens=np.array(query_lens, dtype=np.int32),
num_tokens=num_tokens,
num_tokens_after_padding=num_tokens_padded,
query_start_loc=buffers.query_start_loc[: num_reqs_padded + 1],
query_start_loc_np=query_start_loc_np,
seq_lens=buffers.seq_lens[:num_reqs_padded],
seq_lens_cpu_upper_bound=torch.from_numpy(seq_lens_upper_bound),
num_computed_tokens_np=np.array(seq_lens, dtype=np.int32)
- np.array(query_lens, dtype=np.int32),
prefill_len_np=np.zeros(num_reqs, dtype=np.int32),
num_computed_prefill_tokens_np=np.zeros(num_reqs, dtype=np.int32),
is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_),
input_ids=buffers.input_ids[:num_tokens_padded],
positions=buffers.positions[:num_tokens_padded],
is_padding=buffers.is_padding[:num_tokens_padded],
)
# (query_lens, seq_lens): uniform decode, a boundary that lands inside a
# request, and a boundary that lands exactly on a request start.
BATCHES = {
"uniform_decode": ([1] * 8, [128 + 4 * i for i in range(8)]),
"split_request": ([1, 1, 10, 1, 1], [64, 96, 512, 32, 48]),
"aligned_prefills": ([6, 2, 4, 4], [6, 130, 260, 4]),
"split_first_and_last": ([3, 9, 3], [70, 300, 41]),
# A long prefill trailing a decode: it straddles the boundary and is also
# the last request, so the second microbatch holds nothing else.
"trailing_prefill_spans_boundary": ([2, 20], [64, 300]),
}
@pytest.mark.parametrize("batch_name", list(BATCHES))
def test_slicing_matches_v1_split_attn_metadata(batch_name: str):
"""V2's pre-sliced inputs describe the same microbatches as V1's."""
query_lens, seq_lens = BATCHES[batch_name]
num_scheduled_tokens = np.array(query_lens, dtype=np.int32)
num_tokens = int(num_scheduled_tokens.sum())
ubatch_slices, _ = maybe_create_ubatch_slices(
True, num_scheduled_tokens, num_tokens, len(query_lens), num_ubatches=2
)
assert ubatch_slices is not None
v1_metadata = create_common_attn_metadata(
BatchSpec(seq_lens=seq_lens, query_lens=query_lens),
block_size=16,
device=torch.device("cpu"),
)
v1_ubatches = split_attn_metadata(ubatch_slices, v1_metadata)
buffers = _make_buffers()
ubatch_buffers = _make_ubatch_buffers()
input_batch = _make_input_batch(query_lens, seq_lens, buffers)
for i, (ubatch_slice, v1_ubatch) in enumerate(zip(ubatch_slices, v1_ubatches)):
v2_ubatch = _slice_input_batch(input_batch, ubatch_slice, *ubatch_buffers[i])
assert v2_ubatch.num_reqs == v1_ubatch.num_reqs
assert v2_ubatch.num_tokens == v1_ubatch.num_actual_tokens
assert int(v2_ubatch.num_scheduled_tokens.max()) == v1_ubatch.max_query_len
torch.testing.assert_close(v2_ubatch.query_start_loc, v1_ubatch.query_start_loc)
torch.testing.assert_close(v2_ubatch.seq_lens, v1_ubatch.seq_lens)
torch.testing.assert_close(
torch.from_numpy(v2_ubatch.query_start_loc_np),
v1_ubatch.query_start_loc_cpu,
)
# `prepare_attn` derives max_seq_len from the upper bound. Compare the
# bound itself rather than V1's max_seq_len: V1 floors it with the full
# batch's max_seq_len to keep the value CUDA-graph capture installed,
# which V2 gets from `prepare_attn(for_capture=True)` instead.
torch.testing.assert_close(
v2_ubatch.seq_lens_cpu_upper_bound, v1_ubatch.seq_lens_cpu_upper_bound
)
def test_microbatches_do_not_share_buffers():
"""Both microbatches are live at once, so their buffers must be distinct."""
query_lens, seq_lens = BATCHES["split_request"]
num_scheduled_tokens = np.array(query_lens, dtype=np.int32)
num_tokens = int(num_scheduled_tokens.sum())
ubatch_slices, _ = maybe_create_ubatch_slices(
True, num_scheduled_tokens, num_tokens, len(query_lens), num_ubatches=2
)
assert ubatch_slices is not None
buffers = _make_buffers()
ubatch_buffers = _make_ubatch_buffers()
input_batch = _make_input_batch(query_lens, seq_lens, buffers)
first = _slice_input_batch(input_batch, ubatch_slices[0], *ubatch_buffers[0])
first_query_start_loc = first.query_start_loc.clone()
first_seq_lens = first.seq_lens.clone()
# Slicing the second microbatch must not disturb the first.
_slice_input_batch(input_batch, ubatch_slices[1], *ubatch_buffers[1])
torch.testing.assert_close(first.query_start_loc, first_query_start_loc)
torch.testing.assert_close(first.seq_lens, first_seq_lens)
def test_trailing_microbatch_absorbs_cudagraph_padding():
"""The padded microbatch keeps padded rows empty and query_start_loc flat."""
query_lens, seq_lens = [1] * 6, [64] * 6
num_tokens_padded = 8
buffers = _make_buffers()
input_batch = _make_input_batch(
query_lens,
seq_lens,
buffers,
num_reqs_padded=num_tokens_padded,
num_tokens_padded=num_tokens_padded,
)
ubatch_slices_padded = create_ubatch_slices(input_batch, num_ubatches=2)
last = _slice_input_batch(
input_batch, ubatch_slices_padded[-1], *_make_ubatch_buffers()[1]
)
# Two real decodes plus two padded rows.
assert last.num_reqs == 2
assert last.num_tokens == 2
assert last.num_reqs_after_padding == 4
assert last.num_tokens_after_padding == 4
torch.testing.assert_close(
last.query_start_loc, torch.tensor([0, 1, 2, 2, 2], dtype=torch.int32)
)
torch.testing.assert_close(
last.seq_lens, torch.tensor([64, 64, 0, 0], dtype=torch.int32)
)
DECODE_THRESHOLD = 32
PREFILL_THRESHOLD = 128
DECODE_QUERY_LEN = 1
def test_dummy_batch_can_be_microbatched():
"""A DP rank with no work still has to split, so its dummy batch must slice.
Microbatching is all-or-nothing across the group, so a rank whose step is a
dummy batch runs it microbatched like everyone else.
"""
num_reqs, num_tokens = 1, 32
buffers = _make_buffers()
input_batch = InputBatch.make_dummy(num_reqs, num_tokens, buffers)
ubatch_slices_padded = create_ubatch_slices(input_batch, num_ubatches=2)
ubatch_buffers = _make_ubatch_buffers()
ubatches = [
_slice_input_batch(input_batch, ubatch_slice, *ubatch_buffers[i])
for i, ubatch_slice in enumerate(ubatch_slices_padded)
]
assert [u.num_tokens for u in ubatches] == [16, 16]
# The one dummy request spans both microbatches, 16 of its tokens in each.
assert all(u.num_reqs == 1 for u in ubatches)
for ubatch in ubatches:
torch.testing.assert_close(
ubatch.query_start_loc, torch.tensor([0, 16], dtype=torch.int32)
)
def _sync_dp(
num_tokens_per_rank: list[int],
uniform_token_count_per_rank: list[int] | None = None,
allow_ubatching: bool = True,
) -> tuple[BatchExecutionDescriptor, dp_utils.DPSyncState | None]:
"""Run the DP handshake with the all-reduce stubbed out.
The microbatching decision lives inside `sync_cudagraph_and_dp_padding`, so
the cases below stand in for the collective rather than for the decision,
and need no process group. Every rank asks for eager, which keeps the
non-microbatched path off the cudagraph manager. This rank is rank 0.
"""
dp_size = len(num_tokens_per_rank)
uniform_token_counts = uniform_token_count_per_rank or [0] * dp_size
reduced = torch.zeros(6, dp_size, dtype=torch.int32)
reduced[0] = torch.tensor(num_tokens_per_rank, dtype=torch.int32)
reduced[1] = CUDAGraphMode.NONE.value
reduced[2] = torch.tensor(uniform_token_counts, dtype=torch.int32)
reduced[3] = -1 # max_query_len, -1 means None
reduced[4] = int(allow_ubatching)
reduced[5] = 8 # num_reqs
with (
patch.object(dp_utils.dist, "all_reduce", lambda t, group: t.copy_(reduced)),
patch.object(dp_utils, "get_dp_group", lambda: SimpleNamespace(cpu_group=None)),
):
return dp_utils.sync_cudagraph_and_dp_padding(
cudagraph_manager=None,
desired_batch_desc=BatchExecutionDescriptor(
cg_mode=CUDAGraphMode.NONE,
num_tokens=num_tokens_per_rank[0],
num_reqs=8,
),
num_tokens=num_tokens_per_rank[0],
num_reqs=8,
uniform_token_count=uniform_token_counts[0] or None,
dp_size=dp_size,
dp_rank=0,
parallel_config=ParallelConfig(
enable_dbo=True,
dbo_decode_token_threshold=DECODE_THRESHOLD,
dbo_prefill_token_threshold=PREFILL_THRESHOLD,
),
allow_ubatching=allow_ubatching,
uniform_decode=uniform_token_counts[0] == DECODE_QUERY_LEN,
)
def test_every_dp_rank_must_agree_to_microbatch():
"""One rank below the threshold is enough to keep the group on one batch.
The thresholds are not communicated -- they are checked against the token
counts the all-reduce already carries -- so this also pins that the whole
vector is consulted, not just this rank's own entry.
"""
assert _sync_dp([256, 256])[0].num_ubatches == 2
assert _sync_dp([256, 100])[0].num_ubatches == 1
assert _sync_dp([100, 256])[0].num_ubatches == 1
def test_decode_threshold_only_applies_when_every_rank_is_a_uniform_decode():
"""Otherwise the group is held to the prefill threshold."""
# 64 tokens clears the decode threshold but not the prefill one, so it only
# microbatches when both ranks are uniform decodes.
assert _sync_dp([64, 64], uniform_token_count_per_rank=[1, 1])[0].num_ubatches == 2
assert _sync_dp([64, 64], uniform_token_count_per_rank=[1, 0])[0].num_ubatches == 1
def test_microbatching_pads_all_ranks_to_the_largest():
"""Ranks must run the same token count so microbatch sizes line up."""
desc, dp_sync = _sync_dp([200, 256])
assert desc.num_tokens == 256
assert dp_sync is not None
assert dp_sync.num_tokens_across_dp.tolist() == [256, 256]
def test_microbatching_survives_a_rank_that_cannot_fill_it():
"""A rank whose real tokens all land in the first microbatch still splits.
Rank 0 has 128 real tokens but the split point is at 512 // 2 = 256, so its
second microbatch is pure padding. That microbatch does no work, which is
fine -- it still has to run so the expert all-to-all stays collective.
"""
desc, _ = _sync_dp([128, 512])
assert desc.num_tokens == 512
assert desc.num_ubatches == 2
def test_all_padding_microbatch_has_no_work_to_do():
"""The microbatch past the last real token is well-formed and empty.
It carries the last request with none of its query tokens, so attention and
sampling see nothing to do while the microbatch still runs.
"""
query_lens, seq_lens = [1] * 6, [64] * 6
num_tokens_padded = 16 # split at 8, past all 6 real tokens
buffers = _make_buffers()
input_batch = _make_input_batch(
query_lens, seq_lens, buffers, num_tokens_padded=num_tokens_padded
)
ubatch_slices_padded = create_ubatch_slices(input_batch, num_ubatches=2)
ubatch_buffers = _make_ubatch_buffers()
first = _slice_input_batch(input_batch, ubatch_slices_padded[0], *ubatch_buffers[0])
last = _slice_input_batch(input_batch, ubatch_slices_padded[1], *ubatch_buffers[1])
# The first microbatch keeps every real request; the second gets none.
assert first.num_reqs == 6
assert first.num_tokens == 6
assert last.num_tokens == 0
assert last.num_reqs == 1
assert last.num_reqs_after_padding == 1
assert last.num_tokens_after_padding == 8
# Zero query tokens for the one request it holds, so no attention work.
torch.testing.assert_close(
last.query_start_loc, torch.tensor([0, 0], dtype=torch.int32)
)
assert last.num_scheduled_tokens.tolist() == [0]
# ...and its sequence still ends where its tokens do: they all live in the
# first microbatch, which computes them before this one runs.
torch.testing.assert_close(last.seq_lens, torch.tensor([64], dtype=torch.int32))
torch.testing.assert_close(
last.seq_lens_cpu_upper_bound, torch.tensor([64], dtype=torch.int32)
)
def test_microbatching_off_when_a_rank_does_not_allow_it():
assert _sync_dp([256, 256], allow_ubatching=False)[0].num_ubatches == 1
def test_slice_model_inputs_handles_mrope_positions():
model_inputs = {
"input_ids": torch.arange(8),
# M-RoPE positions carry a leading section dim.
"positions": torch.arange(24).view(3, 8),
"inputs_embeds": None,
"intermediate_tensors": None,
}
sliced = slice_model_inputs(model_inputs, slice(4, 8))
torch.testing.assert_close(sliced["input_ids"], torch.arange(4, 8))
torch.testing.assert_close(sliced["positions"], torch.arange(24).view(3, 8)[:, 4:8])
assert sliced["inputs_embeds"] is None
assert sliced["intermediate_tensors"] is None
def _make_dbo_config() -> VllmConfig:
return VllmConfig(
model_config=ModelConfig(model="facebook/opt-125m", dtype="float16", seed=0),
parallel_config=ParallelConfig(
enable_dbo=True, all2all_backend="deepep_low_latency"
),
)
def _make_model_inputs(num_tokens: int, device: torch.device) -> dict[str, Any]:
return {
"input_ids": torch.arange(num_tokens, device=device),
"positions": torch.arange(num_tokens, device=device),
"inputs_embeds": None,
"intermediate_tensors": None,
}
@pytest.fixture(autouse=True)
def _no_comm_sm_reservation(monkeypatch):
"""CI runs on MIG slices with fewer SMs than the default comm reservation."""
monkeypatch.setenv("VLLM_DBO_COMM_SMS", "0")
def _make_execution_runner(vllm_config: VllmConfig) -> UBatchRunner:
"""A runner for the execution tests below, which never call `prepare()`.
The attention-side dependencies are only read while building per-microbatch
metadata, so they are left out here.
"""
return UBatchRunner(
vllm_config,
torch.device("cuda:0"),
model_state=cast(ModelState, None),
attn_groups=[],
kv_cache_config=cast(KVCacheConfig, None),
max_num_reqs=MAX_NUM_REQS,
)
def _make_ubatch_state(
vllm_config: VllmConfig, ubatch_slices: list[UBatchSlice]
) -> UBatchState:
return UBatchState(
slices=ubatch_slices,
forward_contexts=[
create_forward_context(None, vllm_config) for _ in ubatch_slices
],
)
def test_thresholds_below_the_microbatch_count_are_rejected():
"""A batch with fewer tokens than microbatches cannot be split at all.
Nothing downstream copes with an unsplittable batch, so the thresholds are
what keep one out: a one-token decode split two ways leaves an empty
microbatch.
"""
with pytest.raises(ValueError, match="number of microbatches"):
ParallelConfig(enable_dbo=True, dbo_decode_token_threshold=1)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="DBO needs a GPU")
def test_runner_allocates_one_buffer_pair_per_microbatch():
"""All microbatches are live at once, so none may share rebase buffers."""
vllm_config = _make_dbo_config()
runner = _make_execution_runner(vllm_config)
for buffers in (runner.ubatch_query_start_loc, runner.ubatch_seq_lens):
assert len(buffers) == runner.num_ubatches
storages = {b.untyped_storage().data_ptr() for b in buffers}
assert len(storages) == runner.num_ubatches
class _YieldingModel(torch.nn.Module):
"""Toy model that hands off to the other microbatch mid-forward."""
def __init__(self, trace: list[tuple[str, int]]):
super().__init__()
self.trace = trace
def forward(self, input_ids, positions, intermediate_tensors=None, **kwargs):
self.trace.append(("enter", dbo_current_ubatch_id()))
out = input_ids.float().unsqueeze(-1) * 2 + positions.float().unsqueeze(-1)
# Stands in for the expert all-to-all handoff point.
dbo_yield()
self.trace.append(("exit", dbo_current_ubatch_id()))
return out
@pytest.mark.skipif(not torch.cuda.is_available(), reason="DBO needs a GPU")
def test_ubatch_runner_overlaps_and_matches_single_batch():
"""Microbatches interleave at the yield point and produce the same output."""
vllm_config = _make_dbo_config()
device = torch.device("cuda:0")
runner = _make_execution_runner(vllm_config)
model_inputs = _make_model_inputs(16, device)
ubatch_state = _make_ubatch_state(
vllm_config,
[
UBatchSlice(slice(0, 4), slice(0, 8)),
UBatchSlice(slice(4, 8), slice(8, 16)),
],
)
trace: list[tuple[str, int]] = []
model = _YieldingModel(trace)
output = runner.run(model, model_inputs, ubatch_state)
expected = model_inputs["input_ids"].float().unsqueeze(-1) * 2 + model_inputs[
"positions"
].float().unsqueeze(-1)
torch.testing.assert_close(output, expected)
# Both microbatches reach the handoff before either finishes.
assert trace == [("enter", 0), ("enter", 1), ("exit", 0), ("exit", 1)]
@pytest.mark.skipif(not torch.cuda.is_available(), reason="DBO needs a GPU")
def test_ubatch_runner_names_the_microbatch_that_failed():
"""A failing microbatch surfaces as a named error, not a downstream KeyError.
The model here never yields, so the microbatches run back to back and no
handoff is left outstanding when one of them dies. A microbatch that fails
*while its sibling is parked at a yield* hangs the step instead -- the
shared handoff protocol in `ubatching.py` has no way to unwind a parked
microbatch, and the V1 runner has the same gap. Fixing that means changing
`ubatching.py`, which is out of scope for the V2 runner.
"""
vllm_config = _make_dbo_config()
device = torch.device("cuda:0")
runner = _make_execution_runner(vllm_config)
class _FailingModel(torch.nn.Module):
def forward(self, input_ids, positions, **kwargs):
if dbo_current_ubatch_id() == 0:
raise ValueError("boom")
return input_ids.float().unsqueeze(-1)
model_inputs = _make_model_inputs(8, device)
ubatch_state = _make_ubatch_state(
vllm_config,
[
UBatchSlice(slice(0, 2), slice(0, 4)),
UBatchSlice(slice(2, 4), slice(4, 8)),
],
)
result: dict[str, BaseException] = {}
def _call():
try:
runner.run(_FailingModel(), model_inputs, ubatch_state)
except BaseException as e: # noqa: BLE001
result["error"] = e
# Run behind a watchdog: `UBatchRunner.run` joins without a timeout, so a
# regression here would wedge the suite rather than fail it.
caller = threading.Thread(target=_call, daemon=True)
caller.start()
caller.join(timeout=60.0)
assert not caller.is_alive(), "UBatchRunner.run hung on a failing microbatch"
assert isinstance(result["error"], RuntimeError)
assert "Microbatch 0" in str(result["error"])
assert isinstance(result["error"].__cause__, ValueError)
+11
View File
@@ -1065,6 +1065,17 @@ class ParallelConfig:
"Unable to use nsight profiling unless workers run with Ray."
)
# A batch below one token per microbatch cannot be split, so the
# thresholds have to keep it out rather than the split having to cope.
if self.use_ubatching and (
min(self.dbo_decode_token_threshold, self.dbo_prefill_token_threshold)
< self.num_ubatches
):
raise ValueError(
"dbo_decode_token_threshold and dbo_prefill_token_threshold must "
f"be at least the number of microbatches ({self.num_ubatches})."
)
return self
def reconfigure_for_independent_dp_rank(self) -> None:
+43 -2
View File
@@ -2596,8 +2596,8 @@ class VllmConfig:
):
unsupported.append("EAGLE3 with pipeline parallelism")
if self.parallel_config.enable_dbo:
unsupported.append("dual batch overlap")
if self.parallel_config.use_ubatching:
unsupported.extend(self._get_dbo_unsupported_features())
if self.parallel_config.enable_elastic_ep:
unsupported.append("elastic expert parallelism")
@@ -2739,6 +2739,47 @@ class VllmConfig:
f"{'; '.join(blockers)}."
)
def _get_dbo_unsupported_features(self) -> list[str]:
"""Collect what the V2 model runner cannot combine with DBO.
The V2 runner microbatches a plain decoder forward pass. Anything that
slices or replays the batch differently (drafting, adapters, pipeline
stages, context parallelism, encoders) is not handled yet.
"""
# TODO: DBO with model runner V2 is under development.
# It should be enabled with explicit VLLM_USE_V2_MODEL_RUNNER environ.
# Remove it when stable.
if envs.VLLM_USE_V2_MODEL_RUNNER is None:
return ["dual batch overlap"]
unsupported: list[str] = []
model_config = self.model_config
parallel_config = self.parallel_config
if self.lora_config is not None:
unsupported.append("dual batch overlap with LoRA")
if self.speculative_config is not None:
unsupported.append("dual batch overlap with speculative decoding")
if parallel_config.pipeline_parallel_size > 1:
unsupported.append("dual batch overlap with pipeline parallelism")
if (
parallel_config.decode_context_parallel_size > 1
or parallel_config.prefill_context_parallel_size > 1
):
unsupported.append("dual batch overlap with context parallelism")
if model_config is not None and (
model_config.is_multimodal_model or model_config.is_encoder_decoder
):
unsupported.append("dual batch overlap with multimodal models")
if model_config is not None and model_config.is_hybrid:
unsupported.append("dual batch overlap with hybrid models")
if self.compilation_config.cudagraph_mode != CUDAGraphMode.NONE:
unsupported.append("dual batch overlap with CUDA graphs")
if self.is_mm_encoder_only:
unsupported.append("dual batch overlap with encoder only models")
return unsupported
def _validate_v2_model_runner(self) -> None:
"""Check for features not yet supported by the V2 model runner."""
if not HAS_TRITON:
@@ -988,7 +988,9 @@ class DiffusionGemmaModelState(ModelState):
attn_groups,
kv_cache_config,
for_capture=False,
ubatch_idx: int = 0,
) -> dict[str, Any]:
assert ubatch_idx == 0, "DBO is not supported"
if cudagraph_mode == CUDAGraphMode.FULL:
num_reqs = input_batch.num_reqs_after_padding
num_tokens = input_batch.num_tokens_after_padding
+15 -8
View File
@@ -22,6 +22,7 @@ from vllm.v1.kv_cache_interface import (
UniformTypeKVCacheSpecs,
)
from vllm.v1.worker.gpu.model_states.interface import ModelSpecificAttnMetadata
from vllm.v1.worker.ubatch_utils import get_num_ubatches
from vllm.v1.worker.utils import (
AttentionGroup,
add_kv_sharing_layers_to_kv_cache_groups,
@@ -140,14 +141,19 @@ def init_attn_backend(
vllm_config=vllm_config,
device=device,
kernel_block_size=kernel_block_size,
num_metadata_builders=1,
# Microbatches build attention metadata concurrently, and some
# builders keep the prepared metadata on themselves (MLA stores
# it on the prefill backend), so each ubatch needs its own.
num_metadata_builders=get_num_ubatches(vllm_config.parallel_config),
)
builder = group.get_metadata_builder(0)
if attn_backend_workspace is None:
if hasattr(builder, "_get_workspace_buffer"):
attn_backend_workspace = builder._get_workspace_buffer()
else:
if hasattr(builder, "set_workspace_buffer"):
# The microbatches' builders share the workspace: they all issue
# attention on the one compute stream the threads hand off, so the
# buffer is written serially, as it already is across steps.
for builder in group.metadata_builders:
if attn_backend_workspace is None:
if hasattr(builder, "_get_workspace_buffer"):
attn_backend_workspace = builder._get_workspace_buffer()
elif hasattr(builder, "set_workspace_buffer"):
builder.set_workspace_buffer(attn_backend_workspace)
attn_cg_support_info = get_attn_cg_support(attn_groups, vllm_config)
return attn_groups, attn_cg_support_info, kernel_block_sizes
@@ -271,6 +277,7 @@ def build_attn_metadata(
for_cudagraph_capture: bool = False,
causal: bool | torch.Tensor | Mapping[int, bool] = True,
rswa_prefix_lens: torch.Tensor | None = None,
ubatch_idx: int = 0,
) -> dict[str, Any]:
seq_lens = seq_lens[:num_reqs]
if dcp_local_seq_lens is not None:
@@ -319,7 +326,7 @@ def build_attn_metadata(
)
for attn_group in attn_groups[i]:
attn_metadata_builder = attn_group.get_metadata_builder(0)
attn_metadata_builder = attn_group.get_metadata_builder(ubatch_idx)
if for_cudagraph_capture:
metadata = attn_metadata_builder.build_for_cudagraph_capture(
common_attn_metadata
+8
View File
@@ -68,6 +68,8 @@ class BatchExecutionDescriptor:
# uniform_token_count unset, so this is what keeps a prefill batch out of one.
max_query_len: int | None = None
num_active_loras: int = 0
# Number of microbatches the batch is split into (DBO). 1 means no splitting.
num_ubatches: int = 1
def make_cudagraph_stats(
@@ -100,11 +102,13 @@ def _is_compatible(
uniform_token_count: int | None,
num_active_loras: int,
max_query_len: int | None,
num_ubatches: int,
) -> bool:
# desc.uniform_token_count=None (PIECEWISE) can handle any uniform_token_count
# desc.num_reqs=None means no request padding needed (PIECEWISE)
# desc.max_query_len=None means the graph does not constrain query length; a
# caller that does not track max_query_len must not match one that does
# A graph captured for N microbatches can only serve a batch split N ways.
return (
(
desc.uniform_token_count is None
@@ -117,6 +121,7 @@ def _is_compatible(
and (desc.num_reqs is None or desc.num_reqs >= num_reqs)
and desc.num_tokens >= num_tokens
and desc.num_active_loras == num_active_loras
and desc.num_ubatches == num_ubatches
)
@@ -434,6 +439,7 @@ class CudaGraphManager:
uniform_token_count: int | None,
num_active_loras: int,
max_query_len: int | None = None,
num_ubatches: int = 1,
) -> BatchExecutionDescriptor:
"""Find matching cudagraph descriptor from priority-ordered candidates."""
@@ -448,6 +454,7 @@ class CudaGraphManager:
uniform_token_count,
effective_loras,
max_query_len,
num_ubatches,
):
return desc
return BatchExecutionDescriptor(
@@ -455,6 +462,7 @@ class CudaGraphManager:
num_tokens=num_tokens,
num_reqs=num_reqs,
num_active_loras=effective_loras,
num_ubatches=num_ubatches,
)
def run_fullgraph(self, desc: BatchExecutionDescriptor):
+55 -5
View File
@@ -7,12 +7,14 @@ from dataclasses import dataclass, replace
import torch
import torch.distributed as dist
from vllm.config import ParallelConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.distributed.parallel_state import get_dp_group
from vllm.v1.worker.gpu.cudagraph_utils import (
BatchExecutionDescriptor,
CudaGraphManager,
)
from vllm.v1.worker.ubatch_utils import check_ubatch_thresholds, get_num_ubatches
@dataclass(frozen=True)
@@ -48,27 +50,35 @@ def sync_cudagraph_and_dp_padding(
dp_rank: int,
max_query_len: int | None = None,
num_active_loras: int = 0,
parallel_config: ParallelConfig | None = None,
allow_ubatching: bool = False,
uniform_decode: bool = False,
) -> tuple[BatchExecutionDescriptor, DPSyncState | None]:
"""
Coordinates the batch descriptor and DP padding across all ranks.
`parallel_config` is only needed to decide whether to microbatch, so callers
that never do (`allow_ubatching=False`) can leave it out.
Returns (synced_batch_desc, sync). `sync` is None when no rank has work.
"""
assert dp_size > 1, "DP size must be greater than 1"
group = get_dp_group().cpu_group
tensor = torch.zeros(5, dp_size, dtype=torch.int32, device="cpu")
tensor = torch.zeros(6, dp_size, dtype=torch.int32, device="cpu")
tensor[0][dp_rank] = num_tokens
tensor[1][dp_rank] = desired_batch_desc.cg_mode.value
tensor[2][dp_rank] = uniform_token_count or 0 # (0 means None)
tensor[3][dp_rank] = max_query_len or -1 # (-1 means None)
tensor[4][dp_rank] = num_reqs
tensor[4][dp_rank] = int(allow_ubatching)
tensor[5][dp_rank] = num_reqs
dist.all_reduce(tensor, group=group)
num_tokens_across_dp = tensor[0]
cg_mode_across_dp = tensor[1]
uniform_token_counts_across_dp = tensor[2]
max_query_lens_across_dp = tensor[3]
num_reqs_across_dp = tensor[4]
allow_ubatching_across_dp = tensor[4]
num_reqs_across_dp = tensor[5]
# If ranks disagree on the uniform token count, or its 0 (means None) set to None
synced_uniform_token_count: int | None = int(uniform_token_counts_across_dp[0])
@@ -83,6 +93,40 @@ def sync_cudagraph_and_dp_padding(
)
return synced_desc, None
if torch.all(allow_ubatching_across_dp == 1):
# This rank voted too, so its caller passed a config.
assert parallel_config is not None
# A uniform decode only if every rank runs the same uniform query
# length, so every rank reaches the same answer here.
uniform_decode_across_dp = uniform_decode and torch.all(
uniform_token_counts_across_dp == (uniform_token_count or 0)
)
if check_ubatch_thresholds(
parallel_config,
# Thresholds only grow with the token count, so holding the smallest
# rank to them holds every rank to them.
int(num_tokens_across_dp.min()),
uniform_decode=uniform_decode_across_dp,
):
# Microbatching is all-or-nothing: the expert all-to-all is
# collective, so every rank splits and pads to the same token count.
# A rank too small to fill a microbatch does no work in it, like a
# dummy run. Microbatched steps run eager; nothing is captured yet.
ubatch_num_tokens = int(num_tokens_across_dp.max())
return BatchExecutionDescriptor(
cg_mode=CUDAGraphMode.NONE,
num_tokens=ubatch_num_tokens,
num_reqs=num_reqs,
num_ubatches=get_num_ubatches(parallel_config),
), DPSyncState(
num_tokens_across_dp=torch.full_like(
num_tokens_across_dp, ubatch_num_tokens
),
uniform_token_count=synced_uniform_token_count,
eager=True,
num_reqs=int(num_reqs_across_dp.max()),
)
synced_cg_mode = CUDAGraphMode(int(cg_mode_across_dp.min().item()))
# If any rank wants to run eager, all ranks run eager
@@ -98,7 +142,7 @@ def sync_cudagraph_and_dp_padding(
num_tokens_across_dp=num_tokens_across_dp,
uniform_token_count=synced_uniform_token_count,
eager=True,
num_reqs=int(num_reqs_across_dp.max().item()),
num_reqs=int(num_reqs_across_dp.max()),
),
)
@@ -136,7 +180,7 @@ def sync_cudagraph_and_dp_padding(
synced_desc.num_reqs
if synced_desc.cg_mode == CUDAGraphMode.FULL
and synced_desc.num_reqs is not None
else int(num_reqs_across_dp.max().item())
else int(num_reqs_across_dp.max())
),
)
@@ -151,6 +195,9 @@ def dispatch_cg_and_sync_dp(
max_query_len: int | None = None,
need_eager: bool = False,
num_active_loras: int = 0,
parallel_config: ParallelConfig | None = None,
allow_ubatching: bool = False,
uniform_decode: bool = False,
dp_sync: DPSyncState | None = None,
) -> tuple[BatchExecutionDescriptor, DPSyncState | None]:
"""Pick a cudagraph descriptor for this batch, agreeing it across DP ranks.
@@ -239,4 +286,7 @@ def dispatch_cg_and_sync_dp(
dp_rank,
max_query_len=max_query_len,
num_active_loras=num_active_loras,
parallel_config=parallel_config,
allow_ubatching=allow_ubatching,
uniform_decode=uniform_decode,
)
+41 -3
View File
@@ -161,6 +161,11 @@ from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
from vllm.v1.worker.gpu.spec_decode.utils import DraftTokensHandler
from vllm.v1.worker.gpu.states import RequestState
from vllm.v1.worker.gpu.structured_outputs import StructuredOutputsWorker
from vllm.v1.worker.gpu.ubatch_utils import (
UBatchRunner,
UBatchState,
maybe_build_ubatch_runner,
)
from vllm.v1.worker.lora_model_runner_mixin import LoRAModelRunnerMixin
from vllm.v1.worker.utils import (
KVBlockZeroer,
@@ -230,6 +235,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.dp_size = self.parallel_config.data_parallel_size
self.dp_rank = self.parallel_config.data_parallel_rank
# Dual batch overlap. Created in initialize_kv_cache(), once everything
# it runs the microbatched forward with exists.
self.ubatch_runner: UBatchRunner | None = None
# Detect EP all2all peer faults to prevent emitting corrupted output.
# Only meaningful for MoE + DP with an FT-capable all2all backend.
self.check_ep_fault = False
@@ -634,6 +643,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.block_tables,
cls=self.pcp_manager_cls,
)
self.ubatch_runner = maybe_build_ubatch_runner(
self.vllm_config,
self.device,
self.model_state,
self.attn_groups,
self.kv_cache_config,
self.max_num_reqs,
)
initialize_mamba_ssu_backend(
self.vllm_config.mamba_config,
self.kv_cache_config,
@@ -1597,6 +1614,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
max_query_len=max_query_len,
need_eager=is_profile or skip_compiled,
num_active_loras=num_active_loras,
parallel_config=self.parallel_config,
allow_ubatching=(
self.ubatch_runner is not None and not skip_attn_for_dummy_run
),
uniform_decode=uniform_tok_count == self.decode_query_len,
)
if batch_desc.num_tokens == 0:
@@ -1665,7 +1687,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
attn_metadata = None
slot_mappings_by_layer = None
if not (dummy_run and skip_attn_for_dummy_run):
ubatch_state: UBatchState | None = None
if batch_desc.num_ubatches > 1:
assert self.ubatch_runner is not None
assert block_tables is not None and slot_mappings is not None
ubatch_state = self.ubatch_runner.prepare(
input_batch, block_tables, slot_mappings
)
elif not (dummy_run and skip_attn_for_dummy_run):
assert slot_mappings is not None
slot_mappings_by_layer = build_slot_mappings_by_layer(
slot_mappings, self.kv_cache_config
@@ -1751,7 +1780,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
del intermediate_tensors
# Update the EPLB meta.
self.eplb.prepare_forward(self.model_config, input_batch.num_tokens)
ubatch_slices = ubatch_state.slices if ubatch_state is not None else None
self.eplb.prepare_forward(
self.model_config, input_batch.num_tokens, ubatch_slices
)
self.step_timing.record_batch(
input_batch, batch_desc.cg_mode == CUDAGraphMode.FULL
@@ -1783,12 +1815,18 @@ class GPUModelRunner(LoRAModelRunnerMixin):
dp_sync.num_tokens_across_dp if dp_sync is not None else None
),
batch_descriptor=batch_descriptor,
ubatch_slices=ubatch_slices,
slot_mapping=slot_mappings_by_layer,
skip_compiled=skip_compiled,
is_padding=input_batch.is_padding,
):
self.kv_connector.pre_forward(scheduler_output)
if batch_desc.cg_mode == CUDAGraphMode.PIECEWISE:
if ubatch_state is not None:
assert self.ubatch_runner is not None
model_output = self.ubatch_runner.run(
self.model, model_inputs, ubatch_state
)
elif batch_desc.cg_mode == CUDAGraphMode.PIECEWISE:
# Run the PIECEWISE graph (compiled PW cudagraph or breakable
# cudagraph, chosen inside run_pw_graph). cg_mode is only
# PIECEWISE after the cudagraph manager exists.
@@ -172,6 +172,7 @@ class DefaultModelState(ModelState):
attn_groups: list[list[AttentionGroup]],
kv_cache_config: KVCacheConfig,
for_capture: bool = False,
ubatch_idx: int = 0,
) -> dict[str, Any]:
if cudagraph_mode == CUDAGraphMode.FULL:
# Use padded sizes - padding is handled by model_runner.prepare_attn.
@@ -224,5 +225,6 @@ class DefaultModelState(ModelState):
mm_req_doc_ranges=req_doc_ranges,
for_cudagraph_capture=for_capture,
rswa_prefix_lens=input_batch.prompt_lens,
ubatch_idx=ubatch_idx,
)
return attn_metadata
@@ -116,7 +116,9 @@ class EncoderDecoderModelState(ModelState):
attn_groups: list[list[AttentionGroup]],
kv_cache_config: KVCacheConfig,
for_capture: bool = False,
ubatch_idx: int = 0,
) -> dict[str, Any]:
assert ubatch_idx == 0, "DBO is not supported"
if cudagraph_mode == CUDAGraphMode.FULL:
num_reqs = input_batch.num_reqs_after_padding
num_tokens = input_batch.num_tokens_after_padding
@@ -167,7 +167,9 @@ class EncoderOnlyModelState(DefaultModelState):
attn_groups: list[list[AttentionGroup]],
kv_cache_config: KVCacheConfig,
for_capture: bool = False,
ubatch_idx: int = 0,
) -> dict[str, Any]:
assert ubatch_idx == 0, "DBO is not supported"
attn_metadata = super().prepare_attn(
input_batch,
cudagraph_mode,
@@ -221,6 +221,7 @@ class ModelState(ABC):
attn_groups: list[list[AttentionGroup]],
kv_cache_config: KVCacheConfig,
for_capture: bool = False,
ubatch_idx: int = 0,
) -> dict[str, Any]:
raise NotImplementedError
@@ -238,7 +238,9 @@ class MambaHybridModelState(DefaultModelState):
attn_groups: list[list[AttentionGroup]],
kv_cache_config: KVCacheConfig,
for_capture: bool = False,
ubatch_idx: int = 0,
) -> dict[str, Any]:
assert ubatch_idx == 0, "DBO is not supported"
if cudagraph_mode == CUDAGraphMode.FULL:
num_reqs = input_batch.num_reqs_after_padding
num_tokens = input_batch.num_tokens_after_padding
+473
View File
@@ -0,0 +1,473 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Microbatching (DBO) helpers for the V2 GPU model runner."""
import threading
from dataclasses import replace
from typing import Any, NamedTuple
import numpy as np
import torch
from vllm.config import CUDAGraphMode, VllmConfig
from vllm.forward_context import (
DPMetadata,
ForwardContext,
create_forward_context,
override_forward_context,
)
from vllm.logger import init_logger
from vllm.sequence import IntermediateTensors
from vllm.utils.torch_utils import current_stream
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.model_states.interface import ModelState
from vllm.v1.worker.ubatch_utils import (
UBatchSlice,
UBatchSlices,
create_sm_control_context,
maybe_create_ubatch_slices,
)
from vllm.v1.worker.ubatching import make_ubatch_contexts
from vllm.v1.worker.utils import AttentionGroup
logger = init_logger(__name__)
class UBatchState(NamedTuple):
"""Per-microbatch inputs for one dual-batch-overlap step."""
slices: UBatchSlices
forward_contexts: list[ForwardContext]
def create_ubatch_slices(input_batch: InputBatch, num_ubatches: int) -> UBatchSlices:
"""Split a DP-padded batch into the slices its microbatches run on.
Splitting at the midpoint of the DP-padded token count leaves the trailing
microbatch anywhere from full to entirely inside the padding. Clamping its
request slice onto the last request leaves it holding that request with
zero query tokens -- the shape a straddling request already takes -- so it
stays well-formed with no work to do, like a dummy run.
"""
_, ubatch_slices = maybe_create_ubatch_slices(
True,
input_batch.num_scheduled_tokens,
input_batch.num_tokens_after_padding,
input_batch.num_reqs_after_padding,
num_ubatches,
)
assert ubatch_slices is not None
return [
UBatchSlice(
slice(
min(s.request_slice.start, input_batch.num_reqs - 1),
min(s.request_slice.stop, input_batch.num_reqs_after_padding),
),
s.token_slice,
)
for s in ubatch_slices
]
def _slice_input_batch(
input_batch: InputBatch,
ubatch_slice: UBatchSlice,
query_start_loc_buf: torch.Tensor,
seq_lens_buf: torch.Tensor,
) -> InputBatch:
"""Build the sub-`InputBatch` a single microbatch runs on.
A request straddling the boundary appears in both microbatches, its query
truncated to the tokens each one owns.
Everything except `query_start_loc` and `seq_lens` is a view of the full
batch's buffers, so slicing costs no allocation. Those two are rebased onto
the microbatch's token range and written to the caller's buffers, which
keeps them valid under CUDA graph capture and replay.
The sub-batch describes the forward pass only. Sampling runs once over the
merged batch, so `logits_indices`, `cu_num_logits` and the draft-token
fields are carried over unchanged and must not be read off a sub-batch.
"""
assert not ubatch_slice.is_empty(), f"Ubatch slice {ubatch_slice} is empty"
req_start = ubatch_slice.request_slice.start
req_stop = ubatch_slice.request_slice.stop
tok_start = ubatch_slice.token_slice.start
tok_stop = ubatch_slice.token_slice.stop
num_reqs_padded = req_stop - req_start
num_tokens_padded = tok_stop - tok_start
# The trailing microbatch may extend past the real batch into CUDA graph
# padding; the unpadded counts stop at the last real request/token.
num_reqs = max(0, min(req_stop, input_batch.num_reqs) - req_start)
num_tokens = max(0, min(tok_stop, input_batch.num_tokens) - tok_start)
query_start_loc = _slice_query_start_loc(
input_batch.query_start_loc,
query_start_loc_buf[: num_reqs_padded + 1],
req_start,
req_stop,
tok_start,
num_tokens_padded,
)
query_start_loc_np = np.clip(
input_batch.query_start_loc_np[req_start : req_stop + 1] - tok_start,
0,
num_tokens_padded,
).astype(np.int32)
seq_lens = _slice_seq_lens(
input_batch.seq_lens,
seq_lens_buf[:num_reqs_padded],
input_batch.query_start_loc,
req_start,
req_stop,
tok_stop,
)
# Same truncation as `_slice_seq_lens`, on the host-side upper bound.
seq_lens_cpu_upper_bound = input_batch.seq_lens_cpu_upper_bound[
req_start:req_stop
].clone()
tokens_truncated = max(0, int(input_batch.query_start_loc_np[req_stop]) - tok_stop)
if tokens_truncated:
seq_lens_cpu_upper_bound[-1] -= tokens_truncated
# Query lengths of the truncated requests, so consumers that derive
# max_query_len from this array see the microbatch's own lengths.
num_scheduled_tokens = np.diff(query_start_loc_np)[:num_reqs]
dcp_local_seq_lens = input_batch.dcp_local_seq_lens
if dcp_local_seq_lens is not None:
# NOTE: a request split across microbatches keeps its full local
# seq_len here. DCP is not yet validated with DBO.
dcp_local_seq_lens = dcp_local_seq_lens[req_start:req_stop]
return replace(
input_batch,
req_ids=input_batch.req_ids[req_start : min(req_stop, input_batch.num_reqs)],
num_reqs=num_reqs,
num_reqs_after_padding=num_reqs_padded,
idx_mapping=input_batch.idx_mapping[req_start:req_stop],
idx_mapping_np=input_batch.idx_mapping_np[req_start:req_stop],
num_scheduled_tokens=num_scheduled_tokens,
num_tokens=num_tokens,
num_tokens_after_padding=num_tokens_padded,
query_start_loc=query_start_loc,
query_start_loc_np=query_start_loc_np,
seq_lens=seq_lens,
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
dcp_local_seq_lens=dcp_local_seq_lens,
num_computed_tokens_np=input_batch.num_computed_tokens_np[req_start:req_stop],
prefill_len_np=input_batch.prefill_len_np[req_start:req_stop],
num_computed_prefill_tokens_np=input_batch.num_computed_prefill_tokens_np[
req_start:req_stop
],
is_prefilling_np=input_batch.is_prefilling_np[req_start:req_stop],
input_ids=input_batch.input_ids[tok_start:tok_stop],
positions=input_batch.positions[tok_start:tok_stop],
is_padding=input_batch.is_padding[tok_start:tok_stop],
prompt_lens=(
None
if input_batch.prompt_lens is None
else input_batch.prompt_lens[req_start:req_stop]
),
)
def _slice_query_start_loc(
query_start_loc: torch.Tensor,
out: torch.Tensor,
req_start: int,
req_stop: int,
tok_start: int,
num_tokens_padded: int,
) -> torch.Tensor:
"""Rebase query_start_loc onto the microbatch's token range.
The clamp truncates the requests straddling either boundary to the tokens
this microbatch owns.
"""
torch.sub(query_start_loc[req_start : req_stop + 1], tok_start, out=out)
return out.clamp_(0, num_tokens_padded)
def _slice_seq_lens(
seq_lens: torch.Tensor,
out: torch.Tensor,
query_start_loc: torch.Tensor,
req_start: int,
req_stop: int,
tok_stop: int,
) -> torch.Tensor:
"""Slice seq_lens, shortening the request truncated at the boundary.
Only tokens past the end of this microbatch are missing; tokens it owns in
an *earlier* microbatch are computed before this one reads them, so they
still count. Deriving the truncation from this microbatch's own query
lengths would subtract them twice. It is computed from tensors, not Python
ints, so this stays valid inside a CUDA graph.
"""
out.copy_(seq_lens[req_start:req_stop])
last = req_stop - req_start - 1
out[last] -= (query_start_loc[req_stop] - tok_stop).clamp_(min=0)
return out
def slice_model_inputs(
model_inputs: dict[str, Any], token_slice: slice
) -> dict[str, Any]:
"""Narrow the model's per-token inputs to one microbatch."""
sliced = dict(model_inputs)
for key in ("input_ids", "inputs_embeds"):
value = model_inputs.get(key)
if value is not None:
sliced[key] = value[token_slice]
positions = model_inputs["positions"]
# M-RoPE carries a leading section dim.
sliced["positions"] = (
positions[:, token_slice] if positions.ndim == 2 else positions[token_slice]
)
intermediate_tensors = model_inputs.get("intermediate_tensors")
if intermediate_tensors is not None:
sliced["intermediate_tensors"] = intermediate_tensors[token_slice]
return sliced
def merge_ubatch_outputs(outputs: list[Any]) -> Any:
"""Reassemble the full-batch model output from the per-microbatch ones.
Models return hidden states, a tuple of those plus auxiliary hidden states
(EAGLE3), or IntermediateTensors on non-final pipeline ranks.
"""
first = outputs[0]
if isinstance(first, IntermediateTensors):
return IntermediateTensors(
{
key: torch.cat([out.tensors[key] for out in outputs], dim=0)
for key in first.tensors
}
)
if isinstance(first, tuple):
return tuple(torch.cat(parts, dim=0) for parts in zip(*outputs))
return torch.cat(outputs, dim=0)
class UBatchRunner:
"""Runs the model on two or more microbatches that overlap each other.
Each microbatch gets a thread and its own forward context. Only one thread
holds the GPU at a time; they hand off to each other at the communication
points inside the model (see `vllm.v1.worker.ubatching`), so one
microbatch's expert all-to-all overlaps the other's compute.
"""
def __init__(
self,
vllm_config: VllmConfig,
device: torch.device,
model_state: ModelState,
attn_groups: list[list[AttentionGroup]],
kv_cache_config: KVCacheConfig,
max_num_reqs: int,
):
self.vllm_config = vllm_config
self.parallel_config = vllm_config.parallel_config
self.num_ubatches = self.parallel_config.num_ubatches
self.device = device
self.model_state = model_state
self.attn_groups = attn_groups
self.kv_cache_config = kv_cache_config
# `query_start_loc` and `seq_lens` are rebased onto each microbatch's
# own token range, so they cannot be views of the full batch's buffers.
# Allocating up front keeps their addresses stable across replays.
self.ubatch_query_start_loc = [
torch.zeros(max_num_reqs + 1, dtype=torch.int32, device=device)
for _ in range(self.num_ubatches)
]
self.ubatch_seq_lens = [
torch.zeros(max_num_reqs, dtype=torch.int32, device=device)
for _ in range(self.num_ubatches)
]
self.comm_stream = torch.cuda.Stream(device=device)
# The microbatch threads plus the thread that starts them.
self.ready_barrier = threading.Barrier(self.num_ubatches + 1)
self.sm_control = create_sm_control_context(self.parallel_config)
def prepare(
self,
input_batch: InputBatch,
block_tables: tuple[torch.Tensor, ...],
slot_mappings: torch.Tensor,
) -> UBatchState:
"""Split the batch into the microbatches the step will run on.
Attention metadata is built per microbatch and carried in the forward
contexts the threads install, not in the caller's.
"""
ubatch_slices = create_ubatch_slices(input_batch, self.num_ubatches)
attn_metadata = []
slot_mappings_by_layer = []
is_padding = []
for i, ubatch_slice in enumerate(ubatch_slices):
ubatch = _slice_input_batch(
input_batch,
ubatch_slice,
self.ubatch_query_start_loc[i],
self.ubatch_seq_lens[i],
)
ubatch_slot_mappings = slot_mappings[:, ubatch_slice.token_slice]
ubatch_block_tables = tuple(
block_table[ubatch_slice.request_slice] for block_table in block_tables
)
attn_metadata.append(
self.model_state.prepare_attn(
ubatch,
CUDAGraphMode.NONE,
ubatch_block_tables,
ubatch_slot_mappings,
self.attn_groups,
self.kv_cache_config,
ubatch_idx=i,
)
)
slot_mappings_by_layer.append(
build_slot_mappings_by_layer(ubatch_slot_mappings, self.kv_cache_config)
)
is_padding.append(ubatch.is_padding)
return UBatchState(
slices=ubatch_slices,
forward_contexts=self._make_forward_contexts(
ubatch_slices, attn_metadata, slot_mappings_by_layer, is_padding
),
)
def _make_forward_contexts(
self,
ubatch_slices: UBatchSlices,
attn_metadata: list[dict[str, Any]],
slot_mappings_by_layer: list[dict[str, torch.Tensor]],
is_padding: list[torch.Tensor | None],
) -> list[ForwardContext]:
"""Build one forward context per microbatch.
The expert all-to-all runs once per microbatch, so each context's DP
metadata carries that microbatch's token count. Microbatching pads every
DP rank alike, so each rank can assume that count for the others too.
"""
dp_size = self.parallel_config.data_parallel_size
forward_contexts = []
for i, ubatch_slice in enumerate(ubatch_slices):
num_tokens = ubatch_slice.num_tokens
num_tokens_across_dp = torch.full(
(dp_size,), num_tokens, dtype=torch.int32, device="cpu"
)
forward_contexts.append(
create_forward_context(
attn_metadata[i],
self.vllm_config,
dp_metadata=DPMetadata.make(
self.parallel_config, num_tokens, num_tokens_across_dp
),
cudagraph_runtime_mode=CUDAGraphMode.NONE,
slot_mapping=slot_mappings_by_layer[i],
is_padding=is_padding[i],
)
)
return forward_contexts
def run(
self,
model: Any,
model_inputs: dict[str, Any],
ubatch_state: UBatchState,
) -> Any:
ubatch_slices = ubatch_state.slices
assert len(ubatch_slices) == len(ubatch_state.forward_contexts)
assert len(ubatch_slices) == self.num_ubatches
ubatch_contexts = make_ubatch_contexts(
num_micro_batches=self.num_ubatches,
comm_stream=self.comm_stream,
compute_stream=current_stream(),
forward_contexts=ubatch_state.forward_contexts,
ready_barrier=self.ready_barrier,
)
outputs: dict[int, Any] = {}
errors: dict[int, BaseException] = {}
@torch.inference_mode()
def run_ubatch(ubatch_context, inputs: dict[str, Any]) -> None:
try:
with ubatch_context:
outputs[ubatch_context.id] = model(**inputs)
except BaseException as e: # noqa: BLE001
# Recorded so the join below can report which microbatch broke.
# Note the siblings are not told to unwind: microbatches hand
# control to each other, so one that dies mid-forward leaves
# the others parked on a handoff that never comes and the step
# hangs. The V1 runner behaves the same way; fixing it means
# changing the shared handoff protocol in ubatching.py, which
# is deliberately out of scope here.
errors[ubatch_context.id] = e
# The threads manage the forward context themselves; clear it here so
# it is restored correctly once they are done.
with override_forward_context(None), self.sm_control:
threads = []
for ubatch_context, ubatch_slice in zip(ubatch_contexts, ubatch_slices):
thread = threading.Thread(
target=run_ubatch,
args=(
ubatch_context,
slice_model_inputs(model_inputs, ubatch_slice.token_slice),
),
)
threads.append(thread)
thread.start()
# Wait for every thread to reach its context, then start the first.
self.ready_barrier.wait()
ubatch_contexts[0].cpu_wait_event.set()
for thread in threads:
thread.join()
if errors:
failed = min(errors)
raise RuntimeError(
f"Microbatch {failed} of {self.num_ubatches} failed"
) from errors[failed]
return merge_ubatch_outputs([outputs[i] for i in range(self.num_ubatches)])
def maybe_build_ubatch_runner(
vllm_config: VllmConfig,
device: torch.device,
model_state: ModelState,
attn_groups: list[list[AttentionGroup]],
kv_cache_config: KVCacheConfig,
max_num_reqs: int,
) -> UBatchRunner | None:
"""Build the microbatch runner, or None when DBO is not in use.
Microbatching needs the DP handshake to agree on it, so it is only
available with more than one DP rank (as in the V1 runner).
"""
parallel_config = vllm_config.parallel_config
if not parallel_config.use_ubatching or parallel_config.data_parallel_size <= 1:
return None
logger.info_once(
"Dual batch overlap is enabled. Microbatched steps run without "
"CUDA graphs on the V2 model runner."
)
return UBatchRunner(
vllm_config, device, model_state, attn_groups, kv_cache_config, max_num_reqs
)
+2 -93
View File
@@ -11,7 +11,6 @@ import torch
import vllm.envs as envs
from vllm.compilation.cuda_graph import CUDAGraphWrapper
from vllm.config import CUDAGraphMode, VllmConfig
from vllm.distributed import get_ep_group
from vllm.distributed.device_communicators.pynccl_allocator import set_graph_pool_id
from vllm.forward_context import (
DPMetadata,
@@ -23,9 +22,7 @@ from vllm.logger import init_logger
from vllm.model_executor.offloader.base import get_offloader
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.utils.deep_gemm import set_num_sms as deep_gemm_set_num_sms
from vllm.utils.import_utils import has_deep_gemm
from vllm.utils.platform_utils import num_compute_units
from vllm.v1.worker.ubatch_utils import create_sm_control_context
from vllm.v1.worker.ubatching import UBatchContext, make_ubatch_contexts
logger = init_logger(__name__)
@@ -65,51 +62,6 @@ class CUDAGraphMetaData:
outputs: Any | None = None
class SMControlContextManager:
def __init__(
self,
comm_sms: int,
set_comm_sms: Callable[[int], None],
set_compute_sms: Callable[[int], None],
):
"""
Context manager for controlling SM (Streaming Multiprocessor)
allocation. Upon entering the context, it sets the number of SMs
allocated for communication and computation to comm_sms and
total_sms - comm_sms respectively. Upon exiting, it restores the
allocation to use all available SMs (i.e. total_sms).
Args:
comm_sms (int): The number of SMs to allocate for communication.
(The remainder will be used for computation.)
set_comm_sms (Callable[[int], None]):
A function that sets the number of SMs for communication.
set_compute_sms (Callable[[int], None]):
A function that sets the number of SMs for computation.
"""
assert current_platform.is_cuda() or current_platform.is_rocm(), (
"SM/CU control is supported on CUDA and ROCm platforms"
)
device = torch.accelerator.current_device_index()
total_sms = num_compute_units(device)
assert comm_sms < total_sms
self.total_sms = total_sms
self.compute_sms = total_sms - comm_sms
self.comm_sms = comm_sms
self.set_comm_sms = set_comm_sms
self.set_compute_sms = set_compute_sms
def __enter__(self):
self.set_comm_sms(self.comm_sms)
self.set_compute_sms(self.compute_sms)
def __exit__(self, exc_type, exc_value, traceback):
self.set_comm_sms(self.total_sms)
self.set_compute_sms(self.total_sms)
class UBatchWrapper:
def __init__(
self,
@@ -135,7 +87,7 @@ class UBatchWrapper:
runnable, vllm_config, runtime_mode=runtime_mode
)
self.sm_control = self._create_sm_control_context(vllm_config)
self.sm_control = create_sm_control_context(vllm_config.parallel_config)
self.device = device
self.is_debugging_mode = envs.VLLM_LOGGING_LEVEL == "DEBUG"
self._runnable_str = str(runnable) if self.is_debugging_mode else None
@@ -151,49 +103,6 @@ class UBatchWrapper:
if self.cudagraph_wrapper is not None:
self.cudagraph_wrapper.clear_graphs()
@staticmethod
def _create_sm_control_context(vllm_config: VllmConfig):
comm_sms: int = envs.VLLM_DBO_COMM_SMS
rocm_deepep_ht_dbo = (
current_platform.is_rocm()
and vllm_config.parallel_config.enable_dbo
and vllm_config.parallel_config.all2all_backend == "deepep_high_throughput"
)
if rocm_deepep_ht_dbo:
# On ROCm, reserving CUs for DeepEP HT communication under DBO
# corrupts DP+EP generation accuracy. Keep the backend active, but
# leave all CUs visible to the compute and communication kernels.
comm_sms = 0
set_comm_sms = lambda sms: None
if vllm_config.parallel_config.enable_expert_parallel:
# Currently only DeepEP highthroughput supports SM control so this
# only affects that case.
ep_group = get_ep_group()
device_communicator = ep_group.device_communicator
all2all_manager = None
if device_communicator is not None:
all2all_manager = device_communicator.all2all_manager
if all2all_manager is not None:
max_sms_used = all2all_manager.max_sms_used()
if max_sms_used is not None:
comm_sms = min(comm_sms, max_sms_used)
if comm_sms > 0 and all2all_manager is not None:
set_comm_sms = lambda sms: all2all_manager.set_num_sms(sms)
# TODO(lucas): support other kernels besides DeepGEMM
set_compute_sms = lambda sms: None
if has_deep_gemm() and comm_sms > 0:
set_compute_sms = lambda sms: deep_gemm_set_num_sms(sms)
return SMControlContextManager(
comm_sms=comm_sms,
set_comm_sms=set_comm_sms,
set_compute_sms=set_compute_sms,
)
def __getattr__(self, key: str):
# allow accessing the attributes of the runnable.
if hasattr(self.runnable, key):
+103
View File
@@ -1,12 +1,19 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from dataclasses import dataclass
from typing import TypeAlias
import numpy as np
import torch
import vllm.envs as envs
from vllm.config import ParallelConfig
from vllm.distributed import get_ep_group
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import set_num_sms as deep_gemm_set_num_sms
from vllm.utils.import_utils import has_deep_gemm
from vllm.utils.platform_utils import num_compute_units
from vllm.v1.attention.backend import CommonAttentionMetadata
@@ -29,12 +36,108 @@ class UBatchSlice:
UBatchSlices: TypeAlias = list[UBatchSlice]
class SMControlContextManager:
def __init__(
self,
comm_sms: int,
set_comm_sms: Callable[[int], None],
set_compute_sms: Callable[[int], None],
):
"""
Context manager for controlling SM (Streaming Multiprocessor)
allocation. Upon entering the context, it sets the number of SMs
allocated for communication and computation to comm_sms and
total_sms - comm_sms respectively. Upon exiting, it restores the
allocation to use all available SMs (i.e. total_sms).
Args:
comm_sms (int): The number of SMs to allocate for communication.
(The remainder will be used for computation.)
set_comm_sms (Callable[[int], None]):
A function that sets the number of SMs for communication.
set_compute_sms (Callable[[int], None]):
A function that sets the number of SMs for computation.
"""
assert current_platform.is_cuda() or current_platform.is_rocm(), (
"SM/CU control is supported on CUDA and ROCm platforms"
)
device = torch.accelerator.current_device_index()
total_sms = num_compute_units(device)
assert comm_sms < total_sms
self.total_sms = total_sms
self.compute_sms = total_sms - comm_sms
self.comm_sms = comm_sms
self.set_comm_sms = set_comm_sms
self.set_compute_sms = set_compute_sms
def __enter__(self):
self.set_comm_sms(self.comm_sms)
self.set_compute_sms(self.compute_sms)
def __exit__(self, exc_type, exc_value, traceback):
self.set_comm_sms(self.total_sms)
self.set_compute_sms(self.total_sms)
def create_sm_control_context(
parallel_config: ParallelConfig,
) -> SMControlContextManager:
"""Reserve SMs for communication kernels while microbatches overlap."""
comm_sms: int = envs.VLLM_DBO_COMM_SMS
rocm_deepep_ht_dbo = (
current_platform.is_rocm()
and parallel_config.enable_dbo
and parallel_config.all2all_backend == "deepep_high_throughput"
)
if rocm_deepep_ht_dbo:
# On ROCm, reserving CUs for DeepEP HT communication under DBO
# corrupts DP+EP generation accuracy. Keep the backend active, but
# leave all CUs visible to the compute and communication kernels.
comm_sms = 0
set_comm_sms = lambda sms: None
if parallel_config.enable_expert_parallel:
# Currently only DeepEP highthroughput supports SM control so this
# only affects that case.
ep_group = get_ep_group()
device_communicator = ep_group.device_communicator
all2all_manager = None
if device_communicator is not None:
all2all_manager = device_communicator.all2all_manager
if all2all_manager is not None:
max_sms_used = all2all_manager.max_sms_used()
if max_sms_used is not None:
comm_sms = min(comm_sms, max_sms_used)
if comm_sms > 0 and all2all_manager is not None:
set_comm_sms = lambda sms: all2all_manager.set_num_sms(sms)
# TODO(lucas): support other kernels besides DeepGEMM
set_compute_sms = lambda sms: None
if has_deep_gemm() and comm_sms > 0:
set_compute_sms = lambda sms: deep_gemm_set_num_sms(sms)
return SMControlContextManager(
comm_sms=comm_sms,
set_comm_sms=set_comm_sms,
set_compute_sms=set_compute_sms,
)
def is_last_ubatch_empty(
orig_num_tokens: int, padded_num_tokens: int, num_ubatches: int
) -> bool:
return (padded_num_tokens // num_ubatches) * (num_ubatches - 1) >= orig_num_tokens
def get_num_ubatches(parallel_config: ParallelConfig) -> int:
"""How many microbatches a step is split into; 1 when microbatching is off."""
return parallel_config.num_ubatches if parallel_config.use_ubatching else 1
def check_ubatch_thresholds(
config: ParallelConfig, num_tokens: int, uniform_decode: bool
) -> bool: