[1/2][Model Runner V2] DBO support, eager mode (#50945)
Signed-off-by: specture724 <specture724@gmail.com> Signed-off-by: Anjie Hou <149605198+specture724@users.noreply.github.com> Co-authored-by: jiangkuaixue123 <jiangxiaozhou111@163.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
co-authored by
jiangkuaixue123
Lucas Wilkinson
parent
784cac7c42
commit
3f41d102c5
@@ -453,6 +453,7 @@ def test_v2_model_runner_accepts_routed_experts(monkeypatch):
|
||||
distributed_executor_backend=None,
|
||||
pipeline_parallel_size=1,
|
||||
enable_dbo=False,
|
||||
use_ubatching=False,
|
||||
enable_elastic_ep=False,
|
||||
),
|
||||
compilation_config=SimpleNamespace(
|
||||
|
||||
@@ -0,0 +1,583 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Dual batch overlap (DBO) in the V2 model runner.
|
||||
|
||||
The V2 runner slices the batch *before* building attention metadata, where V1
|
||||
builds metadata for the whole batch and slices it afterwards. Both must describe
|
||||
the same microbatches, so the main test here pins V2's slicing against V1's
|
||||
`split_attn_metadata`. The rest covers the decision to microbatch and the
|
||||
threaded execution of the microbatches.
|
||||
"""
|
||||
|
||||
import threading
|
||||
from dataclasses import replace
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, cast
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
|
||||
from vllm.config import CUDAGraphMode, ModelConfig, ParallelConfig, VllmConfig
|
||||
from vllm.forward_context import create_forward_context
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.worker.gpu import dp_utils
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
from vllm.v1.worker.gpu.ubatch_utils import (
|
||||
UBatchRunner,
|
||||
UBatchState,
|
||||
_slice_input_batch,
|
||||
create_ubatch_slices,
|
||||
slice_model_inputs,
|
||||
)
|
||||
from vllm.v1.worker.ubatch_utils import (
|
||||
UBatchSlice,
|
||||
maybe_create_ubatch_slices,
|
||||
split_attn_metadata,
|
||||
)
|
||||
from vllm.v1.worker.ubatching import dbo_current_ubatch_id, dbo_yield
|
||||
|
||||
MAX_NUM_REQS = 32
|
||||
MAX_NUM_TOKENS = 128
|
||||
|
||||
|
||||
def _make_buffers() -> InputBuffers:
|
||||
return InputBuffers(
|
||||
max_num_reqs=MAX_NUM_REQS,
|
||||
max_num_tokens=MAX_NUM_TOKENS,
|
||||
device=torch.device("cpu"),
|
||||
)
|
||||
|
||||
|
||||
def _make_ubatch_buffers(num_ubatches: int = 2) -> list[tuple[torch.Tensor, ...]]:
|
||||
"""The per-microbatch (query_start_loc, seq_lens) buffers UBatchRunner owns."""
|
||||
return [
|
||||
(
|
||||
torch.zeros(MAX_NUM_REQS + 1, dtype=torch.int32),
|
||||
torch.zeros(MAX_NUM_REQS, dtype=torch.int32),
|
||||
)
|
||||
for _ in range(num_ubatches)
|
||||
]
|
||||
|
||||
|
||||
def _make_input_batch(
|
||||
query_lens: list[int],
|
||||
seq_lens: list[int],
|
||||
buffers: InputBuffers,
|
||||
num_reqs_padded: int | None = None,
|
||||
num_tokens_padded: int | None = None,
|
||||
) -> InputBatch:
|
||||
"""Build an InputBatch shaped like the one `prepare_inputs` produces."""
|
||||
num_reqs = len(query_lens)
|
||||
num_tokens = int(sum(query_lens))
|
||||
num_reqs_padded = num_reqs_padded or num_reqs
|
||||
num_tokens_padded = num_tokens_padded or num_tokens
|
||||
|
||||
# make_dummy writes the shared buffers, so fill them afterwards.
|
||||
base = InputBatch.make_dummy(num_reqs, num_tokens, buffers)
|
||||
|
||||
query_start_loc_np = np.zeros(num_reqs_padded + 1, dtype=np.int32)
|
||||
np.cumsum(query_lens, out=query_start_loc_np[1 : num_reqs + 1])
|
||||
# Padded entries repeat the token count, as `prepare_inputs` does.
|
||||
query_start_loc_np[num_reqs + 1 :] = num_tokens
|
||||
buffers.query_start_loc[: num_reqs_padded + 1] = torch.from_numpy(
|
||||
query_start_loc_np
|
||||
)
|
||||
|
||||
buffers.seq_lens[:num_reqs] = torch.tensor(seq_lens, dtype=torch.int32)
|
||||
buffers.seq_lens[num_reqs:num_reqs_padded] = 0
|
||||
|
||||
seq_lens_upper_bound = np.zeros(num_reqs_padded, dtype=np.int32)
|
||||
seq_lens_upper_bound[:num_reqs] = seq_lens
|
||||
|
||||
return replace(
|
||||
base,
|
||||
req_ids=[f"req_{i}" for i in range(num_reqs)],
|
||||
num_reqs=num_reqs,
|
||||
num_reqs_after_padding=num_reqs_padded,
|
||||
idx_mapping=torch.arange(num_reqs, dtype=torch.int32),
|
||||
idx_mapping_np=np.arange(num_reqs, dtype=np.int32),
|
||||
num_scheduled_tokens=np.array(query_lens, dtype=np.int32),
|
||||
num_tokens=num_tokens,
|
||||
num_tokens_after_padding=num_tokens_padded,
|
||||
query_start_loc=buffers.query_start_loc[: num_reqs_padded + 1],
|
||||
query_start_loc_np=query_start_loc_np,
|
||||
seq_lens=buffers.seq_lens[:num_reqs_padded],
|
||||
seq_lens_cpu_upper_bound=torch.from_numpy(seq_lens_upper_bound),
|
||||
num_computed_tokens_np=np.array(seq_lens, dtype=np.int32)
|
||||
- np.array(query_lens, dtype=np.int32),
|
||||
prefill_len_np=np.zeros(num_reqs, dtype=np.int32),
|
||||
num_computed_prefill_tokens_np=np.zeros(num_reqs, dtype=np.int32),
|
||||
is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_),
|
||||
input_ids=buffers.input_ids[:num_tokens_padded],
|
||||
positions=buffers.positions[:num_tokens_padded],
|
||||
is_padding=buffers.is_padding[:num_tokens_padded],
|
||||
)
|
||||
|
||||
|
||||
# (query_lens, seq_lens): uniform decode, a boundary that lands inside a
|
||||
# request, and a boundary that lands exactly on a request start.
|
||||
BATCHES = {
|
||||
"uniform_decode": ([1] * 8, [128 + 4 * i for i in range(8)]),
|
||||
"split_request": ([1, 1, 10, 1, 1], [64, 96, 512, 32, 48]),
|
||||
"aligned_prefills": ([6, 2, 4, 4], [6, 130, 260, 4]),
|
||||
"split_first_and_last": ([3, 9, 3], [70, 300, 41]),
|
||||
# A long prefill trailing a decode: it straddles the boundary and is also
|
||||
# the last request, so the second microbatch holds nothing else.
|
||||
"trailing_prefill_spans_boundary": ([2, 20], [64, 300]),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("batch_name", list(BATCHES))
|
||||
def test_slicing_matches_v1_split_attn_metadata(batch_name: str):
|
||||
"""V2's pre-sliced inputs describe the same microbatches as V1's."""
|
||||
query_lens, seq_lens = BATCHES[batch_name]
|
||||
num_scheduled_tokens = np.array(query_lens, dtype=np.int32)
|
||||
num_tokens = int(num_scheduled_tokens.sum())
|
||||
|
||||
ubatch_slices, _ = maybe_create_ubatch_slices(
|
||||
True, num_scheduled_tokens, num_tokens, len(query_lens), num_ubatches=2
|
||||
)
|
||||
assert ubatch_slices is not None
|
||||
|
||||
v1_metadata = create_common_attn_metadata(
|
||||
BatchSpec(seq_lens=seq_lens, query_lens=query_lens),
|
||||
block_size=16,
|
||||
device=torch.device("cpu"),
|
||||
)
|
||||
v1_ubatches = split_attn_metadata(ubatch_slices, v1_metadata)
|
||||
|
||||
buffers = _make_buffers()
|
||||
ubatch_buffers = _make_ubatch_buffers()
|
||||
input_batch = _make_input_batch(query_lens, seq_lens, buffers)
|
||||
|
||||
for i, (ubatch_slice, v1_ubatch) in enumerate(zip(ubatch_slices, v1_ubatches)):
|
||||
v2_ubatch = _slice_input_batch(input_batch, ubatch_slice, *ubatch_buffers[i])
|
||||
|
||||
assert v2_ubatch.num_reqs == v1_ubatch.num_reqs
|
||||
assert v2_ubatch.num_tokens == v1_ubatch.num_actual_tokens
|
||||
assert int(v2_ubatch.num_scheduled_tokens.max()) == v1_ubatch.max_query_len
|
||||
torch.testing.assert_close(v2_ubatch.query_start_loc, v1_ubatch.query_start_loc)
|
||||
torch.testing.assert_close(v2_ubatch.seq_lens, v1_ubatch.seq_lens)
|
||||
torch.testing.assert_close(
|
||||
torch.from_numpy(v2_ubatch.query_start_loc_np),
|
||||
v1_ubatch.query_start_loc_cpu,
|
||||
)
|
||||
# `prepare_attn` derives max_seq_len from the upper bound. Compare the
|
||||
# bound itself rather than V1's max_seq_len: V1 floors it with the full
|
||||
# batch's max_seq_len to keep the value CUDA-graph capture installed,
|
||||
# which V2 gets from `prepare_attn(for_capture=True)` instead.
|
||||
torch.testing.assert_close(
|
||||
v2_ubatch.seq_lens_cpu_upper_bound, v1_ubatch.seq_lens_cpu_upper_bound
|
||||
)
|
||||
|
||||
|
||||
def test_microbatches_do_not_share_buffers():
|
||||
"""Both microbatches are live at once, so their buffers must be distinct."""
|
||||
query_lens, seq_lens = BATCHES["split_request"]
|
||||
num_scheduled_tokens = np.array(query_lens, dtype=np.int32)
|
||||
num_tokens = int(num_scheduled_tokens.sum())
|
||||
|
||||
ubatch_slices, _ = maybe_create_ubatch_slices(
|
||||
True, num_scheduled_tokens, num_tokens, len(query_lens), num_ubatches=2
|
||||
)
|
||||
assert ubatch_slices is not None
|
||||
|
||||
buffers = _make_buffers()
|
||||
ubatch_buffers = _make_ubatch_buffers()
|
||||
input_batch = _make_input_batch(query_lens, seq_lens, buffers)
|
||||
|
||||
first = _slice_input_batch(input_batch, ubatch_slices[0], *ubatch_buffers[0])
|
||||
first_query_start_loc = first.query_start_loc.clone()
|
||||
first_seq_lens = first.seq_lens.clone()
|
||||
|
||||
# Slicing the second microbatch must not disturb the first.
|
||||
_slice_input_batch(input_batch, ubatch_slices[1], *ubatch_buffers[1])
|
||||
|
||||
torch.testing.assert_close(first.query_start_loc, first_query_start_loc)
|
||||
torch.testing.assert_close(first.seq_lens, first_seq_lens)
|
||||
|
||||
|
||||
def test_trailing_microbatch_absorbs_cudagraph_padding():
|
||||
"""The padded microbatch keeps padded rows empty and query_start_loc flat."""
|
||||
query_lens, seq_lens = [1] * 6, [64] * 6
|
||||
num_tokens_padded = 8
|
||||
|
||||
buffers = _make_buffers()
|
||||
input_batch = _make_input_batch(
|
||||
query_lens,
|
||||
seq_lens,
|
||||
buffers,
|
||||
num_reqs_padded=num_tokens_padded,
|
||||
num_tokens_padded=num_tokens_padded,
|
||||
)
|
||||
ubatch_slices_padded = create_ubatch_slices(input_batch, num_ubatches=2)
|
||||
|
||||
last = _slice_input_batch(
|
||||
input_batch, ubatch_slices_padded[-1], *_make_ubatch_buffers()[1]
|
||||
)
|
||||
|
||||
# Two real decodes plus two padded rows.
|
||||
assert last.num_reqs == 2
|
||||
assert last.num_tokens == 2
|
||||
assert last.num_reqs_after_padding == 4
|
||||
assert last.num_tokens_after_padding == 4
|
||||
torch.testing.assert_close(
|
||||
last.query_start_loc, torch.tensor([0, 1, 2, 2, 2], dtype=torch.int32)
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
last.seq_lens, torch.tensor([64, 64, 0, 0], dtype=torch.int32)
|
||||
)
|
||||
|
||||
|
||||
DECODE_THRESHOLD = 32
|
||||
PREFILL_THRESHOLD = 128
|
||||
DECODE_QUERY_LEN = 1
|
||||
|
||||
|
||||
def test_dummy_batch_can_be_microbatched():
|
||||
"""A DP rank with no work still has to split, so its dummy batch must slice.
|
||||
|
||||
Microbatching is all-or-nothing across the group, so a rank whose step is a
|
||||
dummy batch runs it microbatched like everyone else.
|
||||
"""
|
||||
num_reqs, num_tokens = 1, 32
|
||||
buffers = _make_buffers()
|
||||
input_batch = InputBatch.make_dummy(num_reqs, num_tokens, buffers)
|
||||
|
||||
ubatch_slices_padded = create_ubatch_slices(input_batch, num_ubatches=2)
|
||||
|
||||
ubatch_buffers = _make_ubatch_buffers()
|
||||
ubatches = [
|
||||
_slice_input_batch(input_batch, ubatch_slice, *ubatch_buffers[i])
|
||||
for i, ubatch_slice in enumerate(ubatch_slices_padded)
|
||||
]
|
||||
|
||||
assert [u.num_tokens for u in ubatches] == [16, 16]
|
||||
# The one dummy request spans both microbatches, 16 of its tokens in each.
|
||||
assert all(u.num_reqs == 1 for u in ubatches)
|
||||
for ubatch in ubatches:
|
||||
torch.testing.assert_close(
|
||||
ubatch.query_start_loc, torch.tensor([0, 16], dtype=torch.int32)
|
||||
)
|
||||
|
||||
|
||||
def _sync_dp(
|
||||
num_tokens_per_rank: list[int],
|
||||
uniform_token_count_per_rank: list[int] | None = None,
|
||||
allow_ubatching: bool = True,
|
||||
) -> tuple[BatchExecutionDescriptor, dp_utils.DPSyncState | None]:
|
||||
"""Run the DP handshake with the all-reduce stubbed out.
|
||||
|
||||
The microbatching decision lives inside `sync_cudagraph_and_dp_padding`, so
|
||||
the cases below stand in for the collective rather than for the decision,
|
||||
and need no process group. Every rank asks for eager, which keeps the
|
||||
non-microbatched path off the cudagraph manager. This rank is rank 0.
|
||||
"""
|
||||
dp_size = len(num_tokens_per_rank)
|
||||
uniform_token_counts = uniform_token_count_per_rank or [0] * dp_size
|
||||
reduced = torch.zeros(6, dp_size, dtype=torch.int32)
|
||||
reduced[0] = torch.tensor(num_tokens_per_rank, dtype=torch.int32)
|
||||
reduced[1] = CUDAGraphMode.NONE.value
|
||||
reduced[2] = torch.tensor(uniform_token_counts, dtype=torch.int32)
|
||||
reduced[3] = -1 # max_query_len, -1 means None
|
||||
reduced[4] = int(allow_ubatching)
|
||||
reduced[5] = 8 # num_reqs
|
||||
|
||||
with (
|
||||
patch.object(dp_utils.dist, "all_reduce", lambda t, group: t.copy_(reduced)),
|
||||
patch.object(dp_utils, "get_dp_group", lambda: SimpleNamespace(cpu_group=None)),
|
||||
):
|
||||
return dp_utils.sync_cudagraph_and_dp_padding(
|
||||
cudagraph_manager=None,
|
||||
desired_batch_desc=BatchExecutionDescriptor(
|
||||
cg_mode=CUDAGraphMode.NONE,
|
||||
num_tokens=num_tokens_per_rank[0],
|
||||
num_reqs=8,
|
||||
),
|
||||
num_tokens=num_tokens_per_rank[0],
|
||||
num_reqs=8,
|
||||
uniform_token_count=uniform_token_counts[0] or None,
|
||||
dp_size=dp_size,
|
||||
dp_rank=0,
|
||||
parallel_config=ParallelConfig(
|
||||
enable_dbo=True,
|
||||
dbo_decode_token_threshold=DECODE_THRESHOLD,
|
||||
dbo_prefill_token_threshold=PREFILL_THRESHOLD,
|
||||
),
|
||||
allow_ubatching=allow_ubatching,
|
||||
uniform_decode=uniform_token_counts[0] == DECODE_QUERY_LEN,
|
||||
)
|
||||
|
||||
|
||||
def test_every_dp_rank_must_agree_to_microbatch():
|
||||
"""One rank below the threshold is enough to keep the group on one batch.
|
||||
|
||||
The thresholds are not communicated -- they are checked against the token
|
||||
counts the all-reduce already carries -- so this also pins that the whole
|
||||
vector is consulted, not just this rank's own entry.
|
||||
"""
|
||||
assert _sync_dp([256, 256])[0].num_ubatches == 2
|
||||
assert _sync_dp([256, 100])[0].num_ubatches == 1
|
||||
assert _sync_dp([100, 256])[0].num_ubatches == 1
|
||||
|
||||
|
||||
def test_decode_threshold_only_applies_when_every_rank_is_a_uniform_decode():
|
||||
"""Otherwise the group is held to the prefill threshold."""
|
||||
# 64 tokens clears the decode threshold but not the prefill one, so it only
|
||||
# microbatches when both ranks are uniform decodes.
|
||||
assert _sync_dp([64, 64], uniform_token_count_per_rank=[1, 1])[0].num_ubatches == 2
|
||||
assert _sync_dp([64, 64], uniform_token_count_per_rank=[1, 0])[0].num_ubatches == 1
|
||||
|
||||
|
||||
def test_microbatching_pads_all_ranks_to_the_largest():
|
||||
"""Ranks must run the same token count so microbatch sizes line up."""
|
||||
desc, dp_sync = _sync_dp([200, 256])
|
||||
assert desc.num_tokens == 256
|
||||
assert dp_sync is not None
|
||||
assert dp_sync.num_tokens_across_dp.tolist() == [256, 256]
|
||||
|
||||
|
||||
def test_microbatching_survives_a_rank_that_cannot_fill_it():
|
||||
"""A rank whose real tokens all land in the first microbatch still splits.
|
||||
|
||||
Rank 0 has 128 real tokens but the split point is at 512 // 2 = 256, so its
|
||||
second microbatch is pure padding. That microbatch does no work, which is
|
||||
fine -- it still has to run so the expert all-to-all stays collective.
|
||||
"""
|
||||
desc, _ = _sync_dp([128, 512])
|
||||
assert desc.num_tokens == 512
|
||||
assert desc.num_ubatches == 2
|
||||
|
||||
|
||||
def test_all_padding_microbatch_has_no_work_to_do():
|
||||
"""The microbatch past the last real token is well-formed and empty.
|
||||
|
||||
It carries the last request with none of its query tokens, so attention and
|
||||
sampling see nothing to do while the microbatch still runs.
|
||||
"""
|
||||
query_lens, seq_lens = [1] * 6, [64] * 6
|
||||
num_tokens_padded = 16 # split at 8, past all 6 real tokens
|
||||
|
||||
buffers = _make_buffers()
|
||||
input_batch = _make_input_batch(
|
||||
query_lens, seq_lens, buffers, num_tokens_padded=num_tokens_padded
|
||||
)
|
||||
ubatch_slices_padded = create_ubatch_slices(input_batch, num_ubatches=2)
|
||||
ubatch_buffers = _make_ubatch_buffers()
|
||||
|
||||
first = _slice_input_batch(input_batch, ubatch_slices_padded[0], *ubatch_buffers[0])
|
||||
last = _slice_input_batch(input_batch, ubatch_slices_padded[1], *ubatch_buffers[1])
|
||||
|
||||
# The first microbatch keeps every real request; the second gets none.
|
||||
assert first.num_reqs == 6
|
||||
assert first.num_tokens == 6
|
||||
assert last.num_tokens == 0
|
||||
assert last.num_reqs == 1
|
||||
assert last.num_reqs_after_padding == 1
|
||||
assert last.num_tokens_after_padding == 8
|
||||
|
||||
# Zero query tokens for the one request it holds, so no attention work.
|
||||
torch.testing.assert_close(
|
||||
last.query_start_loc, torch.tensor([0, 0], dtype=torch.int32)
|
||||
)
|
||||
assert last.num_scheduled_tokens.tolist() == [0]
|
||||
# ...and its sequence still ends where its tokens do: they all live in the
|
||||
# first microbatch, which computes them before this one runs.
|
||||
torch.testing.assert_close(last.seq_lens, torch.tensor([64], dtype=torch.int32))
|
||||
torch.testing.assert_close(
|
||||
last.seq_lens_cpu_upper_bound, torch.tensor([64], dtype=torch.int32)
|
||||
)
|
||||
|
||||
|
||||
def test_microbatching_off_when_a_rank_does_not_allow_it():
|
||||
assert _sync_dp([256, 256], allow_ubatching=False)[0].num_ubatches == 1
|
||||
|
||||
|
||||
def test_slice_model_inputs_handles_mrope_positions():
|
||||
model_inputs = {
|
||||
"input_ids": torch.arange(8),
|
||||
# M-RoPE positions carry a leading section dim.
|
||||
"positions": torch.arange(24).view(3, 8),
|
||||
"inputs_embeds": None,
|
||||
"intermediate_tensors": None,
|
||||
}
|
||||
sliced = slice_model_inputs(model_inputs, slice(4, 8))
|
||||
|
||||
torch.testing.assert_close(sliced["input_ids"], torch.arange(4, 8))
|
||||
torch.testing.assert_close(sliced["positions"], torch.arange(24).view(3, 8)[:, 4:8])
|
||||
assert sliced["inputs_embeds"] is None
|
||||
assert sliced["intermediate_tensors"] is None
|
||||
|
||||
|
||||
def _make_dbo_config() -> VllmConfig:
|
||||
return VllmConfig(
|
||||
model_config=ModelConfig(model="facebook/opt-125m", dtype="float16", seed=0),
|
||||
parallel_config=ParallelConfig(
|
||||
enable_dbo=True, all2all_backend="deepep_low_latency"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _make_model_inputs(num_tokens: int, device: torch.device) -> dict[str, Any]:
|
||||
return {
|
||||
"input_ids": torch.arange(num_tokens, device=device),
|
||||
"positions": torch.arange(num_tokens, device=device),
|
||||
"inputs_embeds": None,
|
||||
"intermediate_tensors": None,
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _no_comm_sm_reservation(monkeypatch):
|
||||
"""CI runs on MIG slices with fewer SMs than the default comm reservation."""
|
||||
monkeypatch.setenv("VLLM_DBO_COMM_SMS", "0")
|
||||
|
||||
|
||||
def _make_execution_runner(vllm_config: VllmConfig) -> UBatchRunner:
|
||||
"""A runner for the execution tests below, which never call `prepare()`.
|
||||
|
||||
The attention-side dependencies are only read while building per-microbatch
|
||||
metadata, so they are left out here.
|
||||
"""
|
||||
return UBatchRunner(
|
||||
vllm_config,
|
||||
torch.device("cuda:0"),
|
||||
model_state=cast(ModelState, None),
|
||||
attn_groups=[],
|
||||
kv_cache_config=cast(KVCacheConfig, None),
|
||||
max_num_reqs=MAX_NUM_REQS,
|
||||
)
|
||||
|
||||
|
||||
def _make_ubatch_state(
|
||||
vllm_config: VllmConfig, ubatch_slices: list[UBatchSlice]
|
||||
) -> UBatchState:
|
||||
return UBatchState(
|
||||
slices=ubatch_slices,
|
||||
forward_contexts=[
|
||||
create_forward_context(None, vllm_config) for _ in ubatch_slices
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_thresholds_below_the_microbatch_count_are_rejected():
|
||||
"""A batch with fewer tokens than microbatches cannot be split at all.
|
||||
|
||||
Nothing downstream copes with an unsplittable batch, so the thresholds are
|
||||
what keep one out: a one-token decode split two ways leaves an empty
|
||||
microbatch.
|
||||
"""
|
||||
with pytest.raises(ValueError, match="number of microbatches"):
|
||||
ParallelConfig(enable_dbo=True, dbo_decode_token_threshold=1)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="DBO needs a GPU")
|
||||
def test_runner_allocates_one_buffer_pair_per_microbatch():
|
||||
"""All microbatches are live at once, so none may share rebase buffers."""
|
||||
vllm_config = _make_dbo_config()
|
||||
runner = _make_execution_runner(vllm_config)
|
||||
|
||||
for buffers in (runner.ubatch_query_start_loc, runner.ubatch_seq_lens):
|
||||
assert len(buffers) == runner.num_ubatches
|
||||
storages = {b.untyped_storage().data_ptr() for b in buffers}
|
||||
assert len(storages) == runner.num_ubatches
|
||||
|
||||
|
||||
class _YieldingModel(torch.nn.Module):
|
||||
"""Toy model that hands off to the other microbatch mid-forward."""
|
||||
|
||||
def __init__(self, trace: list[tuple[str, int]]):
|
||||
super().__init__()
|
||||
self.trace = trace
|
||||
|
||||
def forward(self, input_ids, positions, intermediate_tensors=None, **kwargs):
|
||||
self.trace.append(("enter", dbo_current_ubatch_id()))
|
||||
out = input_ids.float().unsqueeze(-1) * 2 + positions.float().unsqueeze(-1)
|
||||
# Stands in for the expert all-to-all handoff point.
|
||||
dbo_yield()
|
||||
self.trace.append(("exit", dbo_current_ubatch_id()))
|
||||
return out
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="DBO needs a GPU")
|
||||
def test_ubatch_runner_overlaps_and_matches_single_batch():
|
||||
"""Microbatches interleave at the yield point and produce the same output."""
|
||||
vllm_config = _make_dbo_config()
|
||||
device = torch.device("cuda:0")
|
||||
runner = _make_execution_runner(vllm_config)
|
||||
|
||||
model_inputs = _make_model_inputs(16, device)
|
||||
ubatch_state = _make_ubatch_state(
|
||||
vllm_config,
|
||||
[
|
||||
UBatchSlice(slice(0, 4), slice(0, 8)),
|
||||
UBatchSlice(slice(4, 8), slice(8, 16)),
|
||||
],
|
||||
)
|
||||
|
||||
trace: list[tuple[str, int]] = []
|
||||
model = _YieldingModel(trace)
|
||||
output = runner.run(model, model_inputs, ubatch_state)
|
||||
|
||||
expected = model_inputs["input_ids"].float().unsqueeze(-1) * 2 + model_inputs[
|
||||
"positions"
|
||||
].float().unsqueeze(-1)
|
||||
torch.testing.assert_close(output, expected)
|
||||
|
||||
# Both microbatches reach the handoff before either finishes.
|
||||
assert trace == [("enter", 0), ("enter", 1), ("exit", 0), ("exit", 1)]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="DBO needs a GPU")
|
||||
def test_ubatch_runner_names_the_microbatch_that_failed():
|
||||
"""A failing microbatch surfaces as a named error, not a downstream KeyError.
|
||||
|
||||
The model here never yields, so the microbatches run back to back and no
|
||||
handoff is left outstanding when one of them dies. A microbatch that fails
|
||||
*while its sibling is parked at a yield* hangs the step instead -- the
|
||||
shared handoff protocol in `ubatching.py` has no way to unwind a parked
|
||||
microbatch, and the V1 runner has the same gap. Fixing that means changing
|
||||
`ubatching.py`, which is out of scope for the V2 runner.
|
||||
"""
|
||||
vllm_config = _make_dbo_config()
|
||||
device = torch.device("cuda:0")
|
||||
runner = _make_execution_runner(vllm_config)
|
||||
|
||||
class _FailingModel(torch.nn.Module):
|
||||
def forward(self, input_ids, positions, **kwargs):
|
||||
if dbo_current_ubatch_id() == 0:
|
||||
raise ValueError("boom")
|
||||
return input_ids.float().unsqueeze(-1)
|
||||
|
||||
model_inputs = _make_model_inputs(8, device)
|
||||
ubatch_state = _make_ubatch_state(
|
||||
vllm_config,
|
||||
[
|
||||
UBatchSlice(slice(0, 2), slice(0, 4)),
|
||||
UBatchSlice(slice(2, 4), slice(4, 8)),
|
||||
],
|
||||
)
|
||||
|
||||
result: dict[str, BaseException] = {}
|
||||
|
||||
def _call():
|
||||
try:
|
||||
runner.run(_FailingModel(), model_inputs, ubatch_state)
|
||||
except BaseException as e: # noqa: BLE001
|
||||
result["error"] = e
|
||||
|
||||
# Run behind a watchdog: `UBatchRunner.run` joins without a timeout, so a
|
||||
# regression here would wedge the suite rather than fail it.
|
||||
caller = threading.Thread(target=_call, daemon=True)
|
||||
caller.start()
|
||||
caller.join(timeout=60.0)
|
||||
assert not caller.is_alive(), "UBatchRunner.run hung on a failing microbatch"
|
||||
|
||||
assert isinstance(result["error"], RuntimeError)
|
||||
assert "Microbatch 0" in str(result["error"])
|
||||
assert isinstance(result["error"].__cause__, ValueError)
|
||||
@@ -1065,6 +1065,17 @@ class ParallelConfig:
|
||||
"Unable to use nsight profiling unless workers run with Ray."
|
||||
)
|
||||
|
||||
# A batch below one token per microbatch cannot be split, so the
|
||||
# thresholds have to keep it out rather than the split having to cope.
|
||||
if self.use_ubatching and (
|
||||
min(self.dbo_decode_token_threshold, self.dbo_prefill_token_threshold)
|
||||
< self.num_ubatches
|
||||
):
|
||||
raise ValueError(
|
||||
"dbo_decode_token_threshold and dbo_prefill_token_threshold must "
|
||||
f"be at least the number of microbatches ({self.num_ubatches})."
|
||||
)
|
||||
|
||||
return self
|
||||
|
||||
def reconfigure_for_independent_dp_rank(self) -> None:
|
||||
|
||||
+43
-2
@@ -2596,8 +2596,8 @@ class VllmConfig:
|
||||
):
|
||||
unsupported.append("EAGLE3 with pipeline parallelism")
|
||||
|
||||
if self.parallel_config.enable_dbo:
|
||||
unsupported.append("dual batch overlap")
|
||||
if self.parallel_config.use_ubatching:
|
||||
unsupported.extend(self._get_dbo_unsupported_features())
|
||||
|
||||
if self.parallel_config.enable_elastic_ep:
|
||||
unsupported.append("elastic expert parallelism")
|
||||
@@ -2739,6 +2739,47 @@ class VllmConfig:
|
||||
f"{'; '.join(blockers)}."
|
||||
)
|
||||
|
||||
def _get_dbo_unsupported_features(self) -> list[str]:
|
||||
"""Collect what the V2 model runner cannot combine with DBO.
|
||||
|
||||
The V2 runner microbatches a plain decoder forward pass. Anything that
|
||||
slices or replays the batch differently (drafting, adapters, pipeline
|
||||
stages, context parallelism, encoders) is not handled yet.
|
||||
"""
|
||||
# TODO: DBO with model runner V2 is under development.
|
||||
# It should be enabled with explicit VLLM_USE_V2_MODEL_RUNNER environ.
|
||||
# Remove it when stable.
|
||||
if envs.VLLM_USE_V2_MODEL_RUNNER is None:
|
||||
return ["dual batch overlap"]
|
||||
|
||||
unsupported: list[str] = []
|
||||
model_config = self.model_config
|
||||
parallel_config = self.parallel_config
|
||||
|
||||
if self.lora_config is not None:
|
||||
unsupported.append("dual batch overlap with LoRA")
|
||||
if self.speculative_config is not None:
|
||||
unsupported.append("dual batch overlap with speculative decoding")
|
||||
if parallel_config.pipeline_parallel_size > 1:
|
||||
unsupported.append("dual batch overlap with pipeline parallelism")
|
||||
if (
|
||||
parallel_config.decode_context_parallel_size > 1
|
||||
or parallel_config.prefill_context_parallel_size > 1
|
||||
):
|
||||
unsupported.append("dual batch overlap with context parallelism")
|
||||
if model_config is not None and (
|
||||
model_config.is_multimodal_model or model_config.is_encoder_decoder
|
||||
):
|
||||
unsupported.append("dual batch overlap with multimodal models")
|
||||
if model_config is not None and model_config.is_hybrid:
|
||||
unsupported.append("dual batch overlap with hybrid models")
|
||||
if self.compilation_config.cudagraph_mode != CUDAGraphMode.NONE:
|
||||
unsupported.append("dual batch overlap with CUDA graphs")
|
||||
if self.is_mm_encoder_only:
|
||||
unsupported.append("dual batch overlap with encoder only models")
|
||||
|
||||
return unsupported
|
||||
|
||||
def _validate_v2_model_runner(self) -> None:
|
||||
"""Check for features not yet supported by the V2 model runner."""
|
||||
if not HAS_TRITON:
|
||||
|
||||
@@ -988,7 +988,9 @@ class DiffusionGemmaModelState(ModelState):
|
||||
attn_groups,
|
||||
kv_cache_config,
|
||||
for_capture=False,
|
||||
ubatch_idx: int = 0,
|
||||
) -> dict[str, Any]:
|
||||
assert ubatch_idx == 0, "DBO is not supported"
|
||||
if cudagraph_mode == CUDAGraphMode.FULL:
|
||||
num_reqs = input_batch.num_reqs_after_padding
|
||||
num_tokens = input_batch.num_tokens_after_padding
|
||||
|
||||
@@ -22,6 +22,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelSpecificAttnMetadata
|
||||
from vllm.v1.worker.ubatch_utils import get_num_ubatches
|
||||
from vllm.v1.worker.utils import (
|
||||
AttentionGroup,
|
||||
add_kv_sharing_layers_to_kv_cache_groups,
|
||||
@@ -140,14 +141,19 @@ def init_attn_backend(
|
||||
vllm_config=vllm_config,
|
||||
device=device,
|
||||
kernel_block_size=kernel_block_size,
|
||||
num_metadata_builders=1,
|
||||
# Microbatches build attention metadata concurrently, and some
|
||||
# builders keep the prepared metadata on themselves (MLA stores
|
||||
# it on the prefill backend), so each ubatch needs its own.
|
||||
num_metadata_builders=get_num_ubatches(vllm_config.parallel_config),
|
||||
)
|
||||
builder = group.get_metadata_builder(0)
|
||||
if attn_backend_workspace is None:
|
||||
if hasattr(builder, "_get_workspace_buffer"):
|
||||
attn_backend_workspace = builder._get_workspace_buffer()
|
||||
else:
|
||||
if hasattr(builder, "set_workspace_buffer"):
|
||||
# The microbatches' builders share the workspace: they all issue
|
||||
# attention on the one compute stream the threads hand off, so the
|
||||
# buffer is written serially, as it already is across steps.
|
||||
for builder in group.metadata_builders:
|
||||
if attn_backend_workspace is None:
|
||||
if hasattr(builder, "_get_workspace_buffer"):
|
||||
attn_backend_workspace = builder._get_workspace_buffer()
|
||||
elif hasattr(builder, "set_workspace_buffer"):
|
||||
builder.set_workspace_buffer(attn_backend_workspace)
|
||||
attn_cg_support_info = get_attn_cg_support(attn_groups, vllm_config)
|
||||
return attn_groups, attn_cg_support_info, kernel_block_sizes
|
||||
@@ -271,6 +277,7 @@ def build_attn_metadata(
|
||||
for_cudagraph_capture: bool = False,
|
||||
causal: bool | torch.Tensor | Mapping[int, bool] = True,
|
||||
rswa_prefix_lens: torch.Tensor | None = None,
|
||||
ubatch_idx: int = 0,
|
||||
) -> dict[str, Any]:
|
||||
seq_lens = seq_lens[:num_reqs]
|
||||
if dcp_local_seq_lens is not None:
|
||||
@@ -319,7 +326,7 @@ def build_attn_metadata(
|
||||
)
|
||||
|
||||
for attn_group in attn_groups[i]:
|
||||
attn_metadata_builder = attn_group.get_metadata_builder(0)
|
||||
attn_metadata_builder = attn_group.get_metadata_builder(ubatch_idx)
|
||||
if for_cudagraph_capture:
|
||||
metadata = attn_metadata_builder.build_for_cudagraph_capture(
|
||||
common_attn_metadata
|
||||
|
||||
@@ -68,6 +68,8 @@ class BatchExecutionDescriptor:
|
||||
# uniform_token_count unset, so this is what keeps a prefill batch out of one.
|
||||
max_query_len: int | None = None
|
||||
num_active_loras: int = 0
|
||||
# Number of microbatches the batch is split into (DBO). 1 means no splitting.
|
||||
num_ubatches: int = 1
|
||||
|
||||
|
||||
def make_cudagraph_stats(
|
||||
@@ -100,11 +102,13 @@ def _is_compatible(
|
||||
uniform_token_count: int | None,
|
||||
num_active_loras: int,
|
||||
max_query_len: int | None,
|
||||
num_ubatches: int,
|
||||
) -> bool:
|
||||
# desc.uniform_token_count=None (PIECEWISE) can handle any uniform_token_count
|
||||
# desc.num_reqs=None means no request padding needed (PIECEWISE)
|
||||
# desc.max_query_len=None means the graph does not constrain query length; a
|
||||
# caller that does not track max_query_len must not match one that does
|
||||
# A graph captured for N microbatches can only serve a batch split N ways.
|
||||
return (
|
||||
(
|
||||
desc.uniform_token_count is None
|
||||
@@ -117,6 +121,7 @@ def _is_compatible(
|
||||
and (desc.num_reqs is None or desc.num_reqs >= num_reqs)
|
||||
and desc.num_tokens >= num_tokens
|
||||
and desc.num_active_loras == num_active_loras
|
||||
and desc.num_ubatches == num_ubatches
|
||||
)
|
||||
|
||||
|
||||
@@ -434,6 +439,7 @@ class CudaGraphManager:
|
||||
uniform_token_count: int | None,
|
||||
num_active_loras: int,
|
||||
max_query_len: int | None = None,
|
||||
num_ubatches: int = 1,
|
||||
) -> BatchExecutionDescriptor:
|
||||
"""Find matching cudagraph descriptor from priority-ordered candidates."""
|
||||
|
||||
@@ -448,6 +454,7 @@ class CudaGraphManager:
|
||||
uniform_token_count,
|
||||
effective_loras,
|
||||
max_query_len,
|
||||
num_ubatches,
|
||||
):
|
||||
return desc
|
||||
return BatchExecutionDescriptor(
|
||||
@@ -455,6 +462,7 @@ class CudaGraphManager:
|
||||
num_tokens=num_tokens,
|
||||
num_reqs=num_reqs,
|
||||
num_active_loras=effective_loras,
|
||||
num_ubatches=num_ubatches,
|
||||
)
|
||||
|
||||
def run_fullgraph(self, desc: BatchExecutionDescriptor):
|
||||
|
||||
@@ -7,12 +7,14 @@ from dataclasses import dataclass, replace
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.distributed.parallel_state import get_dp_group
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
BatchExecutionDescriptor,
|
||||
CudaGraphManager,
|
||||
)
|
||||
from vllm.v1.worker.ubatch_utils import check_ubatch_thresholds, get_num_ubatches
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -48,27 +50,35 @@ def sync_cudagraph_and_dp_padding(
|
||||
dp_rank: int,
|
||||
max_query_len: int | None = None,
|
||||
num_active_loras: int = 0,
|
||||
parallel_config: ParallelConfig | None = None,
|
||||
allow_ubatching: bool = False,
|
||||
uniform_decode: bool = False,
|
||||
) -> tuple[BatchExecutionDescriptor, DPSyncState | None]:
|
||||
"""
|
||||
Coordinates the batch descriptor and DP padding across all ranks.
|
||||
|
||||
`parallel_config` is only needed to decide whether to microbatch, so callers
|
||||
that never do (`allow_ubatching=False`) can leave it out.
|
||||
|
||||
Returns (synced_batch_desc, sync). `sync` is None when no rank has work.
|
||||
"""
|
||||
assert dp_size > 1, "DP size must be greater than 1"
|
||||
group = get_dp_group().cpu_group
|
||||
tensor = torch.zeros(5, dp_size, dtype=torch.int32, device="cpu")
|
||||
tensor = torch.zeros(6, dp_size, dtype=torch.int32, device="cpu")
|
||||
tensor[0][dp_rank] = num_tokens
|
||||
tensor[1][dp_rank] = desired_batch_desc.cg_mode.value
|
||||
tensor[2][dp_rank] = uniform_token_count or 0 # (0 means None)
|
||||
tensor[3][dp_rank] = max_query_len or -1 # (-1 means None)
|
||||
tensor[4][dp_rank] = num_reqs
|
||||
tensor[4][dp_rank] = int(allow_ubatching)
|
||||
tensor[5][dp_rank] = num_reqs
|
||||
dist.all_reduce(tensor, group=group)
|
||||
|
||||
num_tokens_across_dp = tensor[0]
|
||||
cg_mode_across_dp = tensor[1]
|
||||
uniform_token_counts_across_dp = tensor[2]
|
||||
max_query_lens_across_dp = tensor[3]
|
||||
num_reqs_across_dp = tensor[4]
|
||||
allow_ubatching_across_dp = tensor[4]
|
||||
num_reqs_across_dp = tensor[5]
|
||||
|
||||
# If ranks disagree on the uniform token count, or its 0 (means None) set to None
|
||||
synced_uniform_token_count: int | None = int(uniform_token_counts_across_dp[0])
|
||||
@@ -83,6 +93,40 @@ def sync_cudagraph_and_dp_padding(
|
||||
)
|
||||
return synced_desc, None
|
||||
|
||||
if torch.all(allow_ubatching_across_dp == 1):
|
||||
# This rank voted too, so its caller passed a config.
|
||||
assert parallel_config is not None
|
||||
# A uniform decode only if every rank runs the same uniform query
|
||||
# length, so every rank reaches the same answer here.
|
||||
uniform_decode_across_dp = uniform_decode and torch.all(
|
||||
uniform_token_counts_across_dp == (uniform_token_count or 0)
|
||||
)
|
||||
if check_ubatch_thresholds(
|
||||
parallel_config,
|
||||
# Thresholds only grow with the token count, so holding the smallest
|
||||
# rank to them holds every rank to them.
|
||||
int(num_tokens_across_dp.min()),
|
||||
uniform_decode=uniform_decode_across_dp,
|
||||
):
|
||||
# Microbatching is all-or-nothing: the expert all-to-all is
|
||||
# collective, so every rank splits and pads to the same token count.
|
||||
# A rank too small to fill a microbatch does no work in it, like a
|
||||
# dummy run. Microbatched steps run eager; nothing is captured yet.
|
||||
ubatch_num_tokens = int(num_tokens_across_dp.max())
|
||||
return BatchExecutionDescriptor(
|
||||
cg_mode=CUDAGraphMode.NONE,
|
||||
num_tokens=ubatch_num_tokens,
|
||||
num_reqs=num_reqs,
|
||||
num_ubatches=get_num_ubatches(parallel_config),
|
||||
), DPSyncState(
|
||||
num_tokens_across_dp=torch.full_like(
|
||||
num_tokens_across_dp, ubatch_num_tokens
|
||||
),
|
||||
uniform_token_count=synced_uniform_token_count,
|
||||
eager=True,
|
||||
num_reqs=int(num_reqs_across_dp.max()),
|
||||
)
|
||||
|
||||
synced_cg_mode = CUDAGraphMode(int(cg_mode_across_dp.min().item()))
|
||||
|
||||
# If any rank wants to run eager, all ranks run eager
|
||||
@@ -98,7 +142,7 @@ def sync_cudagraph_and_dp_padding(
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
uniform_token_count=synced_uniform_token_count,
|
||||
eager=True,
|
||||
num_reqs=int(num_reqs_across_dp.max().item()),
|
||||
num_reqs=int(num_reqs_across_dp.max()),
|
||||
),
|
||||
)
|
||||
|
||||
@@ -136,7 +180,7 @@ def sync_cudagraph_and_dp_padding(
|
||||
synced_desc.num_reqs
|
||||
if synced_desc.cg_mode == CUDAGraphMode.FULL
|
||||
and synced_desc.num_reqs is not None
|
||||
else int(num_reqs_across_dp.max().item())
|
||||
else int(num_reqs_across_dp.max())
|
||||
),
|
||||
)
|
||||
|
||||
@@ -151,6 +195,9 @@ def dispatch_cg_and_sync_dp(
|
||||
max_query_len: int | None = None,
|
||||
need_eager: bool = False,
|
||||
num_active_loras: int = 0,
|
||||
parallel_config: ParallelConfig | None = None,
|
||||
allow_ubatching: bool = False,
|
||||
uniform_decode: bool = False,
|
||||
dp_sync: DPSyncState | None = None,
|
||||
) -> tuple[BatchExecutionDescriptor, DPSyncState | None]:
|
||||
"""Pick a cudagraph descriptor for this batch, agreeing it across DP ranks.
|
||||
@@ -239,4 +286,7 @@ def dispatch_cg_and_sync_dp(
|
||||
dp_rank,
|
||||
max_query_len=max_query_len,
|
||||
num_active_loras=num_active_loras,
|
||||
parallel_config=parallel_config,
|
||||
allow_ubatching=allow_ubatching,
|
||||
uniform_decode=uniform_decode,
|
||||
)
|
||||
|
||||
@@ -161,6 +161,11 @@ from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
|
||||
from vllm.v1.worker.gpu.spec_decode.utils import DraftTokensHandler
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
from vllm.v1.worker.gpu.structured_outputs import StructuredOutputsWorker
|
||||
from vllm.v1.worker.gpu.ubatch_utils import (
|
||||
UBatchRunner,
|
||||
UBatchState,
|
||||
maybe_build_ubatch_runner,
|
||||
)
|
||||
from vllm.v1.worker.lora_model_runner_mixin import LoRAModelRunnerMixin
|
||||
from vllm.v1.worker.utils import (
|
||||
KVBlockZeroer,
|
||||
@@ -230,6 +235,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.dp_size = self.parallel_config.data_parallel_size
|
||||
self.dp_rank = self.parallel_config.data_parallel_rank
|
||||
|
||||
# Dual batch overlap. Created in initialize_kv_cache(), once everything
|
||||
# it runs the microbatched forward with exists.
|
||||
self.ubatch_runner: UBatchRunner | None = None
|
||||
|
||||
# Detect EP all2all peer faults to prevent emitting corrupted output.
|
||||
# Only meaningful for MoE + DP with an FT-capable all2all backend.
|
||||
self.check_ep_fault = False
|
||||
@@ -634,6 +643,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.block_tables,
|
||||
cls=self.pcp_manager_cls,
|
||||
)
|
||||
self.ubatch_runner = maybe_build_ubatch_runner(
|
||||
self.vllm_config,
|
||||
self.device,
|
||||
self.model_state,
|
||||
self.attn_groups,
|
||||
self.kv_cache_config,
|
||||
self.max_num_reqs,
|
||||
)
|
||||
initialize_mamba_ssu_backend(
|
||||
self.vllm_config.mamba_config,
|
||||
self.kv_cache_config,
|
||||
@@ -1597,6 +1614,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
max_query_len=max_query_len,
|
||||
need_eager=is_profile or skip_compiled,
|
||||
num_active_loras=num_active_loras,
|
||||
parallel_config=self.parallel_config,
|
||||
allow_ubatching=(
|
||||
self.ubatch_runner is not None and not skip_attn_for_dummy_run
|
||||
),
|
||||
uniform_decode=uniform_tok_count == self.decode_query_len,
|
||||
)
|
||||
|
||||
if batch_desc.num_tokens == 0:
|
||||
@@ -1665,7 +1687,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
attn_metadata = None
|
||||
slot_mappings_by_layer = None
|
||||
if not (dummy_run and skip_attn_for_dummy_run):
|
||||
ubatch_state: UBatchState | None = None
|
||||
if batch_desc.num_ubatches > 1:
|
||||
assert self.ubatch_runner is not None
|
||||
assert block_tables is not None and slot_mappings is not None
|
||||
ubatch_state = self.ubatch_runner.prepare(
|
||||
input_batch, block_tables, slot_mappings
|
||||
)
|
||||
elif not (dummy_run and skip_attn_for_dummy_run):
|
||||
assert slot_mappings is not None
|
||||
slot_mappings_by_layer = build_slot_mappings_by_layer(
|
||||
slot_mappings, self.kv_cache_config
|
||||
@@ -1751,7 +1780,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
del intermediate_tensors
|
||||
|
||||
# Update the EPLB meta.
|
||||
self.eplb.prepare_forward(self.model_config, input_batch.num_tokens)
|
||||
ubatch_slices = ubatch_state.slices if ubatch_state is not None else None
|
||||
self.eplb.prepare_forward(
|
||||
self.model_config, input_batch.num_tokens, ubatch_slices
|
||||
)
|
||||
|
||||
self.step_timing.record_batch(
|
||||
input_batch, batch_desc.cg_mode == CUDAGraphMode.FULL
|
||||
@@ -1783,12 +1815,18 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
dp_sync.num_tokens_across_dp if dp_sync is not None else None
|
||||
),
|
||||
batch_descriptor=batch_descriptor,
|
||||
ubatch_slices=ubatch_slices,
|
||||
slot_mapping=slot_mappings_by_layer,
|
||||
skip_compiled=skip_compiled,
|
||||
is_padding=input_batch.is_padding,
|
||||
):
|
||||
self.kv_connector.pre_forward(scheduler_output)
|
||||
if batch_desc.cg_mode == CUDAGraphMode.PIECEWISE:
|
||||
if ubatch_state is not None:
|
||||
assert self.ubatch_runner is not None
|
||||
model_output = self.ubatch_runner.run(
|
||||
self.model, model_inputs, ubatch_state
|
||||
)
|
||||
elif batch_desc.cg_mode == CUDAGraphMode.PIECEWISE:
|
||||
# Run the PIECEWISE graph (compiled PW cudagraph or breakable
|
||||
# cudagraph, chosen inside run_pw_graph). cg_mode is only
|
||||
# PIECEWISE after the cudagraph manager exists.
|
||||
|
||||
@@ -172,6 +172,7 @@ class DefaultModelState(ModelState):
|
||||
attn_groups: list[list[AttentionGroup]],
|
||||
kv_cache_config: KVCacheConfig,
|
||||
for_capture: bool = False,
|
||||
ubatch_idx: int = 0,
|
||||
) -> dict[str, Any]:
|
||||
if cudagraph_mode == CUDAGraphMode.FULL:
|
||||
# Use padded sizes - padding is handled by model_runner.prepare_attn.
|
||||
@@ -224,5 +225,6 @@ class DefaultModelState(ModelState):
|
||||
mm_req_doc_ranges=req_doc_ranges,
|
||||
for_cudagraph_capture=for_capture,
|
||||
rswa_prefix_lens=input_batch.prompt_lens,
|
||||
ubatch_idx=ubatch_idx,
|
||||
)
|
||||
return attn_metadata
|
||||
|
||||
@@ -116,7 +116,9 @@ class EncoderDecoderModelState(ModelState):
|
||||
attn_groups: list[list[AttentionGroup]],
|
||||
kv_cache_config: KVCacheConfig,
|
||||
for_capture: bool = False,
|
||||
ubatch_idx: int = 0,
|
||||
) -> dict[str, Any]:
|
||||
assert ubatch_idx == 0, "DBO is not supported"
|
||||
if cudagraph_mode == CUDAGraphMode.FULL:
|
||||
num_reqs = input_batch.num_reqs_after_padding
|
||||
num_tokens = input_batch.num_tokens_after_padding
|
||||
|
||||
@@ -167,7 +167,9 @@ class EncoderOnlyModelState(DefaultModelState):
|
||||
attn_groups: list[list[AttentionGroup]],
|
||||
kv_cache_config: KVCacheConfig,
|
||||
for_capture: bool = False,
|
||||
ubatch_idx: int = 0,
|
||||
) -> dict[str, Any]:
|
||||
assert ubatch_idx == 0, "DBO is not supported"
|
||||
attn_metadata = super().prepare_attn(
|
||||
input_batch,
|
||||
cudagraph_mode,
|
||||
|
||||
@@ -221,6 +221,7 @@ class ModelState(ABC):
|
||||
attn_groups: list[list[AttentionGroup]],
|
||||
kv_cache_config: KVCacheConfig,
|
||||
for_capture: bool = False,
|
||||
ubatch_idx: int = 0,
|
||||
) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
@@ -238,7 +238,9 @@ class MambaHybridModelState(DefaultModelState):
|
||||
attn_groups: list[list[AttentionGroup]],
|
||||
kv_cache_config: KVCacheConfig,
|
||||
for_capture: bool = False,
|
||||
ubatch_idx: int = 0,
|
||||
) -> dict[str, Any]:
|
||||
assert ubatch_idx == 0, "DBO is not supported"
|
||||
if cudagraph_mode == CUDAGraphMode.FULL:
|
||||
num_reqs = input_batch.num_reqs_after_padding
|
||||
num_tokens = input_batch.num_tokens_after_padding
|
||||
|
||||
@@ -0,0 +1,473 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Microbatching (DBO) helpers for the V2 GPU model runner."""
|
||||
|
||||
import threading
|
||||
from dataclasses import replace
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.config import CUDAGraphMode, VllmConfig
|
||||
from vllm.forward_context import (
|
||||
DPMetadata,
|
||||
ForwardContext,
|
||||
create_forward_context,
|
||||
override_forward_context,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.torch_utils import current_stream
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
from vllm.v1.worker.ubatch_utils import (
|
||||
UBatchSlice,
|
||||
UBatchSlices,
|
||||
create_sm_control_context,
|
||||
maybe_create_ubatch_slices,
|
||||
)
|
||||
from vllm.v1.worker.ubatching import make_ubatch_contexts
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class UBatchState(NamedTuple):
|
||||
"""Per-microbatch inputs for one dual-batch-overlap step."""
|
||||
|
||||
slices: UBatchSlices
|
||||
forward_contexts: list[ForwardContext]
|
||||
|
||||
|
||||
def create_ubatch_slices(input_batch: InputBatch, num_ubatches: int) -> UBatchSlices:
|
||||
"""Split a DP-padded batch into the slices its microbatches run on.
|
||||
|
||||
Splitting at the midpoint of the DP-padded token count leaves the trailing
|
||||
microbatch anywhere from full to entirely inside the padding. Clamping its
|
||||
request slice onto the last request leaves it holding that request with
|
||||
zero query tokens -- the shape a straddling request already takes -- so it
|
||||
stays well-formed with no work to do, like a dummy run.
|
||||
"""
|
||||
_, ubatch_slices = maybe_create_ubatch_slices(
|
||||
True,
|
||||
input_batch.num_scheduled_tokens,
|
||||
input_batch.num_tokens_after_padding,
|
||||
input_batch.num_reqs_after_padding,
|
||||
num_ubatches,
|
||||
)
|
||||
assert ubatch_slices is not None
|
||||
return [
|
||||
UBatchSlice(
|
||||
slice(
|
||||
min(s.request_slice.start, input_batch.num_reqs - 1),
|
||||
min(s.request_slice.stop, input_batch.num_reqs_after_padding),
|
||||
),
|
||||
s.token_slice,
|
||||
)
|
||||
for s in ubatch_slices
|
||||
]
|
||||
|
||||
|
||||
def _slice_input_batch(
|
||||
input_batch: InputBatch,
|
||||
ubatch_slice: UBatchSlice,
|
||||
query_start_loc_buf: torch.Tensor,
|
||||
seq_lens_buf: torch.Tensor,
|
||||
) -> InputBatch:
|
||||
"""Build the sub-`InputBatch` a single microbatch runs on.
|
||||
|
||||
A request straddling the boundary appears in both microbatches, its query
|
||||
truncated to the tokens each one owns.
|
||||
|
||||
Everything except `query_start_loc` and `seq_lens` is a view of the full
|
||||
batch's buffers, so slicing costs no allocation. Those two are rebased onto
|
||||
the microbatch's token range and written to the caller's buffers, which
|
||||
keeps them valid under CUDA graph capture and replay.
|
||||
|
||||
The sub-batch describes the forward pass only. Sampling runs once over the
|
||||
merged batch, so `logits_indices`, `cu_num_logits` and the draft-token
|
||||
fields are carried over unchanged and must not be read off a sub-batch.
|
||||
"""
|
||||
assert not ubatch_slice.is_empty(), f"Ubatch slice {ubatch_slice} is empty"
|
||||
|
||||
req_start = ubatch_slice.request_slice.start
|
||||
req_stop = ubatch_slice.request_slice.stop
|
||||
tok_start = ubatch_slice.token_slice.start
|
||||
tok_stop = ubatch_slice.token_slice.stop
|
||||
|
||||
num_reqs_padded = req_stop - req_start
|
||||
num_tokens_padded = tok_stop - tok_start
|
||||
# The trailing microbatch may extend past the real batch into CUDA graph
|
||||
# padding; the unpadded counts stop at the last real request/token.
|
||||
num_reqs = max(0, min(req_stop, input_batch.num_reqs) - req_start)
|
||||
num_tokens = max(0, min(tok_stop, input_batch.num_tokens) - tok_start)
|
||||
|
||||
query_start_loc = _slice_query_start_loc(
|
||||
input_batch.query_start_loc,
|
||||
query_start_loc_buf[: num_reqs_padded + 1],
|
||||
req_start,
|
||||
req_stop,
|
||||
tok_start,
|
||||
num_tokens_padded,
|
||||
)
|
||||
query_start_loc_np = np.clip(
|
||||
input_batch.query_start_loc_np[req_start : req_stop + 1] - tok_start,
|
||||
0,
|
||||
num_tokens_padded,
|
||||
).astype(np.int32)
|
||||
|
||||
seq_lens = _slice_seq_lens(
|
||||
input_batch.seq_lens,
|
||||
seq_lens_buf[:num_reqs_padded],
|
||||
input_batch.query_start_loc,
|
||||
req_start,
|
||||
req_stop,
|
||||
tok_stop,
|
||||
)
|
||||
# Same truncation as `_slice_seq_lens`, on the host-side upper bound.
|
||||
seq_lens_cpu_upper_bound = input_batch.seq_lens_cpu_upper_bound[
|
||||
req_start:req_stop
|
||||
].clone()
|
||||
tokens_truncated = max(0, int(input_batch.query_start_loc_np[req_stop]) - tok_stop)
|
||||
if tokens_truncated:
|
||||
seq_lens_cpu_upper_bound[-1] -= tokens_truncated
|
||||
|
||||
# Query lengths of the truncated requests, so consumers that derive
|
||||
# max_query_len from this array see the microbatch's own lengths.
|
||||
num_scheduled_tokens = np.diff(query_start_loc_np)[:num_reqs]
|
||||
|
||||
dcp_local_seq_lens = input_batch.dcp_local_seq_lens
|
||||
if dcp_local_seq_lens is not None:
|
||||
# NOTE: a request split across microbatches keeps its full local
|
||||
# seq_len here. DCP is not yet validated with DBO.
|
||||
dcp_local_seq_lens = dcp_local_seq_lens[req_start:req_stop]
|
||||
|
||||
return replace(
|
||||
input_batch,
|
||||
req_ids=input_batch.req_ids[req_start : min(req_stop, input_batch.num_reqs)],
|
||||
num_reqs=num_reqs,
|
||||
num_reqs_after_padding=num_reqs_padded,
|
||||
idx_mapping=input_batch.idx_mapping[req_start:req_stop],
|
||||
idx_mapping_np=input_batch.idx_mapping_np[req_start:req_stop],
|
||||
num_scheduled_tokens=num_scheduled_tokens,
|
||||
num_tokens=num_tokens,
|
||||
num_tokens_after_padding=num_tokens_padded,
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_np=query_start_loc_np,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||
dcp_local_seq_lens=dcp_local_seq_lens,
|
||||
num_computed_tokens_np=input_batch.num_computed_tokens_np[req_start:req_stop],
|
||||
prefill_len_np=input_batch.prefill_len_np[req_start:req_stop],
|
||||
num_computed_prefill_tokens_np=input_batch.num_computed_prefill_tokens_np[
|
||||
req_start:req_stop
|
||||
],
|
||||
is_prefilling_np=input_batch.is_prefilling_np[req_start:req_stop],
|
||||
input_ids=input_batch.input_ids[tok_start:tok_stop],
|
||||
positions=input_batch.positions[tok_start:tok_stop],
|
||||
is_padding=input_batch.is_padding[tok_start:tok_stop],
|
||||
prompt_lens=(
|
||||
None
|
||||
if input_batch.prompt_lens is None
|
||||
else input_batch.prompt_lens[req_start:req_stop]
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _slice_query_start_loc(
|
||||
query_start_loc: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
req_start: int,
|
||||
req_stop: int,
|
||||
tok_start: int,
|
||||
num_tokens_padded: int,
|
||||
) -> torch.Tensor:
|
||||
"""Rebase query_start_loc onto the microbatch's token range.
|
||||
|
||||
The clamp truncates the requests straddling either boundary to the tokens
|
||||
this microbatch owns.
|
||||
"""
|
||||
torch.sub(query_start_loc[req_start : req_stop + 1], tok_start, out=out)
|
||||
return out.clamp_(0, num_tokens_padded)
|
||||
|
||||
|
||||
def _slice_seq_lens(
|
||||
seq_lens: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
req_start: int,
|
||||
req_stop: int,
|
||||
tok_stop: int,
|
||||
) -> torch.Tensor:
|
||||
"""Slice seq_lens, shortening the request truncated at the boundary.
|
||||
|
||||
Only tokens past the end of this microbatch are missing; tokens it owns in
|
||||
an *earlier* microbatch are computed before this one reads them, so they
|
||||
still count. Deriving the truncation from this microbatch's own query
|
||||
lengths would subtract them twice. It is computed from tensors, not Python
|
||||
ints, so this stays valid inside a CUDA graph.
|
||||
"""
|
||||
out.copy_(seq_lens[req_start:req_stop])
|
||||
last = req_stop - req_start - 1
|
||||
out[last] -= (query_start_loc[req_stop] - tok_stop).clamp_(min=0)
|
||||
return out
|
||||
|
||||
|
||||
def slice_model_inputs(
|
||||
model_inputs: dict[str, Any], token_slice: slice
|
||||
) -> dict[str, Any]:
|
||||
"""Narrow the model's per-token inputs to one microbatch."""
|
||||
sliced = dict(model_inputs)
|
||||
for key in ("input_ids", "inputs_embeds"):
|
||||
value = model_inputs.get(key)
|
||||
if value is not None:
|
||||
sliced[key] = value[token_slice]
|
||||
|
||||
positions = model_inputs["positions"]
|
||||
# M-RoPE carries a leading section dim.
|
||||
sliced["positions"] = (
|
||||
positions[:, token_slice] if positions.ndim == 2 else positions[token_slice]
|
||||
)
|
||||
|
||||
intermediate_tensors = model_inputs.get("intermediate_tensors")
|
||||
if intermediate_tensors is not None:
|
||||
sliced["intermediate_tensors"] = intermediate_tensors[token_slice]
|
||||
return sliced
|
||||
|
||||
|
||||
def merge_ubatch_outputs(outputs: list[Any]) -> Any:
|
||||
"""Reassemble the full-batch model output from the per-microbatch ones.
|
||||
|
||||
Models return hidden states, a tuple of those plus auxiliary hidden states
|
||||
(EAGLE3), or IntermediateTensors on non-final pipeline ranks.
|
||||
"""
|
||||
first = outputs[0]
|
||||
if isinstance(first, IntermediateTensors):
|
||||
return IntermediateTensors(
|
||||
{
|
||||
key: torch.cat([out.tensors[key] for out in outputs], dim=0)
|
||||
for key in first.tensors
|
||||
}
|
||||
)
|
||||
if isinstance(first, tuple):
|
||||
return tuple(torch.cat(parts, dim=0) for parts in zip(*outputs))
|
||||
return torch.cat(outputs, dim=0)
|
||||
|
||||
|
||||
class UBatchRunner:
|
||||
"""Runs the model on two or more microbatches that overlap each other.
|
||||
|
||||
Each microbatch gets a thread and its own forward context. Only one thread
|
||||
holds the GPU at a time; they hand off to each other at the communication
|
||||
points inside the model (see `vllm.v1.worker.ubatching`), so one
|
||||
microbatch's expert all-to-all overlaps the other's compute.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
device: torch.device,
|
||||
model_state: ModelState,
|
||||
attn_groups: list[list[AttentionGroup]],
|
||||
kv_cache_config: KVCacheConfig,
|
||||
max_num_reqs: int,
|
||||
):
|
||||
self.vllm_config = vllm_config
|
||||
self.parallel_config = vllm_config.parallel_config
|
||||
self.num_ubatches = self.parallel_config.num_ubatches
|
||||
self.device = device
|
||||
self.model_state = model_state
|
||||
self.attn_groups = attn_groups
|
||||
self.kv_cache_config = kv_cache_config
|
||||
# `query_start_loc` and `seq_lens` are rebased onto each microbatch's
|
||||
# own token range, so they cannot be views of the full batch's buffers.
|
||||
# Allocating up front keeps their addresses stable across replays.
|
||||
self.ubatch_query_start_loc = [
|
||||
torch.zeros(max_num_reqs + 1, dtype=torch.int32, device=device)
|
||||
for _ in range(self.num_ubatches)
|
||||
]
|
||||
self.ubatch_seq_lens = [
|
||||
torch.zeros(max_num_reqs, dtype=torch.int32, device=device)
|
||||
for _ in range(self.num_ubatches)
|
||||
]
|
||||
self.comm_stream = torch.cuda.Stream(device=device)
|
||||
# The microbatch threads plus the thread that starts them.
|
||||
self.ready_barrier = threading.Barrier(self.num_ubatches + 1)
|
||||
self.sm_control = create_sm_control_context(self.parallel_config)
|
||||
|
||||
def prepare(
|
||||
self,
|
||||
input_batch: InputBatch,
|
||||
block_tables: tuple[torch.Tensor, ...],
|
||||
slot_mappings: torch.Tensor,
|
||||
) -> UBatchState:
|
||||
"""Split the batch into the microbatches the step will run on.
|
||||
|
||||
Attention metadata is built per microbatch and carried in the forward
|
||||
contexts the threads install, not in the caller's.
|
||||
"""
|
||||
ubatch_slices = create_ubatch_slices(input_batch, self.num_ubatches)
|
||||
|
||||
attn_metadata = []
|
||||
slot_mappings_by_layer = []
|
||||
is_padding = []
|
||||
for i, ubatch_slice in enumerate(ubatch_slices):
|
||||
ubatch = _slice_input_batch(
|
||||
input_batch,
|
||||
ubatch_slice,
|
||||
self.ubatch_query_start_loc[i],
|
||||
self.ubatch_seq_lens[i],
|
||||
)
|
||||
ubatch_slot_mappings = slot_mappings[:, ubatch_slice.token_slice]
|
||||
ubatch_block_tables = tuple(
|
||||
block_table[ubatch_slice.request_slice] for block_table in block_tables
|
||||
)
|
||||
attn_metadata.append(
|
||||
self.model_state.prepare_attn(
|
||||
ubatch,
|
||||
CUDAGraphMode.NONE,
|
||||
ubatch_block_tables,
|
||||
ubatch_slot_mappings,
|
||||
self.attn_groups,
|
||||
self.kv_cache_config,
|
||||
ubatch_idx=i,
|
||||
)
|
||||
)
|
||||
slot_mappings_by_layer.append(
|
||||
build_slot_mappings_by_layer(ubatch_slot_mappings, self.kv_cache_config)
|
||||
)
|
||||
is_padding.append(ubatch.is_padding)
|
||||
|
||||
return UBatchState(
|
||||
slices=ubatch_slices,
|
||||
forward_contexts=self._make_forward_contexts(
|
||||
ubatch_slices, attn_metadata, slot_mappings_by_layer, is_padding
|
||||
),
|
||||
)
|
||||
|
||||
def _make_forward_contexts(
|
||||
self,
|
||||
ubatch_slices: UBatchSlices,
|
||||
attn_metadata: list[dict[str, Any]],
|
||||
slot_mappings_by_layer: list[dict[str, torch.Tensor]],
|
||||
is_padding: list[torch.Tensor | None],
|
||||
) -> list[ForwardContext]:
|
||||
"""Build one forward context per microbatch.
|
||||
|
||||
The expert all-to-all runs once per microbatch, so each context's DP
|
||||
metadata carries that microbatch's token count. Microbatching pads every
|
||||
DP rank alike, so each rank can assume that count for the others too.
|
||||
"""
|
||||
dp_size = self.parallel_config.data_parallel_size
|
||||
forward_contexts = []
|
||||
for i, ubatch_slice in enumerate(ubatch_slices):
|
||||
num_tokens = ubatch_slice.num_tokens
|
||||
num_tokens_across_dp = torch.full(
|
||||
(dp_size,), num_tokens, dtype=torch.int32, device="cpu"
|
||||
)
|
||||
forward_contexts.append(
|
||||
create_forward_context(
|
||||
attn_metadata[i],
|
||||
self.vllm_config,
|
||||
dp_metadata=DPMetadata.make(
|
||||
self.parallel_config, num_tokens, num_tokens_across_dp
|
||||
),
|
||||
cudagraph_runtime_mode=CUDAGraphMode.NONE,
|
||||
slot_mapping=slot_mappings_by_layer[i],
|
||||
is_padding=is_padding[i],
|
||||
)
|
||||
)
|
||||
return forward_contexts
|
||||
|
||||
def run(
|
||||
self,
|
||||
model: Any,
|
||||
model_inputs: dict[str, Any],
|
||||
ubatch_state: UBatchState,
|
||||
) -> Any:
|
||||
ubatch_slices = ubatch_state.slices
|
||||
assert len(ubatch_slices) == len(ubatch_state.forward_contexts)
|
||||
assert len(ubatch_slices) == self.num_ubatches
|
||||
|
||||
ubatch_contexts = make_ubatch_contexts(
|
||||
num_micro_batches=self.num_ubatches,
|
||||
comm_stream=self.comm_stream,
|
||||
compute_stream=current_stream(),
|
||||
forward_contexts=ubatch_state.forward_contexts,
|
||||
ready_barrier=self.ready_barrier,
|
||||
)
|
||||
|
||||
outputs: dict[int, Any] = {}
|
||||
errors: dict[int, BaseException] = {}
|
||||
|
||||
@torch.inference_mode()
|
||||
def run_ubatch(ubatch_context, inputs: dict[str, Any]) -> None:
|
||||
try:
|
||||
with ubatch_context:
|
||||
outputs[ubatch_context.id] = model(**inputs)
|
||||
except BaseException as e: # noqa: BLE001
|
||||
# Recorded so the join below can report which microbatch broke.
|
||||
# Note the siblings are not told to unwind: microbatches hand
|
||||
# control to each other, so one that dies mid-forward leaves
|
||||
# the others parked on a handoff that never comes and the step
|
||||
# hangs. The V1 runner behaves the same way; fixing it means
|
||||
# changing the shared handoff protocol in ubatching.py, which
|
||||
# is deliberately out of scope here.
|
||||
errors[ubatch_context.id] = e
|
||||
|
||||
# The threads manage the forward context themselves; clear it here so
|
||||
# it is restored correctly once they are done.
|
||||
with override_forward_context(None), self.sm_control:
|
||||
threads = []
|
||||
for ubatch_context, ubatch_slice in zip(ubatch_contexts, ubatch_slices):
|
||||
thread = threading.Thread(
|
||||
target=run_ubatch,
|
||||
args=(
|
||||
ubatch_context,
|
||||
slice_model_inputs(model_inputs, ubatch_slice.token_slice),
|
||||
),
|
||||
)
|
||||
threads.append(thread)
|
||||
thread.start()
|
||||
|
||||
# Wait for every thread to reach its context, then start the first.
|
||||
self.ready_barrier.wait()
|
||||
ubatch_contexts[0].cpu_wait_event.set()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
|
||||
if errors:
|
||||
failed = min(errors)
|
||||
raise RuntimeError(
|
||||
f"Microbatch {failed} of {self.num_ubatches} failed"
|
||||
) from errors[failed]
|
||||
return merge_ubatch_outputs([outputs[i] for i in range(self.num_ubatches)])
|
||||
|
||||
|
||||
def maybe_build_ubatch_runner(
|
||||
vllm_config: VllmConfig,
|
||||
device: torch.device,
|
||||
model_state: ModelState,
|
||||
attn_groups: list[list[AttentionGroup]],
|
||||
kv_cache_config: KVCacheConfig,
|
||||
max_num_reqs: int,
|
||||
) -> UBatchRunner | None:
|
||||
"""Build the microbatch runner, or None when DBO is not in use.
|
||||
|
||||
Microbatching needs the DP handshake to agree on it, so it is only
|
||||
available with more than one DP rank (as in the V1 runner).
|
||||
"""
|
||||
parallel_config = vllm_config.parallel_config
|
||||
if not parallel_config.use_ubatching or parallel_config.data_parallel_size <= 1:
|
||||
return None
|
||||
|
||||
logger.info_once(
|
||||
"Dual batch overlap is enabled. Microbatched steps run without "
|
||||
"CUDA graphs on the V2 model runner."
|
||||
)
|
||||
return UBatchRunner(
|
||||
vllm_config, device, model_state, attn_groups, kv_cache_config, max_num_reqs
|
||||
)
|
||||
@@ -11,7 +11,6 @@ import torch
|
||||
import vllm.envs as envs
|
||||
from vllm.compilation.cuda_graph import CUDAGraphWrapper
|
||||
from vllm.config import CUDAGraphMode, VllmConfig
|
||||
from vllm.distributed import get_ep_group
|
||||
from vllm.distributed.device_communicators.pynccl_allocator import set_graph_pool_id
|
||||
from vllm.forward_context import (
|
||||
DPMetadata,
|
||||
@@ -23,9 +22,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.offloader.base import get_offloader
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.deep_gemm import set_num_sms as deep_gemm_set_num_sms
|
||||
from vllm.utils.import_utils import has_deep_gemm
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
from vllm.v1.worker.ubatch_utils import create_sm_control_context
|
||||
from vllm.v1.worker.ubatching import UBatchContext, make_ubatch_contexts
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -65,51 +62,6 @@ class CUDAGraphMetaData:
|
||||
outputs: Any | None = None
|
||||
|
||||
|
||||
class SMControlContextManager:
|
||||
def __init__(
|
||||
self,
|
||||
comm_sms: int,
|
||||
set_comm_sms: Callable[[int], None],
|
||||
set_compute_sms: Callable[[int], None],
|
||||
):
|
||||
"""
|
||||
Context manager for controlling SM (Streaming Multiprocessor)
|
||||
allocation. Upon entering the context, it sets the number of SMs
|
||||
allocated for communication and computation to comm_sms and
|
||||
total_sms - comm_sms respectively. Upon exiting, it restores the
|
||||
allocation to use all available SMs (i.e. total_sms).
|
||||
|
||||
Args:
|
||||
comm_sms (int): The number of SMs to allocate for communication.
|
||||
(The remainder will be used for computation.)
|
||||
set_comm_sms (Callable[[int], None]):
|
||||
A function that sets the number of SMs for communication.
|
||||
set_compute_sms (Callable[[int], None]):
|
||||
A function that sets the number of SMs for computation.
|
||||
"""
|
||||
|
||||
assert current_platform.is_cuda() or current_platform.is_rocm(), (
|
||||
"SM/CU control is supported on CUDA and ROCm platforms"
|
||||
)
|
||||
device = torch.accelerator.current_device_index()
|
||||
total_sms = num_compute_units(device)
|
||||
|
||||
assert comm_sms < total_sms
|
||||
self.total_sms = total_sms
|
||||
self.compute_sms = total_sms - comm_sms
|
||||
self.comm_sms = comm_sms
|
||||
self.set_comm_sms = set_comm_sms
|
||||
self.set_compute_sms = set_compute_sms
|
||||
|
||||
def __enter__(self):
|
||||
self.set_comm_sms(self.comm_sms)
|
||||
self.set_compute_sms(self.compute_sms)
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
self.set_comm_sms(self.total_sms)
|
||||
self.set_compute_sms(self.total_sms)
|
||||
|
||||
|
||||
class UBatchWrapper:
|
||||
def __init__(
|
||||
self,
|
||||
@@ -135,7 +87,7 @@ class UBatchWrapper:
|
||||
runnable, vllm_config, runtime_mode=runtime_mode
|
||||
)
|
||||
|
||||
self.sm_control = self._create_sm_control_context(vllm_config)
|
||||
self.sm_control = create_sm_control_context(vllm_config.parallel_config)
|
||||
self.device = device
|
||||
self.is_debugging_mode = envs.VLLM_LOGGING_LEVEL == "DEBUG"
|
||||
self._runnable_str = str(runnable) if self.is_debugging_mode else None
|
||||
@@ -151,49 +103,6 @@ class UBatchWrapper:
|
||||
if self.cudagraph_wrapper is not None:
|
||||
self.cudagraph_wrapper.clear_graphs()
|
||||
|
||||
@staticmethod
|
||||
def _create_sm_control_context(vllm_config: VllmConfig):
|
||||
comm_sms: int = envs.VLLM_DBO_COMM_SMS
|
||||
rocm_deepep_ht_dbo = (
|
||||
current_platform.is_rocm()
|
||||
and vllm_config.parallel_config.enable_dbo
|
||||
and vllm_config.parallel_config.all2all_backend == "deepep_high_throughput"
|
||||
)
|
||||
if rocm_deepep_ht_dbo:
|
||||
# On ROCm, reserving CUs for DeepEP HT communication under DBO
|
||||
# corrupts DP+EP generation accuracy. Keep the backend active, but
|
||||
# leave all CUs visible to the compute and communication kernels.
|
||||
comm_sms = 0
|
||||
|
||||
set_comm_sms = lambda sms: None
|
||||
if vllm_config.parallel_config.enable_expert_parallel:
|
||||
# Currently only DeepEP highthroughput supports SM control so this
|
||||
# only affects that case.
|
||||
ep_group = get_ep_group()
|
||||
device_communicator = ep_group.device_communicator
|
||||
all2all_manager = None
|
||||
if device_communicator is not None:
|
||||
all2all_manager = device_communicator.all2all_manager
|
||||
|
||||
if all2all_manager is not None:
|
||||
max_sms_used = all2all_manager.max_sms_used()
|
||||
if max_sms_used is not None:
|
||||
comm_sms = min(comm_sms, max_sms_used)
|
||||
|
||||
if comm_sms > 0 and all2all_manager is not None:
|
||||
set_comm_sms = lambda sms: all2all_manager.set_num_sms(sms)
|
||||
|
||||
# TODO(lucas): support other kernels besides DeepGEMM
|
||||
set_compute_sms = lambda sms: None
|
||||
if has_deep_gemm() and comm_sms > 0:
|
||||
set_compute_sms = lambda sms: deep_gemm_set_num_sms(sms)
|
||||
|
||||
return SMControlContextManager(
|
||||
comm_sms=comm_sms,
|
||||
set_comm_sms=set_comm_sms,
|
||||
set_compute_sms=set_compute_sms,
|
||||
)
|
||||
|
||||
def __getattr__(self, key: str):
|
||||
# allow accessing the attributes of the runnable.
|
||||
if hasattr(self.runnable, key):
|
||||
|
||||
@@ -1,12 +1,19 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import TypeAlias
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.distributed import get_ep_group
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import set_num_sms as deep_gemm_set_num_sms
|
||||
from vllm.utils.import_utils import has_deep_gemm
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
|
||||
|
||||
@@ -29,12 +36,108 @@ class UBatchSlice:
|
||||
UBatchSlices: TypeAlias = list[UBatchSlice]
|
||||
|
||||
|
||||
class SMControlContextManager:
|
||||
def __init__(
|
||||
self,
|
||||
comm_sms: int,
|
||||
set_comm_sms: Callable[[int], None],
|
||||
set_compute_sms: Callable[[int], None],
|
||||
):
|
||||
"""
|
||||
Context manager for controlling SM (Streaming Multiprocessor)
|
||||
allocation. Upon entering the context, it sets the number of SMs
|
||||
allocated for communication and computation to comm_sms and
|
||||
total_sms - comm_sms respectively. Upon exiting, it restores the
|
||||
allocation to use all available SMs (i.e. total_sms).
|
||||
|
||||
Args:
|
||||
comm_sms (int): The number of SMs to allocate for communication.
|
||||
(The remainder will be used for computation.)
|
||||
set_comm_sms (Callable[[int], None]):
|
||||
A function that sets the number of SMs for communication.
|
||||
set_compute_sms (Callable[[int], None]):
|
||||
A function that sets the number of SMs for computation.
|
||||
"""
|
||||
|
||||
assert current_platform.is_cuda() or current_platform.is_rocm(), (
|
||||
"SM/CU control is supported on CUDA and ROCm platforms"
|
||||
)
|
||||
device = torch.accelerator.current_device_index()
|
||||
total_sms = num_compute_units(device)
|
||||
|
||||
assert comm_sms < total_sms
|
||||
self.total_sms = total_sms
|
||||
self.compute_sms = total_sms - comm_sms
|
||||
self.comm_sms = comm_sms
|
||||
self.set_comm_sms = set_comm_sms
|
||||
self.set_compute_sms = set_compute_sms
|
||||
|
||||
def __enter__(self):
|
||||
self.set_comm_sms(self.comm_sms)
|
||||
self.set_compute_sms(self.compute_sms)
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
self.set_comm_sms(self.total_sms)
|
||||
self.set_compute_sms(self.total_sms)
|
||||
|
||||
|
||||
def create_sm_control_context(
|
||||
parallel_config: ParallelConfig,
|
||||
) -> SMControlContextManager:
|
||||
"""Reserve SMs for communication kernels while microbatches overlap."""
|
||||
comm_sms: int = envs.VLLM_DBO_COMM_SMS
|
||||
rocm_deepep_ht_dbo = (
|
||||
current_platform.is_rocm()
|
||||
and parallel_config.enable_dbo
|
||||
and parallel_config.all2all_backend == "deepep_high_throughput"
|
||||
)
|
||||
if rocm_deepep_ht_dbo:
|
||||
# On ROCm, reserving CUs for DeepEP HT communication under DBO
|
||||
# corrupts DP+EP generation accuracy. Keep the backend active, but
|
||||
# leave all CUs visible to the compute and communication kernels.
|
||||
comm_sms = 0
|
||||
|
||||
set_comm_sms = lambda sms: None
|
||||
if parallel_config.enable_expert_parallel:
|
||||
# Currently only DeepEP highthroughput supports SM control so this
|
||||
# only affects that case.
|
||||
ep_group = get_ep_group()
|
||||
device_communicator = ep_group.device_communicator
|
||||
all2all_manager = None
|
||||
if device_communicator is not None:
|
||||
all2all_manager = device_communicator.all2all_manager
|
||||
|
||||
if all2all_manager is not None:
|
||||
max_sms_used = all2all_manager.max_sms_used()
|
||||
if max_sms_used is not None:
|
||||
comm_sms = min(comm_sms, max_sms_used)
|
||||
|
||||
if comm_sms > 0 and all2all_manager is not None:
|
||||
set_comm_sms = lambda sms: all2all_manager.set_num_sms(sms)
|
||||
|
||||
# TODO(lucas): support other kernels besides DeepGEMM
|
||||
set_compute_sms = lambda sms: None
|
||||
if has_deep_gemm() and comm_sms > 0:
|
||||
set_compute_sms = lambda sms: deep_gemm_set_num_sms(sms)
|
||||
|
||||
return SMControlContextManager(
|
||||
comm_sms=comm_sms,
|
||||
set_comm_sms=set_comm_sms,
|
||||
set_compute_sms=set_compute_sms,
|
||||
)
|
||||
|
||||
|
||||
def is_last_ubatch_empty(
|
||||
orig_num_tokens: int, padded_num_tokens: int, num_ubatches: int
|
||||
) -> bool:
|
||||
return (padded_num_tokens // num_ubatches) * (num_ubatches - 1) >= orig_num_tokens
|
||||
|
||||
|
||||
def get_num_ubatches(parallel_config: ParallelConfig) -> int:
|
||||
"""How many microbatches a step is split into; 1 when microbatching is off."""
|
||||
return parallel_config.num_ubatches if parallel_config.use_ubatching else 1
|
||||
|
||||
|
||||
def check_ubatch_thresholds(
|
||||
config: ParallelConfig, num_tokens: int, uniform_decode: bool
|
||||
) -> bool:
|
||||
|
||||
Reference in New Issue
Block a user