From 3f41d102c5b8636f7206a5fd5949dd8ae0b30a0d Mon Sep 17 00:00:00 2001 From: Anjie Hou <149605198+specture724@users.noreply.github.com> Date: Sat, 5 Sep 2026 05:17:39 +0800 Subject: [PATCH] [1/2][Model Runner V2] DBO support, eager mode (#50945) Signed-off-by: specture724 Signed-off-by: Anjie Hou <149605198+specture724@users.noreply.github.com> Co-authored-by: jiangkuaixue123 Co-authored-by: Lucas Wilkinson --- .../test_routed_experts_capture.py | 1 + tests/v1/worker/test_gpu_ubatch_slicing.py | 583 ++++++++++++++++++ vllm/config/parallel.py | 11 + vllm/config/vllm.py | 45 +- vllm/model_executor/models/diffusion_gemma.py | 2 + vllm/v1/worker/gpu/attn_utils.py | 23 +- vllm/v1/worker/gpu/cudagraph_utils.py | 8 + vllm/v1/worker/gpu/dp_utils.py | 60 +- vllm/v1/worker/gpu/model_runner.py | 44 +- vllm/v1/worker/gpu/model_states/default.py | 2 + .../gpu/model_states/encoder_decoder.py | 2 + .../worker/gpu/model_states/encoder_only.py | 2 + vllm/v1/worker/gpu/model_states/interface.py | 1 + .../worker/gpu/model_states/mamba_hybrid.py | 2 + vllm/v1/worker/gpu/ubatch_utils.py | 473 ++++++++++++++ vllm/v1/worker/gpu_ubatch_wrapper.py | 95 +-- vllm/v1/worker/ubatch_utils.py | 103 ++++ 17 files changed, 1346 insertions(+), 111 deletions(-) create mode 100644 tests/v1/worker/test_gpu_ubatch_slicing.py create mode 100644 vllm/v1/worker/gpu/ubatch_utils.py diff --git a/tests/model_executor/test_routed_experts_capture.py b/tests/model_executor/test_routed_experts_capture.py index a556b72488..49986a991e 100644 --- a/tests/model_executor/test_routed_experts_capture.py +++ b/tests/model_executor/test_routed_experts_capture.py @@ -453,6 +453,7 @@ def test_v2_model_runner_accepts_routed_experts(monkeypatch): distributed_executor_backend=None, pipeline_parallel_size=1, enable_dbo=False, + use_ubatching=False, enable_elastic_ep=False, ), compilation_config=SimpleNamespace( diff --git a/tests/v1/worker/test_gpu_ubatch_slicing.py b/tests/v1/worker/test_gpu_ubatch_slicing.py new file mode 100644 index 0000000000..5c76471803 --- /dev/null +++ b/tests/v1/worker/test_gpu_ubatch_slicing.py @@ -0,0 +1,583 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Dual batch overlap (DBO) in the V2 model runner. + +The V2 runner slices the batch *before* building attention metadata, where V1 +builds metadata for the whole batch and slices it afterwards. Both must describe +the same microbatches, so the main test here pins V2's slicing against V1's +`split_attn_metadata`. The rest covers the decision to microbatch and the +threaded execution of the microbatches. +""" + +import threading +from dataclasses import replace +from types import SimpleNamespace +from typing import Any, cast +from unittest.mock import patch + +import numpy as np +import pytest +import torch + +from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata +from vllm.config import CUDAGraphMode, ModelConfig, ParallelConfig, VllmConfig +from vllm.forward_context import create_forward_context +from vllm.v1.kv_cache_interface import KVCacheConfig +from vllm.v1.worker.gpu import dp_utils +from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor +from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers +from vllm.v1.worker.gpu.model_states.interface import ModelState +from vllm.v1.worker.gpu.ubatch_utils import ( + UBatchRunner, + UBatchState, + _slice_input_batch, + create_ubatch_slices, + slice_model_inputs, +) +from vllm.v1.worker.ubatch_utils import ( + UBatchSlice, + maybe_create_ubatch_slices, + split_attn_metadata, +) +from vllm.v1.worker.ubatching import dbo_current_ubatch_id, dbo_yield + +MAX_NUM_REQS = 32 +MAX_NUM_TOKENS = 128 + + +def _make_buffers() -> InputBuffers: + return InputBuffers( + max_num_reqs=MAX_NUM_REQS, + max_num_tokens=MAX_NUM_TOKENS, + device=torch.device("cpu"), + ) + + +def _make_ubatch_buffers(num_ubatches: int = 2) -> list[tuple[torch.Tensor, ...]]: + """The per-microbatch (query_start_loc, seq_lens) buffers UBatchRunner owns.""" + return [ + ( + torch.zeros(MAX_NUM_REQS + 1, dtype=torch.int32), + torch.zeros(MAX_NUM_REQS, dtype=torch.int32), + ) + for _ in range(num_ubatches) + ] + + +def _make_input_batch( + query_lens: list[int], + seq_lens: list[int], + buffers: InputBuffers, + num_reqs_padded: int | None = None, + num_tokens_padded: int | None = None, +) -> InputBatch: + """Build an InputBatch shaped like the one `prepare_inputs` produces.""" + num_reqs = len(query_lens) + num_tokens = int(sum(query_lens)) + num_reqs_padded = num_reqs_padded or num_reqs + num_tokens_padded = num_tokens_padded or num_tokens + + # make_dummy writes the shared buffers, so fill them afterwards. + base = InputBatch.make_dummy(num_reqs, num_tokens, buffers) + + query_start_loc_np = np.zeros(num_reqs_padded + 1, dtype=np.int32) + np.cumsum(query_lens, out=query_start_loc_np[1 : num_reqs + 1]) + # Padded entries repeat the token count, as `prepare_inputs` does. + query_start_loc_np[num_reqs + 1 :] = num_tokens + buffers.query_start_loc[: num_reqs_padded + 1] = torch.from_numpy( + query_start_loc_np + ) + + buffers.seq_lens[:num_reqs] = torch.tensor(seq_lens, dtype=torch.int32) + buffers.seq_lens[num_reqs:num_reqs_padded] = 0 + + seq_lens_upper_bound = np.zeros(num_reqs_padded, dtype=np.int32) + seq_lens_upper_bound[:num_reqs] = seq_lens + + return replace( + base, + req_ids=[f"req_{i}" for i in range(num_reqs)], + num_reqs=num_reqs, + num_reqs_after_padding=num_reqs_padded, + idx_mapping=torch.arange(num_reqs, dtype=torch.int32), + idx_mapping_np=np.arange(num_reqs, dtype=np.int32), + num_scheduled_tokens=np.array(query_lens, dtype=np.int32), + num_tokens=num_tokens, + num_tokens_after_padding=num_tokens_padded, + query_start_loc=buffers.query_start_loc[: num_reqs_padded + 1], + query_start_loc_np=query_start_loc_np, + seq_lens=buffers.seq_lens[:num_reqs_padded], + seq_lens_cpu_upper_bound=torch.from_numpy(seq_lens_upper_bound), + num_computed_tokens_np=np.array(seq_lens, dtype=np.int32) + - np.array(query_lens, dtype=np.int32), + prefill_len_np=np.zeros(num_reqs, dtype=np.int32), + num_computed_prefill_tokens_np=np.zeros(num_reqs, dtype=np.int32), + is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_), + input_ids=buffers.input_ids[:num_tokens_padded], + positions=buffers.positions[:num_tokens_padded], + is_padding=buffers.is_padding[:num_tokens_padded], + ) + + +# (query_lens, seq_lens): uniform decode, a boundary that lands inside a +# request, and a boundary that lands exactly on a request start. +BATCHES = { + "uniform_decode": ([1] * 8, [128 + 4 * i for i in range(8)]), + "split_request": ([1, 1, 10, 1, 1], [64, 96, 512, 32, 48]), + "aligned_prefills": ([6, 2, 4, 4], [6, 130, 260, 4]), + "split_first_and_last": ([3, 9, 3], [70, 300, 41]), + # A long prefill trailing a decode: it straddles the boundary and is also + # the last request, so the second microbatch holds nothing else. + "trailing_prefill_spans_boundary": ([2, 20], [64, 300]), +} + + +@pytest.mark.parametrize("batch_name", list(BATCHES)) +def test_slicing_matches_v1_split_attn_metadata(batch_name: str): + """V2's pre-sliced inputs describe the same microbatches as V1's.""" + query_lens, seq_lens = BATCHES[batch_name] + num_scheduled_tokens = np.array(query_lens, dtype=np.int32) + num_tokens = int(num_scheduled_tokens.sum()) + + ubatch_slices, _ = maybe_create_ubatch_slices( + True, num_scheduled_tokens, num_tokens, len(query_lens), num_ubatches=2 + ) + assert ubatch_slices is not None + + v1_metadata = create_common_attn_metadata( + BatchSpec(seq_lens=seq_lens, query_lens=query_lens), + block_size=16, + device=torch.device("cpu"), + ) + v1_ubatches = split_attn_metadata(ubatch_slices, v1_metadata) + + buffers = _make_buffers() + ubatch_buffers = _make_ubatch_buffers() + input_batch = _make_input_batch(query_lens, seq_lens, buffers) + + for i, (ubatch_slice, v1_ubatch) in enumerate(zip(ubatch_slices, v1_ubatches)): + v2_ubatch = _slice_input_batch(input_batch, ubatch_slice, *ubatch_buffers[i]) + + assert v2_ubatch.num_reqs == v1_ubatch.num_reqs + assert v2_ubatch.num_tokens == v1_ubatch.num_actual_tokens + assert int(v2_ubatch.num_scheduled_tokens.max()) == v1_ubatch.max_query_len + torch.testing.assert_close(v2_ubatch.query_start_loc, v1_ubatch.query_start_loc) + torch.testing.assert_close(v2_ubatch.seq_lens, v1_ubatch.seq_lens) + torch.testing.assert_close( + torch.from_numpy(v2_ubatch.query_start_loc_np), + v1_ubatch.query_start_loc_cpu, + ) + # `prepare_attn` derives max_seq_len from the upper bound. Compare the + # bound itself rather than V1's max_seq_len: V1 floors it with the full + # batch's max_seq_len to keep the value CUDA-graph capture installed, + # which V2 gets from `prepare_attn(for_capture=True)` instead. + torch.testing.assert_close( + v2_ubatch.seq_lens_cpu_upper_bound, v1_ubatch.seq_lens_cpu_upper_bound + ) + + +def test_microbatches_do_not_share_buffers(): + """Both microbatches are live at once, so their buffers must be distinct.""" + query_lens, seq_lens = BATCHES["split_request"] + num_scheduled_tokens = np.array(query_lens, dtype=np.int32) + num_tokens = int(num_scheduled_tokens.sum()) + + ubatch_slices, _ = maybe_create_ubatch_slices( + True, num_scheduled_tokens, num_tokens, len(query_lens), num_ubatches=2 + ) + assert ubatch_slices is not None + + buffers = _make_buffers() + ubatch_buffers = _make_ubatch_buffers() + input_batch = _make_input_batch(query_lens, seq_lens, buffers) + + first = _slice_input_batch(input_batch, ubatch_slices[0], *ubatch_buffers[0]) + first_query_start_loc = first.query_start_loc.clone() + first_seq_lens = first.seq_lens.clone() + + # Slicing the second microbatch must not disturb the first. + _slice_input_batch(input_batch, ubatch_slices[1], *ubatch_buffers[1]) + + torch.testing.assert_close(first.query_start_loc, first_query_start_loc) + torch.testing.assert_close(first.seq_lens, first_seq_lens) + + +def test_trailing_microbatch_absorbs_cudagraph_padding(): + """The padded microbatch keeps padded rows empty and query_start_loc flat.""" + query_lens, seq_lens = [1] * 6, [64] * 6 + num_tokens_padded = 8 + + buffers = _make_buffers() + input_batch = _make_input_batch( + query_lens, + seq_lens, + buffers, + num_reqs_padded=num_tokens_padded, + num_tokens_padded=num_tokens_padded, + ) + ubatch_slices_padded = create_ubatch_slices(input_batch, num_ubatches=2) + + last = _slice_input_batch( + input_batch, ubatch_slices_padded[-1], *_make_ubatch_buffers()[1] + ) + + # Two real decodes plus two padded rows. + assert last.num_reqs == 2 + assert last.num_tokens == 2 + assert last.num_reqs_after_padding == 4 + assert last.num_tokens_after_padding == 4 + torch.testing.assert_close( + last.query_start_loc, torch.tensor([0, 1, 2, 2, 2], dtype=torch.int32) + ) + torch.testing.assert_close( + last.seq_lens, torch.tensor([64, 64, 0, 0], dtype=torch.int32) + ) + + +DECODE_THRESHOLD = 32 +PREFILL_THRESHOLD = 128 +DECODE_QUERY_LEN = 1 + + +def test_dummy_batch_can_be_microbatched(): + """A DP rank with no work still has to split, so its dummy batch must slice. + + Microbatching is all-or-nothing across the group, so a rank whose step is a + dummy batch runs it microbatched like everyone else. + """ + num_reqs, num_tokens = 1, 32 + buffers = _make_buffers() + input_batch = InputBatch.make_dummy(num_reqs, num_tokens, buffers) + + ubatch_slices_padded = create_ubatch_slices(input_batch, num_ubatches=2) + + ubatch_buffers = _make_ubatch_buffers() + ubatches = [ + _slice_input_batch(input_batch, ubatch_slice, *ubatch_buffers[i]) + for i, ubatch_slice in enumerate(ubatch_slices_padded) + ] + + assert [u.num_tokens for u in ubatches] == [16, 16] + # The one dummy request spans both microbatches, 16 of its tokens in each. + assert all(u.num_reqs == 1 for u in ubatches) + for ubatch in ubatches: + torch.testing.assert_close( + ubatch.query_start_loc, torch.tensor([0, 16], dtype=torch.int32) + ) + + +def _sync_dp( + num_tokens_per_rank: list[int], + uniform_token_count_per_rank: list[int] | None = None, + allow_ubatching: bool = True, +) -> tuple[BatchExecutionDescriptor, dp_utils.DPSyncState | None]: + """Run the DP handshake with the all-reduce stubbed out. + + The microbatching decision lives inside `sync_cudagraph_and_dp_padding`, so + the cases below stand in for the collective rather than for the decision, + and need no process group. Every rank asks for eager, which keeps the + non-microbatched path off the cudagraph manager. This rank is rank 0. + """ + dp_size = len(num_tokens_per_rank) + uniform_token_counts = uniform_token_count_per_rank or [0] * dp_size + reduced = torch.zeros(6, dp_size, dtype=torch.int32) + reduced[0] = torch.tensor(num_tokens_per_rank, dtype=torch.int32) + reduced[1] = CUDAGraphMode.NONE.value + reduced[2] = torch.tensor(uniform_token_counts, dtype=torch.int32) + reduced[3] = -1 # max_query_len, -1 means None + reduced[4] = int(allow_ubatching) + reduced[5] = 8 # num_reqs + + with ( + patch.object(dp_utils.dist, "all_reduce", lambda t, group: t.copy_(reduced)), + patch.object(dp_utils, "get_dp_group", lambda: SimpleNamespace(cpu_group=None)), + ): + return dp_utils.sync_cudagraph_and_dp_padding( + cudagraph_manager=None, + desired_batch_desc=BatchExecutionDescriptor( + cg_mode=CUDAGraphMode.NONE, + num_tokens=num_tokens_per_rank[0], + num_reqs=8, + ), + num_tokens=num_tokens_per_rank[0], + num_reqs=8, + uniform_token_count=uniform_token_counts[0] or None, + dp_size=dp_size, + dp_rank=0, + parallel_config=ParallelConfig( + enable_dbo=True, + dbo_decode_token_threshold=DECODE_THRESHOLD, + dbo_prefill_token_threshold=PREFILL_THRESHOLD, + ), + allow_ubatching=allow_ubatching, + uniform_decode=uniform_token_counts[0] == DECODE_QUERY_LEN, + ) + + +def test_every_dp_rank_must_agree_to_microbatch(): + """One rank below the threshold is enough to keep the group on one batch. + + The thresholds are not communicated -- they are checked against the token + counts the all-reduce already carries -- so this also pins that the whole + vector is consulted, not just this rank's own entry. + """ + assert _sync_dp([256, 256])[0].num_ubatches == 2 + assert _sync_dp([256, 100])[0].num_ubatches == 1 + assert _sync_dp([100, 256])[0].num_ubatches == 1 + + +def test_decode_threshold_only_applies_when_every_rank_is_a_uniform_decode(): + """Otherwise the group is held to the prefill threshold.""" + # 64 tokens clears the decode threshold but not the prefill one, so it only + # microbatches when both ranks are uniform decodes. + assert _sync_dp([64, 64], uniform_token_count_per_rank=[1, 1])[0].num_ubatches == 2 + assert _sync_dp([64, 64], uniform_token_count_per_rank=[1, 0])[0].num_ubatches == 1 + + +def test_microbatching_pads_all_ranks_to_the_largest(): + """Ranks must run the same token count so microbatch sizes line up.""" + desc, dp_sync = _sync_dp([200, 256]) + assert desc.num_tokens == 256 + assert dp_sync is not None + assert dp_sync.num_tokens_across_dp.tolist() == [256, 256] + + +def test_microbatching_survives_a_rank_that_cannot_fill_it(): + """A rank whose real tokens all land in the first microbatch still splits. + + Rank 0 has 128 real tokens but the split point is at 512 // 2 = 256, so its + second microbatch is pure padding. That microbatch does no work, which is + fine -- it still has to run so the expert all-to-all stays collective. + """ + desc, _ = _sync_dp([128, 512]) + assert desc.num_tokens == 512 + assert desc.num_ubatches == 2 + + +def test_all_padding_microbatch_has_no_work_to_do(): + """The microbatch past the last real token is well-formed and empty. + + It carries the last request with none of its query tokens, so attention and + sampling see nothing to do while the microbatch still runs. + """ + query_lens, seq_lens = [1] * 6, [64] * 6 + num_tokens_padded = 16 # split at 8, past all 6 real tokens + + buffers = _make_buffers() + input_batch = _make_input_batch( + query_lens, seq_lens, buffers, num_tokens_padded=num_tokens_padded + ) + ubatch_slices_padded = create_ubatch_slices(input_batch, num_ubatches=2) + ubatch_buffers = _make_ubatch_buffers() + + first = _slice_input_batch(input_batch, ubatch_slices_padded[0], *ubatch_buffers[0]) + last = _slice_input_batch(input_batch, ubatch_slices_padded[1], *ubatch_buffers[1]) + + # The first microbatch keeps every real request; the second gets none. + assert first.num_reqs == 6 + assert first.num_tokens == 6 + assert last.num_tokens == 0 + assert last.num_reqs == 1 + assert last.num_reqs_after_padding == 1 + assert last.num_tokens_after_padding == 8 + + # Zero query tokens for the one request it holds, so no attention work. + torch.testing.assert_close( + last.query_start_loc, torch.tensor([0, 0], dtype=torch.int32) + ) + assert last.num_scheduled_tokens.tolist() == [0] + # ...and its sequence still ends where its tokens do: they all live in the + # first microbatch, which computes them before this one runs. + torch.testing.assert_close(last.seq_lens, torch.tensor([64], dtype=torch.int32)) + torch.testing.assert_close( + last.seq_lens_cpu_upper_bound, torch.tensor([64], dtype=torch.int32) + ) + + +def test_microbatching_off_when_a_rank_does_not_allow_it(): + assert _sync_dp([256, 256], allow_ubatching=False)[0].num_ubatches == 1 + + +def test_slice_model_inputs_handles_mrope_positions(): + model_inputs = { + "input_ids": torch.arange(8), + # M-RoPE positions carry a leading section dim. + "positions": torch.arange(24).view(3, 8), + "inputs_embeds": None, + "intermediate_tensors": None, + } + sliced = slice_model_inputs(model_inputs, slice(4, 8)) + + torch.testing.assert_close(sliced["input_ids"], torch.arange(4, 8)) + torch.testing.assert_close(sliced["positions"], torch.arange(24).view(3, 8)[:, 4:8]) + assert sliced["inputs_embeds"] is None + assert sliced["intermediate_tensors"] is None + + +def _make_dbo_config() -> VllmConfig: + return VllmConfig( + model_config=ModelConfig(model="facebook/opt-125m", dtype="float16", seed=0), + parallel_config=ParallelConfig( + enable_dbo=True, all2all_backend="deepep_low_latency" + ), + ) + + +def _make_model_inputs(num_tokens: int, device: torch.device) -> dict[str, Any]: + return { + "input_ids": torch.arange(num_tokens, device=device), + "positions": torch.arange(num_tokens, device=device), + "inputs_embeds": None, + "intermediate_tensors": None, + } + + +@pytest.fixture(autouse=True) +def _no_comm_sm_reservation(monkeypatch): + """CI runs on MIG slices with fewer SMs than the default comm reservation.""" + monkeypatch.setenv("VLLM_DBO_COMM_SMS", "0") + + +def _make_execution_runner(vllm_config: VllmConfig) -> UBatchRunner: + """A runner for the execution tests below, which never call `prepare()`. + + The attention-side dependencies are only read while building per-microbatch + metadata, so they are left out here. + """ + return UBatchRunner( + vllm_config, + torch.device("cuda:0"), + model_state=cast(ModelState, None), + attn_groups=[], + kv_cache_config=cast(KVCacheConfig, None), + max_num_reqs=MAX_NUM_REQS, + ) + + +def _make_ubatch_state( + vllm_config: VllmConfig, ubatch_slices: list[UBatchSlice] +) -> UBatchState: + return UBatchState( + slices=ubatch_slices, + forward_contexts=[ + create_forward_context(None, vllm_config) for _ in ubatch_slices + ], + ) + + +def test_thresholds_below_the_microbatch_count_are_rejected(): + """A batch with fewer tokens than microbatches cannot be split at all. + + Nothing downstream copes with an unsplittable batch, so the thresholds are + what keep one out: a one-token decode split two ways leaves an empty + microbatch. + """ + with pytest.raises(ValueError, match="number of microbatches"): + ParallelConfig(enable_dbo=True, dbo_decode_token_threshold=1) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="DBO needs a GPU") +def test_runner_allocates_one_buffer_pair_per_microbatch(): + """All microbatches are live at once, so none may share rebase buffers.""" + vllm_config = _make_dbo_config() + runner = _make_execution_runner(vllm_config) + + for buffers in (runner.ubatch_query_start_loc, runner.ubatch_seq_lens): + assert len(buffers) == runner.num_ubatches + storages = {b.untyped_storage().data_ptr() for b in buffers} + assert len(storages) == runner.num_ubatches + + +class _YieldingModel(torch.nn.Module): + """Toy model that hands off to the other microbatch mid-forward.""" + + def __init__(self, trace: list[tuple[str, int]]): + super().__init__() + self.trace = trace + + def forward(self, input_ids, positions, intermediate_tensors=None, **kwargs): + self.trace.append(("enter", dbo_current_ubatch_id())) + out = input_ids.float().unsqueeze(-1) * 2 + positions.float().unsqueeze(-1) + # Stands in for the expert all-to-all handoff point. + dbo_yield() + self.trace.append(("exit", dbo_current_ubatch_id())) + return out + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="DBO needs a GPU") +def test_ubatch_runner_overlaps_and_matches_single_batch(): + """Microbatches interleave at the yield point and produce the same output.""" + vllm_config = _make_dbo_config() + device = torch.device("cuda:0") + runner = _make_execution_runner(vllm_config) + + model_inputs = _make_model_inputs(16, device) + ubatch_state = _make_ubatch_state( + vllm_config, + [ + UBatchSlice(slice(0, 4), slice(0, 8)), + UBatchSlice(slice(4, 8), slice(8, 16)), + ], + ) + + trace: list[tuple[str, int]] = [] + model = _YieldingModel(trace) + output = runner.run(model, model_inputs, ubatch_state) + + expected = model_inputs["input_ids"].float().unsqueeze(-1) * 2 + model_inputs[ + "positions" + ].float().unsqueeze(-1) + torch.testing.assert_close(output, expected) + + # Both microbatches reach the handoff before either finishes. + assert trace == [("enter", 0), ("enter", 1), ("exit", 0), ("exit", 1)] + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="DBO needs a GPU") +def test_ubatch_runner_names_the_microbatch_that_failed(): + """A failing microbatch surfaces as a named error, not a downstream KeyError. + + The model here never yields, so the microbatches run back to back and no + handoff is left outstanding when one of them dies. A microbatch that fails + *while its sibling is parked at a yield* hangs the step instead -- the + shared handoff protocol in `ubatching.py` has no way to unwind a parked + microbatch, and the V1 runner has the same gap. Fixing that means changing + `ubatching.py`, which is out of scope for the V2 runner. + """ + vllm_config = _make_dbo_config() + device = torch.device("cuda:0") + runner = _make_execution_runner(vllm_config) + + class _FailingModel(torch.nn.Module): + def forward(self, input_ids, positions, **kwargs): + if dbo_current_ubatch_id() == 0: + raise ValueError("boom") + return input_ids.float().unsqueeze(-1) + + model_inputs = _make_model_inputs(8, device) + ubatch_state = _make_ubatch_state( + vllm_config, + [ + UBatchSlice(slice(0, 2), slice(0, 4)), + UBatchSlice(slice(2, 4), slice(4, 8)), + ], + ) + + result: dict[str, BaseException] = {} + + def _call(): + try: + runner.run(_FailingModel(), model_inputs, ubatch_state) + except BaseException as e: # noqa: BLE001 + result["error"] = e + + # Run behind a watchdog: `UBatchRunner.run` joins without a timeout, so a + # regression here would wedge the suite rather than fail it. + caller = threading.Thread(target=_call, daemon=True) + caller.start() + caller.join(timeout=60.0) + assert not caller.is_alive(), "UBatchRunner.run hung on a failing microbatch" + + assert isinstance(result["error"], RuntimeError) + assert "Microbatch 0" in str(result["error"]) + assert isinstance(result["error"].__cause__, ValueError) diff --git a/vllm/config/parallel.py b/vllm/config/parallel.py index e7710c59ea..26b27622c8 100644 --- a/vllm/config/parallel.py +++ b/vllm/config/parallel.py @@ -1065,6 +1065,17 @@ class ParallelConfig: "Unable to use nsight profiling unless workers run with Ray." ) + # A batch below one token per microbatch cannot be split, so the + # thresholds have to keep it out rather than the split having to cope. + if self.use_ubatching and ( + min(self.dbo_decode_token_threshold, self.dbo_prefill_token_threshold) + < self.num_ubatches + ): + raise ValueError( + "dbo_decode_token_threshold and dbo_prefill_token_threshold must " + f"be at least the number of microbatches ({self.num_ubatches})." + ) + return self def reconfigure_for_independent_dp_rank(self) -> None: diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 413691eeb6..25620e0367 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -2596,8 +2596,8 @@ class VllmConfig: ): unsupported.append("EAGLE3 with pipeline parallelism") - if self.parallel_config.enable_dbo: - unsupported.append("dual batch overlap") + if self.parallel_config.use_ubatching: + unsupported.extend(self._get_dbo_unsupported_features()) if self.parallel_config.enable_elastic_ep: unsupported.append("elastic expert parallelism") @@ -2739,6 +2739,47 @@ class VllmConfig: f"{'; '.join(blockers)}." ) + def _get_dbo_unsupported_features(self) -> list[str]: + """Collect what the V2 model runner cannot combine with DBO. + + The V2 runner microbatches a plain decoder forward pass. Anything that + slices or replays the batch differently (drafting, adapters, pipeline + stages, context parallelism, encoders) is not handled yet. + """ + # TODO: DBO with model runner V2 is under development. + # It should be enabled with explicit VLLM_USE_V2_MODEL_RUNNER environ. + # Remove it when stable. + if envs.VLLM_USE_V2_MODEL_RUNNER is None: + return ["dual batch overlap"] + + unsupported: list[str] = [] + model_config = self.model_config + parallel_config = self.parallel_config + + if self.lora_config is not None: + unsupported.append("dual batch overlap with LoRA") + if self.speculative_config is not None: + unsupported.append("dual batch overlap with speculative decoding") + if parallel_config.pipeline_parallel_size > 1: + unsupported.append("dual batch overlap with pipeline parallelism") + if ( + parallel_config.decode_context_parallel_size > 1 + or parallel_config.prefill_context_parallel_size > 1 + ): + unsupported.append("dual batch overlap with context parallelism") + if model_config is not None and ( + model_config.is_multimodal_model or model_config.is_encoder_decoder + ): + unsupported.append("dual batch overlap with multimodal models") + if model_config is not None and model_config.is_hybrid: + unsupported.append("dual batch overlap with hybrid models") + if self.compilation_config.cudagraph_mode != CUDAGraphMode.NONE: + unsupported.append("dual batch overlap with CUDA graphs") + if self.is_mm_encoder_only: + unsupported.append("dual batch overlap with encoder only models") + + return unsupported + def _validate_v2_model_runner(self) -> None: """Check for features not yet supported by the V2 model runner.""" if not HAS_TRITON: diff --git a/vllm/model_executor/models/diffusion_gemma.py b/vllm/model_executor/models/diffusion_gemma.py index 3a51e5aad6..603d68410d 100644 --- a/vllm/model_executor/models/diffusion_gemma.py +++ b/vllm/model_executor/models/diffusion_gemma.py @@ -988,7 +988,9 @@ class DiffusionGemmaModelState(ModelState): attn_groups, kv_cache_config, for_capture=False, + ubatch_idx: int = 0, ) -> dict[str, Any]: + assert ubatch_idx == 0, "DBO is not supported" if cudagraph_mode == CUDAGraphMode.FULL: num_reqs = input_batch.num_reqs_after_padding num_tokens = input_batch.num_tokens_after_padding diff --git a/vllm/v1/worker/gpu/attn_utils.py b/vllm/v1/worker/gpu/attn_utils.py index 8a7043e54a..05949c25ac 100644 --- a/vllm/v1/worker/gpu/attn_utils.py +++ b/vllm/v1/worker/gpu/attn_utils.py @@ -22,6 +22,7 @@ from vllm.v1.kv_cache_interface import ( UniformTypeKVCacheSpecs, ) from vllm.v1.worker.gpu.model_states.interface import ModelSpecificAttnMetadata +from vllm.v1.worker.ubatch_utils import get_num_ubatches from vllm.v1.worker.utils import ( AttentionGroup, add_kv_sharing_layers_to_kv_cache_groups, @@ -140,14 +141,19 @@ def init_attn_backend( vllm_config=vllm_config, device=device, kernel_block_size=kernel_block_size, - num_metadata_builders=1, + # Microbatches build attention metadata concurrently, and some + # builders keep the prepared metadata on themselves (MLA stores + # it on the prefill backend), so each ubatch needs its own. + num_metadata_builders=get_num_ubatches(vllm_config.parallel_config), ) - builder = group.get_metadata_builder(0) - if attn_backend_workspace is None: - if hasattr(builder, "_get_workspace_buffer"): - attn_backend_workspace = builder._get_workspace_buffer() - else: - if hasattr(builder, "set_workspace_buffer"): + # The microbatches' builders share the workspace: they all issue + # attention on the one compute stream the threads hand off, so the + # buffer is written serially, as it already is across steps. + for builder in group.metadata_builders: + if attn_backend_workspace is None: + if hasattr(builder, "_get_workspace_buffer"): + attn_backend_workspace = builder._get_workspace_buffer() + elif hasattr(builder, "set_workspace_buffer"): builder.set_workspace_buffer(attn_backend_workspace) attn_cg_support_info = get_attn_cg_support(attn_groups, vllm_config) return attn_groups, attn_cg_support_info, kernel_block_sizes @@ -271,6 +277,7 @@ def build_attn_metadata( for_cudagraph_capture: bool = False, causal: bool | torch.Tensor | Mapping[int, bool] = True, rswa_prefix_lens: torch.Tensor | None = None, + ubatch_idx: int = 0, ) -> dict[str, Any]: seq_lens = seq_lens[:num_reqs] if dcp_local_seq_lens is not None: @@ -319,7 +326,7 @@ def build_attn_metadata( ) for attn_group in attn_groups[i]: - attn_metadata_builder = attn_group.get_metadata_builder(0) + attn_metadata_builder = attn_group.get_metadata_builder(ubatch_idx) if for_cudagraph_capture: metadata = attn_metadata_builder.build_for_cudagraph_capture( common_attn_metadata diff --git a/vllm/v1/worker/gpu/cudagraph_utils.py b/vllm/v1/worker/gpu/cudagraph_utils.py index 291863779c..084429b3f8 100644 --- a/vllm/v1/worker/gpu/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/cudagraph_utils.py @@ -68,6 +68,8 @@ class BatchExecutionDescriptor: # uniform_token_count unset, so this is what keeps a prefill batch out of one. max_query_len: int | None = None num_active_loras: int = 0 + # Number of microbatches the batch is split into (DBO). 1 means no splitting. + num_ubatches: int = 1 def make_cudagraph_stats( @@ -100,11 +102,13 @@ def _is_compatible( uniform_token_count: int | None, num_active_loras: int, max_query_len: int | None, + num_ubatches: int, ) -> bool: # desc.uniform_token_count=None (PIECEWISE) can handle any uniform_token_count # desc.num_reqs=None means no request padding needed (PIECEWISE) # desc.max_query_len=None means the graph does not constrain query length; a # caller that does not track max_query_len must not match one that does + # A graph captured for N microbatches can only serve a batch split N ways. return ( ( desc.uniform_token_count is None @@ -117,6 +121,7 @@ def _is_compatible( and (desc.num_reqs is None or desc.num_reqs >= num_reqs) and desc.num_tokens >= num_tokens and desc.num_active_loras == num_active_loras + and desc.num_ubatches == num_ubatches ) @@ -434,6 +439,7 @@ class CudaGraphManager: uniform_token_count: int | None, num_active_loras: int, max_query_len: int | None = None, + num_ubatches: int = 1, ) -> BatchExecutionDescriptor: """Find matching cudagraph descriptor from priority-ordered candidates.""" @@ -448,6 +454,7 @@ class CudaGraphManager: uniform_token_count, effective_loras, max_query_len, + num_ubatches, ): return desc return BatchExecutionDescriptor( @@ -455,6 +462,7 @@ class CudaGraphManager: num_tokens=num_tokens, num_reqs=num_reqs, num_active_loras=effective_loras, + num_ubatches=num_ubatches, ) def run_fullgraph(self, desc: BatchExecutionDescriptor): diff --git a/vllm/v1/worker/gpu/dp_utils.py b/vllm/v1/worker/gpu/dp_utils.py index d720d50ca0..dddf630cc3 100644 --- a/vllm/v1/worker/gpu/dp_utils.py +++ b/vllm/v1/worker/gpu/dp_utils.py @@ -7,12 +7,14 @@ from dataclasses import dataclass, replace import torch import torch.distributed as dist +from vllm.config import ParallelConfig from vllm.config.compilation import CUDAGraphMode from vllm.distributed.parallel_state import get_dp_group from vllm.v1.worker.gpu.cudagraph_utils import ( BatchExecutionDescriptor, CudaGraphManager, ) +from vllm.v1.worker.ubatch_utils import check_ubatch_thresholds, get_num_ubatches @dataclass(frozen=True) @@ -48,27 +50,35 @@ def sync_cudagraph_and_dp_padding( dp_rank: int, max_query_len: int | None = None, num_active_loras: int = 0, + parallel_config: ParallelConfig | None = None, + allow_ubatching: bool = False, + uniform_decode: bool = False, ) -> tuple[BatchExecutionDescriptor, DPSyncState | None]: """ Coordinates the batch descriptor and DP padding across all ranks. + `parallel_config` is only needed to decide whether to microbatch, so callers + that never do (`allow_ubatching=False`) can leave it out. + Returns (synced_batch_desc, sync). `sync` is None when no rank has work. """ assert dp_size > 1, "DP size must be greater than 1" group = get_dp_group().cpu_group - tensor = torch.zeros(5, dp_size, dtype=torch.int32, device="cpu") + tensor = torch.zeros(6, dp_size, dtype=torch.int32, device="cpu") tensor[0][dp_rank] = num_tokens tensor[1][dp_rank] = desired_batch_desc.cg_mode.value tensor[2][dp_rank] = uniform_token_count or 0 # (0 means None) tensor[3][dp_rank] = max_query_len or -1 # (-1 means None) - tensor[4][dp_rank] = num_reqs + tensor[4][dp_rank] = int(allow_ubatching) + tensor[5][dp_rank] = num_reqs dist.all_reduce(tensor, group=group) num_tokens_across_dp = tensor[0] cg_mode_across_dp = tensor[1] uniform_token_counts_across_dp = tensor[2] max_query_lens_across_dp = tensor[3] - num_reqs_across_dp = tensor[4] + allow_ubatching_across_dp = tensor[4] + num_reqs_across_dp = tensor[5] # If ranks disagree on the uniform token count, or its 0 (means None) set to None synced_uniform_token_count: int | None = int(uniform_token_counts_across_dp[0]) @@ -83,6 +93,40 @@ def sync_cudagraph_and_dp_padding( ) return synced_desc, None + if torch.all(allow_ubatching_across_dp == 1): + # This rank voted too, so its caller passed a config. + assert parallel_config is not None + # A uniform decode only if every rank runs the same uniform query + # length, so every rank reaches the same answer here. + uniform_decode_across_dp = uniform_decode and torch.all( + uniform_token_counts_across_dp == (uniform_token_count or 0) + ) + if check_ubatch_thresholds( + parallel_config, + # Thresholds only grow with the token count, so holding the smallest + # rank to them holds every rank to them. + int(num_tokens_across_dp.min()), + uniform_decode=uniform_decode_across_dp, + ): + # Microbatching is all-or-nothing: the expert all-to-all is + # collective, so every rank splits and pads to the same token count. + # A rank too small to fill a microbatch does no work in it, like a + # dummy run. Microbatched steps run eager; nothing is captured yet. + ubatch_num_tokens = int(num_tokens_across_dp.max()) + return BatchExecutionDescriptor( + cg_mode=CUDAGraphMode.NONE, + num_tokens=ubatch_num_tokens, + num_reqs=num_reqs, + num_ubatches=get_num_ubatches(parallel_config), + ), DPSyncState( + num_tokens_across_dp=torch.full_like( + num_tokens_across_dp, ubatch_num_tokens + ), + uniform_token_count=synced_uniform_token_count, + eager=True, + num_reqs=int(num_reqs_across_dp.max()), + ) + synced_cg_mode = CUDAGraphMode(int(cg_mode_across_dp.min().item())) # If any rank wants to run eager, all ranks run eager @@ -98,7 +142,7 @@ def sync_cudagraph_and_dp_padding( num_tokens_across_dp=num_tokens_across_dp, uniform_token_count=synced_uniform_token_count, eager=True, - num_reqs=int(num_reqs_across_dp.max().item()), + num_reqs=int(num_reqs_across_dp.max()), ), ) @@ -136,7 +180,7 @@ def sync_cudagraph_and_dp_padding( synced_desc.num_reqs if synced_desc.cg_mode == CUDAGraphMode.FULL and synced_desc.num_reqs is not None - else int(num_reqs_across_dp.max().item()) + else int(num_reqs_across_dp.max()) ), ) @@ -151,6 +195,9 @@ def dispatch_cg_and_sync_dp( max_query_len: int | None = None, need_eager: bool = False, num_active_loras: int = 0, + parallel_config: ParallelConfig | None = None, + allow_ubatching: bool = False, + uniform_decode: bool = False, dp_sync: DPSyncState | None = None, ) -> tuple[BatchExecutionDescriptor, DPSyncState | None]: """Pick a cudagraph descriptor for this batch, agreeing it across DP ranks. @@ -239,4 +286,7 @@ def dispatch_cg_and_sync_dp( dp_rank, max_query_len=max_query_len, num_active_loras=num_active_loras, + parallel_config=parallel_config, + allow_ubatching=allow_ubatching, + uniform_decode=uniform_decode, ) diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 15005338d4..901bd68754 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -161,6 +161,11 @@ from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator from vllm.v1.worker.gpu.spec_decode.utils import DraftTokensHandler from vllm.v1.worker.gpu.states import RequestState from vllm.v1.worker.gpu.structured_outputs import StructuredOutputsWorker +from vllm.v1.worker.gpu.ubatch_utils import ( + UBatchRunner, + UBatchState, + maybe_build_ubatch_runner, +) from vllm.v1.worker.lora_model_runner_mixin import LoRAModelRunnerMixin from vllm.v1.worker.utils import ( KVBlockZeroer, @@ -230,6 +235,10 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.dp_size = self.parallel_config.data_parallel_size self.dp_rank = self.parallel_config.data_parallel_rank + # Dual batch overlap. Created in initialize_kv_cache(), once everything + # it runs the microbatched forward with exists. + self.ubatch_runner: UBatchRunner | None = None + # Detect EP all2all peer faults to prevent emitting corrupted output. # Only meaningful for MoE + DP with an FT-capable all2all backend. self.check_ep_fault = False @@ -634,6 +643,14 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.block_tables, cls=self.pcp_manager_cls, ) + self.ubatch_runner = maybe_build_ubatch_runner( + self.vllm_config, + self.device, + self.model_state, + self.attn_groups, + self.kv_cache_config, + self.max_num_reqs, + ) initialize_mamba_ssu_backend( self.vllm_config.mamba_config, self.kv_cache_config, @@ -1597,6 +1614,11 @@ class GPUModelRunner(LoRAModelRunnerMixin): max_query_len=max_query_len, need_eager=is_profile or skip_compiled, num_active_loras=num_active_loras, + parallel_config=self.parallel_config, + allow_ubatching=( + self.ubatch_runner is not None and not skip_attn_for_dummy_run + ), + uniform_decode=uniform_tok_count == self.decode_query_len, ) if batch_desc.num_tokens == 0: @@ -1665,7 +1687,14 @@ class GPUModelRunner(LoRAModelRunnerMixin): attn_metadata = None slot_mappings_by_layer = None - if not (dummy_run and skip_attn_for_dummy_run): + ubatch_state: UBatchState | None = None + if batch_desc.num_ubatches > 1: + assert self.ubatch_runner is not None + assert block_tables is not None and slot_mappings is not None + ubatch_state = self.ubatch_runner.prepare( + input_batch, block_tables, slot_mappings + ) + elif not (dummy_run and skip_attn_for_dummy_run): assert slot_mappings is not None slot_mappings_by_layer = build_slot_mappings_by_layer( slot_mappings, self.kv_cache_config @@ -1751,7 +1780,10 @@ class GPUModelRunner(LoRAModelRunnerMixin): del intermediate_tensors # Update the EPLB meta. - self.eplb.prepare_forward(self.model_config, input_batch.num_tokens) + ubatch_slices = ubatch_state.slices if ubatch_state is not None else None + self.eplb.prepare_forward( + self.model_config, input_batch.num_tokens, ubatch_slices + ) self.step_timing.record_batch( input_batch, batch_desc.cg_mode == CUDAGraphMode.FULL @@ -1783,12 +1815,18 @@ class GPUModelRunner(LoRAModelRunnerMixin): dp_sync.num_tokens_across_dp if dp_sync is not None else None ), batch_descriptor=batch_descriptor, + ubatch_slices=ubatch_slices, slot_mapping=slot_mappings_by_layer, skip_compiled=skip_compiled, is_padding=input_batch.is_padding, ): self.kv_connector.pre_forward(scheduler_output) - if batch_desc.cg_mode == CUDAGraphMode.PIECEWISE: + if ubatch_state is not None: + assert self.ubatch_runner is not None + model_output = self.ubatch_runner.run( + self.model, model_inputs, ubatch_state + ) + elif batch_desc.cg_mode == CUDAGraphMode.PIECEWISE: # Run the PIECEWISE graph (compiled PW cudagraph or breakable # cudagraph, chosen inside run_pw_graph). cg_mode is only # PIECEWISE after the cudagraph manager exists. diff --git a/vllm/v1/worker/gpu/model_states/default.py b/vllm/v1/worker/gpu/model_states/default.py index e1b6d7d3de..7d42745d10 100644 --- a/vllm/v1/worker/gpu/model_states/default.py +++ b/vllm/v1/worker/gpu/model_states/default.py @@ -172,6 +172,7 @@ class DefaultModelState(ModelState): attn_groups: list[list[AttentionGroup]], kv_cache_config: KVCacheConfig, for_capture: bool = False, + ubatch_idx: int = 0, ) -> dict[str, Any]: if cudagraph_mode == CUDAGraphMode.FULL: # Use padded sizes - padding is handled by model_runner.prepare_attn. @@ -224,5 +225,6 @@ class DefaultModelState(ModelState): mm_req_doc_ranges=req_doc_ranges, for_cudagraph_capture=for_capture, rswa_prefix_lens=input_batch.prompt_lens, + ubatch_idx=ubatch_idx, ) return attn_metadata diff --git a/vllm/v1/worker/gpu/model_states/encoder_decoder.py b/vllm/v1/worker/gpu/model_states/encoder_decoder.py index 5bd069312e..a61384ed7f 100644 --- a/vllm/v1/worker/gpu/model_states/encoder_decoder.py +++ b/vllm/v1/worker/gpu/model_states/encoder_decoder.py @@ -116,7 +116,9 @@ class EncoderDecoderModelState(ModelState): attn_groups: list[list[AttentionGroup]], kv_cache_config: KVCacheConfig, for_capture: bool = False, + ubatch_idx: int = 0, ) -> dict[str, Any]: + assert ubatch_idx == 0, "DBO is not supported" if cudagraph_mode == CUDAGraphMode.FULL: num_reqs = input_batch.num_reqs_after_padding num_tokens = input_batch.num_tokens_after_padding diff --git a/vllm/v1/worker/gpu/model_states/encoder_only.py b/vllm/v1/worker/gpu/model_states/encoder_only.py index 10ba0a6a39..32a70e7331 100644 --- a/vllm/v1/worker/gpu/model_states/encoder_only.py +++ b/vllm/v1/worker/gpu/model_states/encoder_only.py @@ -167,7 +167,9 @@ class EncoderOnlyModelState(DefaultModelState): attn_groups: list[list[AttentionGroup]], kv_cache_config: KVCacheConfig, for_capture: bool = False, + ubatch_idx: int = 0, ) -> dict[str, Any]: + assert ubatch_idx == 0, "DBO is not supported" attn_metadata = super().prepare_attn( input_batch, cudagraph_mode, diff --git a/vllm/v1/worker/gpu/model_states/interface.py b/vllm/v1/worker/gpu/model_states/interface.py index cd1b5b64da..a5c8d6f4cf 100644 --- a/vllm/v1/worker/gpu/model_states/interface.py +++ b/vllm/v1/worker/gpu/model_states/interface.py @@ -221,6 +221,7 @@ class ModelState(ABC): attn_groups: list[list[AttentionGroup]], kv_cache_config: KVCacheConfig, for_capture: bool = False, + ubatch_idx: int = 0, ) -> dict[str, Any]: raise NotImplementedError diff --git a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py index 9564120f23..0ac3f2f63b 100644 --- a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py +++ b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py @@ -238,7 +238,9 @@ class MambaHybridModelState(DefaultModelState): attn_groups: list[list[AttentionGroup]], kv_cache_config: KVCacheConfig, for_capture: bool = False, + ubatch_idx: int = 0, ) -> dict[str, Any]: + assert ubatch_idx == 0, "DBO is not supported" if cudagraph_mode == CUDAGraphMode.FULL: num_reqs = input_batch.num_reqs_after_padding num_tokens = input_batch.num_tokens_after_padding diff --git a/vllm/v1/worker/gpu/ubatch_utils.py b/vllm/v1/worker/gpu/ubatch_utils.py new file mode 100644 index 0000000000..350805707d --- /dev/null +++ b/vllm/v1/worker/gpu/ubatch_utils.py @@ -0,0 +1,473 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Microbatching (DBO) helpers for the V2 GPU model runner.""" + +import threading +from dataclasses import replace +from typing import Any, NamedTuple + +import numpy as np +import torch + +from vllm.config import CUDAGraphMode, VllmConfig +from vllm.forward_context import ( + DPMetadata, + ForwardContext, + create_forward_context, + override_forward_context, +) +from vllm.logger import init_logger +from vllm.sequence import IntermediateTensors +from vllm.utils.torch_utils import current_stream +from vllm.v1.kv_cache_interface import KVCacheConfig +from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer +from vllm.v1.worker.gpu.input_batch import InputBatch +from vllm.v1.worker.gpu.model_states.interface import ModelState +from vllm.v1.worker.ubatch_utils import ( + UBatchSlice, + UBatchSlices, + create_sm_control_context, + maybe_create_ubatch_slices, +) +from vllm.v1.worker.ubatching import make_ubatch_contexts +from vllm.v1.worker.utils import AttentionGroup + +logger = init_logger(__name__) + + +class UBatchState(NamedTuple): + """Per-microbatch inputs for one dual-batch-overlap step.""" + + slices: UBatchSlices + forward_contexts: list[ForwardContext] + + +def create_ubatch_slices(input_batch: InputBatch, num_ubatches: int) -> UBatchSlices: + """Split a DP-padded batch into the slices its microbatches run on. + + Splitting at the midpoint of the DP-padded token count leaves the trailing + microbatch anywhere from full to entirely inside the padding. Clamping its + request slice onto the last request leaves it holding that request with + zero query tokens -- the shape a straddling request already takes -- so it + stays well-formed with no work to do, like a dummy run. + """ + _, ubatch_slices = maybe_create_ubatch_slices( + True, + input_batch.num_scheduled_tokens, + input_batch.num_tokens_after_padding, + input_batch.num_reqs_after_padding, + num_ubatches, + ) + assert ubatch_slices is not None + return [ + UBatchSlice( + slice( + min(s.request_slice.start, input_batch.num_reqs - 1), + min(s.request_slice.stop, input_batch.num_reqs_after_padding), + ), + s.token_slice, + ) + for s in ubatch_slices + ] + + +def _slice_input_batch( + input_batch: InputBatch, + ubatch_slice: UBatchSlice, + query_start_loc_buf: torch.Tensor, + seq_lens_buf: torch.Tensor, +) -> InputBatch: + """Build the sub-`InputBatch` a single microbatch runs on. + + A request straddling the boundary appears in both microbatches, its query + truncated to the tokens each one owns. + + Everything except `query_start_loc` and `seq_lens` is a view of the full + batch's buffers, so slicing costs no allocation. Those two are rebased onto + the microbatch's token range and written to the caller's buffers, which + keeps them valid under CUDA graph capture and replay. + + The sub-batch describes the forward pass only. Sampling runs once over the + merged batch, so `logits_indices`, `cu_num_logits` and the draft-token + fields are carried over unchanged and must not be read off a sub-batch. + """ + assert not ubatch_slice.is_empty(), f"Ubatch slice {ubatch_slice} is empty" + + req_start = ubatch_slice.request_slice.start + req_stop = ubatch_slice.request_slice.stop + tok_start = ubatch_slice.token_slice.start + tok_stop = ubatch_slice.token_slice.stop + + num_reqs_padded = req_stop - req_start + num_tokens_padded = tok_stop - tok_start + # The trailing microbatch may extend past the real batch into CUDA graph + # padding; the unpadded counts stop at the last real request/token. + num_reqs = max(0, min(req_stop, input_batch.num_reqs) - req_start) + num_tokens = max(0, min(tok_stop, input_batch.num_tokens) - tok_start) + + query_start_loc = _slice_query_start_loc( + input_batch.query_start_loc, + query_start_loc_buf[: num_reqs_padded + 1], + req_start, + req_stop, + tok_start, + num_tokens_padded, + ) + query_start_loc_np = np.clip( + input_batch.query_start_loc_np[req_start : req_stop + 1] - tok_start, + 0, + num_tokens_padded, + ).astype(np.int32) + + seq_lens = _slice_seq_lens( + input_batch.seq_lens, + seq_lens_buf[:num_reqs_padded], + input_batch.query_start_loc, + req_start, + req_stop, + tok_stop, + ) + # Same truncation as `_slice_seq_lens`, on the host-side upper bound. + seq_lens_cpu_upper_bound = input_batch.seq_lens_cpu_upper_bound[ + req_start:req_stop + ].clone() + tokens_truncated = max(0, int(input_batch.query_start_loc_np[req_stop]) - tok_stop) + if tokens_truncated: + seq_lens_cpu_upper_bound[-1] -= tokens_truncated + + # Query lengths of the truncated requests, so consumers that derive + # max_query_len from this array see the microbatch's own lengths. + num_scheduled_tokens = np.diff(query_start_loc_np)[:num_reqs] + + dcp_local_seq_lens = input_batch.dcp_local_seq_lens + if dcp_local_seq_lens is not None: + # NOTE: a request split across microbatches keeps its full local + # seq_len here. DCP is not yet validated with DBO. + dcp_local_seq_lens = dcp_local_seq_lens[req_start:req_stop] + + return replace( + input_batch, + req_ids=input_batch.req_ids[req_start : min(req_stop, input_batch.num_reqs)], + num_reqs=num_reqs, + num_reqs_after_padding=num_reqs_padded, + idx_mapping=input_batch.idx_mapping[req_start:req_stop], + idx_mapping_np=input_batch.idx_mapping_np[req_start:req_stop], + num_scheduled_tokens=num_scheduled_tokens, + num_tokens=num_tokens, + num_tokens_after_padding=num_tokens_padded, + query_start_loc=query_start_loc, + query_start_loc_np=query_start_loc_np, + seq_lens=seq_lens, + seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound, + dcp_local_seq_lens=dcp_local_seq_lens, + num_computed_tokens_np=input_batch.num_computed_tokens_np[req_start:req_stop], + prefill_len_np=input_batch.prefill_len_np[req_start:req_stop], + num_computed_prefill_tokens_np=input_batch.num_computed_prefill_tokens_np[ + req_start:req_stop + ], + is_prefilling_np=input_batch.is_prefilling_np[req_start:req_stop], + input_ids=input_batch.input_ids[tok_start:tok_stop], + positions=input_batch.positions[tok_start:tok_stop], + is_padding=input_batch.is_padding[tok_start:tok_stop], + prompt_lens=( + None + if input_batch.prompt_lens is None + else input_batch.prompt_lens[req_start:req_stop] + ), + ) + + +def _slice_query_start_loc( + query_start_loc: torch.Tensor, + out: torch.Tensor, + req_start: int, + req_stop: int, + tok_start: int, + num_tokens_padded: int, +) -> torch.Tensor: + """Rebase query_start_loc onto the microbatch's token range. + + The clamp truncates the requests straddling either boundary to the tokens + this microbatch owns. + """ + torch.sub(query_start_loc[req_start : req_stop + 1], tok_start, out=out) + return out.clamp_(0, num_tokens_padded) + + +def _slice_seq_lens( + seq_lens: torch.Tensor, + out: torch.Tensor, + query_start_loc: torch.Tensor, + req_start: int, + req_stop: int, + tok_stop: int, +) -> torch.Tensor: + """Slice seq_lens, shortening the request truncated at the boundary. + + Only tokens past the end of this microbatch are missing; tokens it owns in + an *earlier* microbatch are computed before this one reads them, so they + still count. Deriving the truncation from this microbatch's own query + lengths would subtract them twice. It is computed from tensors, not Python + ints, so this stays valid inside a CUDA graph. + """ + out.copy_(seq_lens[req_start:req_stop]) + last = req_stop - req_start - 1 + out[last] -= (query_start_loc[req_stop] - tok_stop).clamp_(min=0) + return out + + +def slice_model_inputs( + model_inputs: dict[str, Any], token_slice: slice +) -> dict[str, Any]: + """Narrow the model's per-token inputs to one microbatch.""" + sliced = dict(model_inputs) + for key in ("input_ids", "inputs_embeds"): + value = model_inputs.get(key) + if value is not None: + sliced[key] = value[token_slice] + + positions = model_inputs["positions"] + # M-RoPE carries a leading section dim. + sliced["positions"] = ( + positions[:, token_slice] if positions.ndim == 2 else positions[token_slice] + ) + + intermediate_tensors = model_inputs.get("intermediate_tensors") + if intermediate_tensors is not None: + sliced["intermediate_tensors"] = intermediate_tensors[token_slice] + return sliced + + +def merge_ubatch_outputs(outputs: list[Any]) -> Any: + """Reassemble the full-batch model output from the per-microbatch ones. + + Models return hidden states, a tuple of those plus auxiliary hidden states + (EAGLE3), or IntermediateTensors on non-final pipeline ranks. + """ + first = outputs[0] + if isinstance(first, IntermediateTensors): + return IntermediateTensors( + { + key: torch.cat([out.tensors[key] for out in outputs], dim=0) + for key in first.tensors + } + ) + if isinstance(first, tuple): + return tuple(torch.cat(parts, dim=0) for parts in zip(*outputs)) + return torch.cat(outputs, dim=0) + + +class UBatchRunner: + """Runs the model on two or more microbatches that overlap each other. + + Each microbatch gets a thread and its own forward context. Only one thread + holds the GPU at a time; they hand off to each other at the communication + points inside the model (see `vllm.v1.worker.ubatching`), so one + microbatch's expert all-to-all overlaps the other's compute. + """ + + def __init__( + self, + vllm_config: VllmConfig, + device: torch.device, + model_state: ModelState, + attn_groups: list[list[AttentionGroup]], + kv_cache_config: KVCacheConfig, + max_num_reqs: int, + ): + self.vllm_config = vllm_config + self.parallel_config = vllm_config.parallel_config + self.num_ubatches = self.parallel_config.num_ubatches + self.device = device + self.model_state = model_state + self.attn_groups = attn_groups + self.kv_cache_config = kv_cache_config + # `query_start_loc` and `seq_lens` are rebased onto each microbatch's + # own token range, so they cannot be views of the full batch's buffers. + # Allocating up front keeps their addresses stable across replays. + self.ubatch_query_start_loc = [ + torch.zeros(max_num_reqs + 1, dtype=torch.int32, device=device) + for _ in range(self.num_ubatches) + ] + self.ubatch_seq_lens = [ + torch.zeros(max_num_reqs, dtype=torch.int32, device=device) + for _ in range(self.num_ubatches) + ] + self.comm_stream = torch.cuda.Stream(device=device) + # The microbatch threads plus the thread that starts them. + self.ready_barrier = threading.Barrier(self.num_ubatches + 1) + self.sm_control = create_sm_control_context(self.parallel_config) + + def prepare( + self, + input_batch: InputBatch, + block_tables: tuple[torch.Tensor, ...], + slot_mappings: torch.Tensor, + ) -> UBatchState: + """Split the batch into the microbatches the step will run on. + + Attention metadata is built per microbatch and carried in the forward + contexts the threads install, not in the caller's. + """ + ubatch_slices = create_ubatch_slices(input_batch, self.num_ubatches) + + attn_metadata = [] + slot_mappings_by_layer = [] + is_padding = [] + for i, ubatch_slice in enumerate(ubatch_slices): + ubatch = _slice_input_batch( + input_batch, + ubatch_slice, + self.ubatch_query_start_loc[i], + self.ubatch_seq_lens[i], + ) + ubatch_slot_mappings = slot_mappings[:, ubatch_slice.token_slice] + ubatch_block_tables = tuple( + block_table[ubatch_slice.request_slice] for block_table in block_tables + ) + attn_metadata.append( + self.model_state.prepare_attn( + ubatch, + CUDAGraphMode.NONE, + ubatch_block_tables, + ubatch_slot_mappings, + self.attn_groups, + self.kv_cache_config, + ubatch_idx=i, + ) + ) + slot_mappings_by_layer.append( + build_slot_mappings_by_layer(ubatch_slot_mappings, self.kv_cache_config) + ) + is_padding.append(ubatch.is_padding) + + return UBatchState( + slices=ubatch_slices, + forward_contexts=self._make_forward_contexts( + ubatch_slices, attn_metadata, slot_mappings_by_layer, is_padding + ), + ) + + def _make_forward_contexts( + self, + ubatch_slices: UBatchSlices, + attn_metadata: list[dict[str, Any]], + slot_mappings_by_layer: list[dict[str, torch.Tensor]], + is_padding: list[torch.Tensor | None], + ) -> list[ForwardContext]: + """Build one forward context per microbatch. + + The expert all-to-all runs once per microbatch, so each context's DP + metadata carries that microbatch's token count. Microbatching pads every + DP rank alike, so each rank can assume that count for the others too. + """ + dp_size = self.parallel_config.data_parallel_size + forward_contexts = [] + for i, ubatch_slice in enumerate(ubatch_slices): + num_tokens = ubatch_slice.num_tokens + num_tokens_across_dp = torch.full( + (dp_size,), num_tokens, dtype=torch.int32, device="cpu" + ) + forward_contexts.append( + create_forward_context( + attn_metadata[i], + self.vllm_config, + dp_metadata=DPMetadata.make( + self.parallel_config, num_tokens, num_tokens_across_dp + ), + cudagraph_runtime_mode=CUDAGraphMode.NONE, + slot_mapping=slot_mappings_by_layer[i], + is_padding=is_padding[i], + ) + ) + return forward_contexts + + def run( + self, + model: Any, + model_inputs: dict[str, Any], + ubatch_state: UBatchState, + ) -> Any: + ubatch_slices = ubatch_state.slices + assert len(ubatch_slices) == len(ubatch_state.forward_contexts) + assert len(ubatch_slices) == self.num_ubatches + + ubatch_contexts = make_ubatch_contexts( + num_micro_batches=self.num_ubatches, + comm_stream=self.comm_stream, + compute_stream=current_stream(), + forward_contexts=ubatch_state.forward_contexts, + ready_barrier=self.ready_barrier, + ) + + outputs: dict[int, Any] = {} + errors: dict[int, BaseException] = {} + + @torch.inference_mode() + def run_ubatch(ubatch_context, inputs: dict[str, Any]) -> None: + try: + with ubatch_context: + outputs[ubatch_context.id] = model(**inputs) + except BaseException as e: # noqa: BLE001 + # Recorded so the join below can report which microbatch broke. + # Note the siblings are not told to unwind: microbatches hand + # control to each other, so one that dies mid-forward leaves + # the others parked on a handoff that never comes and the step + # hangs. The V1 runner behaves the same way; fixing it means + # changing the shared handoff protocol in ubatching.py, which + # is deliberately out of scope here. + errors[ubatch_context.id] = e + + # The threads manage the forward context themselves; clear it here so + # it is restored correctly once they are done. + with override_forward_context(None), self.sm_control: + threads = [] + for ubatch_context, ubatch_slice in zip(ubatch_contexts, ubatch_slices): + thread = threading.Thread( + target=run_ubatch, + args=( + ubatch_context, + slice_model_inputs(model_inputs, ubatch_slice.token_slice), + ), + ) + threads.append(thread) + thread.start() + + # Wait for every thread to reach its context, then start the first. + self.ready_barrier.wait() + ubatch_contexts[0].cpu_wait_event.set() + for thread in threads: + thread.join() + + if errors: + failed = min(errors) + raise RuntimeError( + f"Microbatch {failed} of {self.num_ubatches} failed" + ) from errors[failed] + return merge_ubatch_outputs([outputs[i] for i in range(self.num_ubatches)]) + + +def maybe_build_ubatch_runner( + vllm_config: VllmConfig, + device: torch.device, + model_state: ModelState, + attn_groups: list[list[AttentionGroup]], + kv_cache_config: KVCacheConfig, + max_num_reqs: int, +) -> UBatchRunner | None: + """Build the microbatch runner, or None when DBO is not in use. + + Microbatching needs the DP handshake to agree on it, so it is only + available with more than one DP rank (as in the V1 runner). + """ + parallel_config = vllm_config.parallel_config + if not parallel_config.use_ubatching or parallel_config.data_parallel_size <= 1: + return None + + logger.info_once( + "Dual batch overlap is enabled. Microbatched steps run without " + "CUDA graphs on the V2 model runner." + ) + return UBatchRunner( + vllm_config, device, model_state, attn_groups, kv_cache_config, max_num_reqs + ) diff --git a/vllm/v1/worker/gpu_ubatch_wrapper.py b/vllm/v1/worker/gpu_ubatch_wrapper.py index 7b61999843..0735b39f81 100644 --- a/vllm/v1/worker/gpu_ubatch_wrapper.py +++ b/vllm/v1/worker/gpu_ubatch_wrapper.py @@ -11,7 +11,6 @@ import torch import vllm.envs as envs from vllm.compilation.cuda_graph import CUDAGraphWrapper from vllm.config import CUDAGraphMode, VllmConfig -from vllm.distributed import get_ep_group from vllm.distributed.device_communicators.pynccl_allocator import set_graph_pool_id from vllm.forward_context import ( DPMetadata, @@ -23,9 +22,7 @@ from vllm.logger import init_logger from vllm.model_executor.offloader.base import get_offloader from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors -from vllm.utils.deep_gemm import set_num_sms as deep_gemm_set_num_sms -from vllm.utils.import_utils import has_deep_gemm -from vllm.utils.platform_utils import num_compute_units +from vllm.v1.worker.ubatch_utils import create_sm_control_context from vllm.v1.worker.ubatching import UBatchContext, make_ubatch_contexts logger = init_logger(__name__) @@ -65,51 +62,6 @@ class CUDAGraphMetaData: outputs: Any | None = None -class SMControlContextManager: - def __init__( - self, - comm_sms: int, - set_comm_sms: Callable[[int], None], - set_compute_sms: Callable[[int], None], - ): - """ - Context manager for controlling SM (Streaming Multiprocessor) - allocation. Upon entering the context, it sets the number of SMs - allocated for communication and computation to comm_sms and - total_sms - comm_sms respectively. Upon exiting, it restores the - allocation to use all available SMs (i.e. total_sms). - - Args: - comm_sms (int): The number of SMs to allocate for communication. - (The remainder will be used for computation.) - set_comm_sms (Callable[[int], None]): - A function that sets the number of SMs for communication. - set_compute_sms (Callable[[int], None]): - A function that sets the number of SMs for computation. - """ - - assert current_platform.is_cuda() or current_platform.is_rocm(), ( - "SM/CU control is supported on CUDA and ROCm platforms" - ) - device = torch.accelerator.current_device_index() - total_sms = num_compute_units(device) - - assert comm_sms < total_sms - self.total_sms = total_sms - self.compute_sms = total_sms - comm_sms - self.comm_sms = comm_sms - self.set_comm_sms = set_comm_sms - self.set_compute_sms = set_compute_sms - - def __enter__(self): - self.set_comm_sms(self.comm_sms) - self.set_compute_sms(self.compute_sms) - - def __exit__(self, exc_type, exc_value, traceback): - self.set_comm_sms(self.total_sms) - self.set_compute_sms(self.total_sms) - - class UBatchWrapper: def __init__( self, @@ -135,7 +87,7 @@ class UBatchWrapper: runnable, vllm_config, runtime_mode=runtime_mode ) - self.sm_control = self._create_sm_control_context(vllm_config) + self.sm_control = create_sm_control_context(vllm_config.parallel_config) self.device = device self.is_debugging_mode = envs.VLLM_LOGGING_LEVEL == "DEBUG" self._runnable_str = str(runnable) if self.is_debugging_mode else None @@ -151,49 +103,6 @@ class UBatchWrapper: if self.cudagraph_wrapper is not None: self.cudagraph_wrapper.clear_graphs() - @staticmethod - def _create_sm_control_context(vllm_config: VllmConfig): - comm_sms: int = envs.VLLM_DBO_COMM_SMS - rocm_deepep_ht_dbo = ( - current_platform.is_rocm() - and vllm_config.parallel_config.enable_dbo - and vllm_config.parallel_config.all2all_backend == "deepep_high_throughput" - ) - if rocm_deepep_ht_dbo: - # On ROCm, reserving CUs for DeepEP HT communication under DBO - # corrupts DP+EP generation accuracy. Keep the backend active, but - # leave all CUs visible to the compute and communication kernels. - comm_sms = 0 - - set_comm_sms = lambda sms: None - if vllm_config.parallel_config.enable_expert_parallel: - # Currently only DeepEP highthroughput supports SM control so this - # only affects that case. - ep_group = get_ep_group() - device_communicator = ep_group.device_communicator - all2all_manager = None - if device_communicator is not None: - all2all_manager = device_communicator.all2all_manager - - if all2all_manager is not None: - max_sms_used = all2all_manager.max_sms_used() - if max_sms_used is not None: - comm_sms = min(comm_sms, max_sms_used) - - if comm_sms > 0 and all2all_manager is not None: - set_comm_sms = lambda sms: all2all_manager.set_num_sms(sms) - - # TODO(lucas): support other kernels besides DeepGEMM - set_compute_sms = lambda sms: None - if has_deep_gemm() and comm_sms > 0: - set_compute_sms = lambda sms: deep_gemm_set_num_sms(sms) - - return SMControlContextManager( - comm_sms=comm_sms, - set_comm_sms=set_comm_sms, - set_compute_sms=set_compute_sms, - ) - def __getattr__(self, key: str): # allow accessing the attributes of the runnable. if hasattr(self.runnable, key): diff --git a/vllm/v1/worker/ubatch_utils.py b/vllm/v1/worker/ubatch_utils.py index f4a7652902..f9612b1e7f 100644 --- a/vllm/v1/worker/ubatch_utils.py +++ b/vllm/v1/worker/ubatch_utils.py @@ -1,12 +1,19 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from collections.abc import Callable from dataclasses import dataclass from typing import TypeAlias import numpy as np import torch +import vllm.envs as envs from vllm.config import ParallelConfig +from vllm.distributed import get_ep_group +from vllm.platforms import current_platform +from vllm.utils.deep_gemm import set_num_sms as deep_gemm_set_num_sms +from vllm.utils.import_utils import has_deep_gemm +from vllm.utils.platform_utils import num_compute_units from vllm.v1.attention.backend import CommonAttentionMetadata @@ -29,12 +36,108 @@ class UBatchSlice: UBatchSlices: TypeAlias = list[UBatchSlice] +class SMControlContextManager: + def __init__( + self, + comm_sms: int, + set_comm_sms: Callable[[int], None], + set_compute_sms: Callable[[int], None], + ): + """ + Context manager for controlling SM (Streaming Multiprocessor) + allocation. Upon entering the context, it sets the number of SMs + allocated for communication and computation to comm_sms and + total_sms - comm_sms respectively. Upon exiting, it restores the + allocation to use all available SMs (i.e. total_sms). + + Args: + comm_sms (int): The number of SMs to allocate for communication. + (The remainder will be used for computation.) + set_comm_sms (Callable[[int], None]): + A function that sets the number of SMs for communication. + set_compute_sms (Callable[[int], None]): + A function that sets the number of SMs for computation. + """ + + assert current_platform.is_cuda() or current_platform.is_rocm(), ( + "SM/CU control is supported on CUDA and ROCm platforms" + ) + device = torch.accelerator.current_device_index() + total_sms = num_compute_units(device) + + assert comm_sms < total_sms + self.total_sms = total_sms + self.compute_sms = total_sms - comm_sms + self.comm_sms = comm_sms + self.set_comm_sms = set_comm_sms + self.set_compute_sms = set_compute_sms + + def __enter__(self): + self.set_comm_sms(self.comm_sms) + self.set_compute_sms(self.compute_sms) + + def __exit__(self, exc_type, exc_value, traceback): + self.set_comm_sms(self.total_sms) + self.set_compute_sms(self.total_sms) + + +def create_sm_control_context( + parallel_config: ParallelConfig, +) -> SMControlContextManager: + """Reserve SMs for communication kernels while microbatches overlap.""" + comm_sms: int = envs.VLLM_DBO_COMM_SMS + rocm_deepep_ht_dbo = ( + current_platform.is_rocm() + and parallel_config.enable_dbo + and parallel_config.all2all_backend == "deepep_high_throughput" + ) + if rocm_deepep_ht_dbo: + # On ROCm, reserving CUs for DeepEP HT communication under DBO + # corrupts DP+EP generation accuracy. Keep the backend active, but + # leave all CUs visible to the compute and communication kernels. + comm_sms = 0 + + set_comm_sms = lambda sms: None + if parallel_config.enable_expert_parallel: + # Currently only DeepEP highthroughput supports SM control so this + # only affects that case. + ep_group = get_ep_group() + device_communicator = ep_group.device_communicator + all2all_manager = None + if device_communicator is not None: + all2all_manager = device_communicator.all2all_manager + + if all2all_manager is not None: + max_sms_used = all2all_manager.max_sms_used() + if max_sms_used is not None: + comm_sms = min(comm_sms, max_sms_used) + + if comm_sms > 0 and all2all_manager is not None: + set_comm_sms = lambda sms: all2all_manager.set_num_sms(sms) + + # TODO(lucas): support other kernels besides DeepGEMM + set_compute_sms = lambda sms: None + if has_deep_gemm() and comm_sms > 0: + set_compute_sms = lambda sms: deep_gemm_set_num_sms(sms) + + return SMControlContextManager( + comm_sms=comm_sms, + set_comm_sms=set_comm_sms, + set_compute_sms=set_compute_sms, + ) + + def is_last_ubatch_empty( orig_num_tokens: int, padded_num_tokens: int, num_ubatches: int ) -> bool: return (padded_num_tokens // num_ubatches) * (num_ubatches - 1) >= orig_num_tokens +def get_num_ubatches(parallel_config: ParallelConfig) -> int: + """How many microbatches a step is split into; 1 when microbatching is off.""" + return parallel_config.num_ubatches if parallel_config.use_ubatching else 1 + + def check_ubatch_thresholds( config: ParallelConfig, num_tokens: int, uniform_decode: bool ) -> bool: