From b2198670b19ac597c630f0b8d4f7a7addb764a62 Mon Sep 17 00:00:00 2001 From: Yufeng He <40085740+he-yufeng@users.noreply.github.com> Date: Thu, 14 May 2026 06:47:51 +0800 Subject: [PATCH] [Bugfix] V1: support tuple model outputs in ubatch wrapper (dbo + spec decode) (#40789) Signed-off-by: Yufeng He <40085740+he-yufeng@users.noreply.github.com> Co-authored-by: Matthew Bonanni --- vllm/v1/worker/gpu_ubatch_wrapper.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/vllm/v1/worker/gpu_ubatch_wrapper.py b/vllm/v1/worker/gpu_ubatch_wrapper.py index a8c54c4f06..657fc82673 100644 --- a/vllm/v1/worker/gpu_ubatch_wrapper.py +++ b/vllm/v1/worker/gpu_ubatch_wrapper.py @@ -31,6 +31,23 @@ from vllm.v1.worker.ubatching import UBatchContext, make_ubatch_contexts logger = init_logger(__name__) +def _cat_ubatch_outputs( + sorted_results: list, +) -> "torch.Tensor | tuple[torch.Tensor, ...]": + """Concatenate per-ubatch model outputs along the batch dim. + + Most models return a single hidden-states tensor per ubatch. Target + models running with auxiliary output (e.g. EAGLE3 speculative decoding, + which collects aux hidden states for the drafter) return a tuple of + tensors instead. Fan out over tuple components so `torch.cat` sees + matching shapes and the caller receives the same structure the model + produced for a single ubatch (#40769). + """ + if sorted_results and isinstance(sorted_results[0], tuple): + return tuple(torch.cat(parts, dim=0) for parts in zip(*sorted_results)) + return torch.cat(sorted_results, dim=0) + + @dataclass class UbatchMetadata: context: UBatchContext @@ -266,7 +283,7 @@ class UBatchWrapper: for thread in ubatch_threads: thread.join() sorted_results = [value for position, value in sorted(results)] - result = torch.cat(sorted_results, dim=0) + result = _cat_ubatch_outputs(sorted_results) cudagraph_metadata.outputs = result # Join offloader's copy stream after forward to avoid unjoined # stream error. The last layer's start_prefetch forks copy_stream, @@ -310,7 +327,7 @@ class UBatchWrapper: for thread in ubatch_threads: thread.join() sorted_results = [value for position, value in sorted(results)] - result = torch.cat(sorted_results, dim=0) + result = _cat_ubatch_outputs(sorted_results) return result def _make_ubatch_metadata(