[Core][MRV2] Support eagle3 spec decode with pipeline parallel (#50514)

Signed-off-by: Yongqin Wang <yongqinwang@roblox.com>
Co-authored-by: Cursor <noreply@cursor.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
yongqinwang-cmd
2026-09-05 11:45:34 +08:00
committed by GitHub
co-authored by Cursor Wentao Ye
parent d4ef5e53fc
commit d87a440f88
27 changed files with 582 additions and 152 deletions
+58 -1
View File
@@ -3,6 +3,7 @@
import asyncio
import weakref
from bisect import bisect_right
from collections.abc import (
AsyncGenerator,
Callable,
@@ -1572,10 +1573,36 @@ class LocalArgmaxMixin:
class EagleModelMixin:
start_layer: int
aux_hidden_state_layers: tuple[int, ...] = ()
supports_aux_hidden_states_over_pp: ClassVar[bool] = False
AUX_HIDDEN_STATE_KEY: ClassVar[str] = "aux_hidden_states_"
_aux_slot_base_cached: int = 0
_aux_upstream_total_cached: int = 0
def _set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None:
self.aux_hidden_state_layers = layers
self.aux_hidden_state_layers = tuple(sorted(set(layers)))
self._aux_slot_base_cached = 0
self._aux_upstream_total_cached = 0
self._cache_aux_pp_layout()
def _cache_aux_pp_layout(self) -> None:
from vllm.distributed.parallel_state import (
get_pp_group,
model_parallel_is_initialized,
)
if not model_parallel_is_initialized():
return
pp = get_pp_group()
if pp.world_size < 2:
return
if not pp.is_first_rank:
self._aux_slot_base_cached = bisect_right(
self.aux_hidden_state_layers, self.start_layer
)
if pp.is_last_rank:
self._aux_upstream_total_cached = self._aux_slot_base_cached
def _maybe_add_hidden_state(
self,
@@ -1589,6 +1616,36 @@ class EagleModelMixin:
aux_hidden_states.append(value)
return aux_hidden_states
def pack_local_aux_hidden_states(
self, aux_hidden_states: list[torch.Tensor]
) -> dict[str, torch.Tensor]:
if not aux_hidden_states:
return {}
base = self._aux_slot_base_cached
return {
f"{self.AUX_HIDDEN_STATE_KEY}{base + i}": t
for i, t in enumerate(aux_hidden_states)
}
def collect_remote_aux_hidden_states(
self, intermediate_tensors: "IntermediateTensors | None"
) -> list[torch.Tensor]:
total = self._aux_upstream_total_cached
if total == 0:
return []
assert intermediate_tensors is not None
out: list[torch.Tensor] = []
for i in range(total):
key = f"{self.AUX_HIDDEN_STATE_KEY}{i}"
if key not in intermediate_tensors.tensors:
raise RuntimeError(
f"Missing {key} from PP intermediate tensors: "
f"{sorted(intermediate_tensors.tensors)}"
)
out.append(intermediate_tensors[key])
return out
@runtime_checkable
class SupportsEagle(SupportsEagleBase, Protocol):