[Core][MRV2] Support eagle3 spec decode with pipeline parallel (#50514)
Signed-off-by: Yongqin Wang <yongqinwang@roblox.com> Co-authored-by: Cursor <noreply@cursor.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
Cursor
Wentao Ye
parent
d4ef5e53fc
commit
d87a440f88
@@ -3,6 +3,7 @@
|
||||
|
||||
import asyncio
|
||||
import weakref
|
||||
from bisect import bisect_right
|
||||
from collections.abc import (
|
||||
AsyncGenerator,
|
||||
Callable,
|
||||
@@ -1572,10 +1573,36 @@ class LocalArgmaxMixin:
|
||||
|
||||
|
||||
class EagleModelMixin:
|
||||
start_layer: int
|
||||
aux_hidden_state_layers: tuple[int, ...] = ()
|
||||
supports_aux_hidden_states_over_pp: ClassVar[bool] = False
|
||||
AUX_HIDDEN_STATE_KEY: ClassVar[str] = "aux_hidden_states_"
|
||||
_aux_slot_base_cached: int = 0
|
||||
_aux_upstream_total_cached: int = 0
|
||||
|
||||
def _set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None:
|
||||
self.aux_hidden_state_layers = layers
|
||||
self.aux_hidden_state_layers = tuple(sorted(set(layers)))
|
||||
self._aux_slot_base_cached = 0
|
||||
self._aux_upstream_total_cached = 0
|
||||
self._cache_aux_pp_layout()
|
||||
|
||||
def _cache_aux_pp_layout(self) -> None:
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_pp_group,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
|
||||
if not model_parallel_is_initialized():
|
||||
return
|
||||
pp = get_pp_group()
|
||||
if pp.world_size < 2:
|
||||
return
|
||||
if not pp.is_first_rank:
|
||||
self._aux_slot_base_cached = bisect_right(
|
||||
self.aux_hidden_state_layers, self.start_layer
|
||||
)
|
||||
if pp.is_last_rank:
|
||||
self._aux_upstream_total_cached = self._aux_slot_base_cached
|
||||
|
||||
def _maybe_add_hidden_state(
|
||||
self,
|
||||
@@ -1589,6 +1616,36 @@ class EagleModelMixin:
|
||||
aux_hidden_states.append(value)
|
||||
return aux_hidden_states
|
||||
|
||||
def pack_local_aux_hidden_states(
|
||||
self, aux_hidden_states: list[torch.Tensor]
|
||||
) -> dict[str, torch.Tensor]:
|
||||
if not aux_hidden_states:
|
||||
return {}
|
||||
base = self._aux_slot_base_cached
|
||||
return {
|
||||
f"{self.AUX_HIDDEN_STATE_KEY}{base + i}": t
|
||||
for i, t in enumerate(aux_hidden_states)
|
||||
}
|
||||
|
||||
def collect_remote_aux_hidden_states(
|
||||
self, intermediate_tensors: "IntermediateTensors | None"
|
||||
) -> list[torch.Tensor]:
|
||||
total = self._aux_upstream_total_cached
|
||||
if total == 0:
|
||||
return []
|
||||
|
||||
assert intermediate_tensors is not None
|
||||
out: list[torch.Tensor] = []
|
||||
for i in range(total):
|
||||
key = f"{self.AUX_HIDDEN_STATE_KEY}{i}"
|
||||
if key not in intermediate_tensors.tensors:
|
||||
raise RuntimeError(
|
||||
f"Missing {key} from PP intermediate tensors: "
|
||||
f"{sorted(intermediate_tensors.tensors)}"
|
||||
)
|
||||
out.append(intermediate_tensors[key])
|
||||
return out
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsEagle(SupportsEagleBase, Protocol):
|
||||
|
||||
Reference in New Issue
Block a user