[Core][MRV2] Support eagle3 spec decode with pipeline parallel (#50514)
Signed-off-by: Yongqin Wang <yongqinwang@roblox.com> Co-authored-by: Cursor <noreply@cursor.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
Cursor
Wentao Ye
parent
d4ef5e53fc
commit
d87a440f88
@@ -285,9 +285,7 @@ class Eagle3DeepseekV2ForCausalLM(LocalArgmaxMixin, DeepseekV2ForCausalLM):
|
||||
base_vocab_size = getattr(self.config, "vocab_size", None)
|
||||
self.config.draft_vocab_size = base_vocab_size
|
||||
|
||||
target_layer_num = vllm_config.model_config.get_num_layers(
|
||||
vllm_config.parallel_config
|
||||
)
|
||||
target_layer_num = vllm_config.model_config.get_total_num_hidden_layers()
|
||||
|
||||
# Store target layer count in draft config
|
||||
self.config.target_layer_count = target_layer_num
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
import asyncio
|
||||
import weakref
|
||||
from bisect import bisect_right
|
||||
from collections.abc import (
|
||||
AsyncGenerator,
|
||||
Callable,
|
||||
@@ -1572,10 +1573,36 @@ class LocalArgmaxMixin:
|
||||
|
||||
|
||||
class EagleModelMixin:
|
||||
start_layer: int
|
||||
aux_hidden_state_layers: tuple[int, ...] = ()
|
||||
supports_aux_hidden_states_over_pp: ClassVar[bool] = False
|
||||
AUX_HIDDEN_STATE_KEY: ClassVar[str] = "aux_hidden_states_"
|
||||
_aux_slot_base_cached: int = 0
|
||||
_aux_upstream_total_cached: int = 0
|
||||
|
||||
def _set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None:
|
||||
self.aux_hidden_state_layers = layers
|
||||
self.aux_hidden_state_layers = tuple(sorted(set(layers)))
|
||||
self._aux_slot_base_cached = 0
|
||||
self._aux_upstream_total_cached = 0
|
||||
self._cache_aux_pp_layout()
|
||||
|
||||
def _cache_aux_pp_layout(self) -> None:
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_pp_group,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
|
||||
if not model_parallel_is_initialized():
|
||||
return
|
||||
pp = get_pp_group()
|
||||
if pp.world_size < 2:
|
||||
return
|
||||
if not pp.is_first_rank:
|
||||
self._aux_slot_base_cached = bisect_right(
|
||||
self.aux_hidden_state_layers, self.start_layer
|
||||
)
|
||||
if pp.is_last_rank:
|
||||
self._aux_upstream_total_cached = self._aux_slot_base_cached
|
||||
|
||||
def _maybe_add_hidden_state(
|
||||
self,
|
||||
@@ -1589,6 +1616,36 @@ class EagleModelMixin:
|
||||
aux_hidden_states.append(value)
|
||||
return aux_hidden_states
|
||||
|
||||
def pack_local_aux_hidden_states(
|
||||
self, aux_hidden_states: list[torch.Tensor]
|
||||
) -> dict[str, torch.Tensor]:
|
||||
if not aux_hidden_states:
|
||||
return {}
|
||||
base = self._aux_slot_base_cached
|
||||
return {
|
||||
f"{self.AUX_HIDDEN_STATE_KEY}{base + i}": t
|
||||
for i, t in enumerate(aux_hidden_states)
|
||||
}
|
||||
|
||||
def collect_remote_aux_hidden_states(
|
||||
self, intermediate_tensors: "IntermediateTensors | None"
|
||||
) -> list[torch.Tensor]:
|
||||
total = self._aux_upstream_total_cached
|
||||
if total == 0:
|
||||
return []
|
||||
|
||||
assert intermediate_tensors is not None
|
||||
out: list[torch.Tensor] = []
|
||||
for i in range(total):
|
||||
key = f"{self.AUX_HIDDEN_STATE_KEY}{i}"
|
||||
if key not in intermediate_tensors.tensors:
|
||||
raise RuntimeError(
|
||||
f"Missing {key} from PP intermediate tensors: "
|
||||
f"{sorted(intermediate_tensors.tensors)}"
|
||||
)
|
||||
out.append(intermediate_tensors[key])
|
||||
return out
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsEagle(SupportsEagleBase, Protocol):
|
||||
|
||||
@@ -245,9 +245,7 @@ class DFlashLagunaForCausalLM(nn.Module, SupportsEagle3):
|
||||
raise ValueError("Laguna DFlash config requires `draft_vocab_size`.")
|
||||
self.has_own_embed_tokens = False
|
||||
self.has_own_lm_head = False
|
||||
target_layer_num = vllm_config.model_config.get_num_layers(
|
||||
vllm_config.parallel_config
|
||||
)
|
||||
target_layer_num = vllm_config.model_config.get_total_num_hidden_layers()
|
||||
self.config.target_layer_count = target_layer_num
|
||||
target_vocab_size = vllm_config.model_config.get_vocab_size()
|
||||
if self.config.draft_vocab_size != target_vocab_size:
|
||||
|
||||
@@ -75,6 +75,7 @@ from .utils import (
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
spec_decode_needs_target_embed,
|
||||
)
|
||||
|
||||
|
||||
@@ -354,6 +355,7 @@ class LlamaModel(nn.Module, EagleModelMixin):
|
||||
".up_proj": (".gate_up_proj", 1),
|
||||
}
|
||||
)
|
||||
supports_aux_hidden_states_over_pp = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -372,8 +374,10 @@ class LlamaModel(nn.Module, EagleModelMixin):
|
||||
|
||||
self.vocab_size = config.vocab_size
|
||||
|
||||
if get_pp_group().is_first_rank or (
|
||||
config.tie_word_embeddings and get_pp_group().is_last_rank
|
||||
if (
|
||||
get_pp_group().is_first_rank
|
||||
or (config.tie_word_embeddings and get_pp_group().is_last_rank)
|
||||
or spec_decode_needs_target_embed(vllm_config)
|
||||
):
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
self.vocab_size,
|
||||
@@ -418,9 +422,16 @@ class LlamaModel(nn.Module, EagleModelMixin):
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
residual = intermediate_tensors["residual"]
|
||||
|
||||
aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual)
|
||||
remote_aux = self.collect_remote_aux_hidden_states(intermediate_tensors)
|
||||
|
||||
aux_hidden_states: list[torch.Tensor] = []
|
||||
if get_pp_group().is_first_rank:
|
||||
self._maybe_add_hidden_state(
|
||||
aux_hidden_states, self.start_layer, hidden_states, residual
|
||||
)
|
||||
for idx, layer in enumerate(
|
||||
islice(self.layers, self.start_layer, self.end_layer)
|
||||
islice(self.layers, self.start_layer, self.end_layer),
|
||||
start=self.start_layer,
|
||||
):
|
||||
hidden_states, residual = layer(
|
||||
positions, hidden_states, residual, **extra_layer_kwargs
|
||||
@@ -431,11 +442,16 @@ class LlamaModel(nn.Module, EagleModelMixin):
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
return IntermediateTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
"residual": residual,
|
||||
**self.pack_local_aux_hidden_states(aux_hidden_states),
|
||||
}
|
||||
)
|
||||
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
|
||||
aux_hidden_states = remote_aux + aux_hidden_states
|
||||
if len(aux_hidden_states) > 0:
|
||||
return hidden_states, aux_hidden_states
|
||||
return hidden_states
|
||||
|
||||
@@ -314,9 +314,7 @@ class Eagle3LlamaForCausalLM(_Eagle3LlamaForCausalLMBase):
|
||||
if getattr(self.config, "draft_vocab_size", None) is None:
|
||||
base_vocab_size = getattr(self.config, "vocab_size", None)
|
||||
self.config.draft_vocab_size = base_vocab_size
|
||||
target_layer_num = vllm_config.model_config.get_num_layers(
|
||||
vllm_config.parallel_config
|
||||
)
|
||||
target_layer_num = vllm_config.model_config.get_total_num_hidden_layers()
|
||||
|
||||
# Store target layer count in draft config for
|
||||
# proper layer_types indexing in draft models
|
||||
|
||||
@@ -55,6 +55,8 @@ logger = init_logger(__name__)
|
||||
}
|
||||
)
|
||||
class MiMoModel(Qwen2Model):
|
||||
supports_aux_hidden_states_over_pp = False
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
|
||||
@@ -74,6 +74,7 @@ from .utils import (
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
spec_decode_needs_target_embed,
|
||||
)
|
||||
|
||||
|
||||
@@ -330,6 +331,7 @@ class Qwen2Model(nn.Module, EagleModelMixin):
|
||||
".up_proj": (".gate_up_proj", 1),
|
||||
}
|
||||
)
|
||||
supports_aux_hidden_states_over_pp = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -360,8 +362,10 @@ class Qwen2Model(nn.Module, EagleModelMixin):
|
||||
self.quant_config = quant_config
|
||||
self.vocab_size = config.vocab_size
|
||||
|
||||
if get_pp_group().is_first_rank or (
|
||||
config.tie_word_embeddings and get_pp_group().is_last_rank
|
||||
if (
|
||||
get_pp_group().is_first_rank
|
||||
or (config.tie_word_embeddings and get_pp_group().is_last_rank)
|
||||
or spec_decode_needs_target_embed(vllm_config)
|
||||
):
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
@@ -412,9 +416,16 @@ class Qwen2Model(nn.Module, EagleModelMixin):
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
residual = intermediate_tensors["residual"]
|
||||
|
||||
aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual)
|
||||
remote_aux = self.collect_remote_aux_hidden_states(intermediate_tensors)
|
||||
|
||||
aux_hidden_states: list[torch.Tensor] = []
|
||||
if get_pp_group().is_first_rank:
|
||||
self._maybe_add_hidden_state(
|
||||
aux_hidden_states, self.start_layer, hidden_states, residual
|
||||
)
|
||||
for idx, layer in enumerate(
|
||||
islice(self.layers, self.start_layer, self.end_layer)
|
||||
islice(self.layers, self.start_layer, self.end_layer),
|
||||
start=self.start_layer,
|
||||
):
|
||||
hidden_states, residual = layer(positions, hidden_states, residual)
|
||||
self._maybe_add_hidden_state(
|
||||
@@ -423,11 +434,16 @@ class Qwen2Model(nn.Module, EagleModelMixin):
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
return IntermediateTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
"residual": residual,
|
||||
**self.pack_local_aux_hidden_states(aux_hidden_states),
|
||||
}
|
||||
)
|
||||
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
|
||||
aux_hidden_states = remote_aux + aux_hidden_states
|
||||
if len(aux_hidden_states) > 0:
|
||||
return hidden_states, aux_hidden_states
|
||||
|
||||
|
||||
@@ -694,9 +694,7 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
|
||||
self.config = self.draft_model_config.hf_config
|
||||
if getattr(self.config, "draft_vocab_size", None) is None:
|
||||
self.config.draft_vocab_size = getattr(self.config, "vocab_size", None)
|
||||
target_layer_num = vllm_config.model_config.get_num_layers(
|
||||
vllm_config.parallel_config
|
||||
)
|
||||
target_layer_num = vllm_config.model_config.get_total_num_hidden_layers()
|
||||
self.model = self.model_cls(
|
||||
vllm_config=vllm_config,
|
||||
prefix=maybe_prefix(prefix, "model"),
|
||||
|
||||
@@ -182,9 +182,7 @@ class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM):
|
||||
self.config = self.draft_model_config.hf_config
|
||||
if getattr(self.config, "draft_vocab_size", None) is None:
|
||||
self.config.draft_vocab_size = getattr(self.config, "vocab_size", None)
|
||||
target_layer_num = vllm_config.model_config.get_num_layers(
|
||||
vllm_config.parallel_config
|
||||
)
|
||||
target_layer_num = vllm_config.model_config.get_total_num_hidden_layers()
|
||||
self.model = Qwen3DSparkModel(
|
||||
vllm_config=vllm_config,
|
||||
prefix=maybe_prefix(prefix, "model"),
|
||||
|
||||
@@ -288,9 +288,7 @@ class Eagle3Qwen3ForCausalLM(Qwen3ForCausalLM):
|
||||
if getattr(self.config, "draft_vocab_size", None) is None:
|
||||
base_vocab_size = getattr(self.config, "vocab_size", None)
|
||||
self.config.draft_vocab_size = base_vocab_size
|
||||
target_layer_num = vllm_config.model_config.get_num_layers(
|
||||
vllm_config.parallel_config
|
||||
)
|
||||
target_layer_num = vllm_config.model_config.get_total_num_hidden_layers()
|
||||
|
||||
# Store target layer count in draft config for
|
||||
# proper layer_types indexing in draft models
|
||||
|
||||
@@ -835,6 +835,22 @@ class PPMissingLayer(torch.nn.Identity):
|
||||
return args[0] if args else next(iter(kwargs.values()))
|
||||
|
||||
|
||||
def spec_decode_needs_target_embed(vllm_config: VllmConfig) -> bool:
|
||||
"""Whether the last PP rank needs the target input embedding."""
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
|
||||
speculative_config = vllm_config.speculative_config
|
||||
if speculative_config is None or speculative_config.method not in (
|
||||
"eagle",
|
||||
"eagle3",
|
||||
"dflash",
|
||||
"dspark",
|
||||
):
|
||||
return False
|
||||
pp = get_pp_group()
|
||||
return pp.world_size > 1 and pp.is_last_rank
|
||||
|
||||
|
||||
def make_layers(
|
||||
num_hidden_layers: int,
|
||||
layer_fn: LayerFn,
|
||||
|
||||
Reference in New Issue
Block a user