[BugFix][Spec Decode] Fix out-of-range index triggered by eagle3; re-enable test for LlamaForCausalLMEagle3 (#24392)
Signed-off-by: wwl2755 <wangwenlong2755@gmail.com>
This commit is contained in:
@@ -171,7 +171,22 @@ class LlamaAttention(nn.Module):
|
||||
|
||||
sliding_window = None
|
||||
if layer_types := getattr(config, "layer_types", None):
|
||||
is_sliding = layer_types[layer_idx] == "sliding_attention"
|
||||
# Fix for Eagle3 compatibility:
|
||||
# for draft models, subtract target layer count
|
||||
# to get draft-relative layer index starting from 0
|
||||
if hasattr(config, 'target_layer_count'):
|
||||
# This is a draft model,
|
||||
# adjust layer_idx to be relative to draft layers
|
||||
effective_layer_idx = layer_idx - config.target_layer_count
|
||||
else:
|
||||
# This is a target model, use layer_idx directly
|
||||
effective_layer_idx = layer_idx
|
||||
assert effective_layer_idx < len(layer_types), \
|
||||
f"effective_layer_idx: {effective_layer_idx} \
|
||||
is out of bounds for layer_types: {layer_types}"
|
||||
|
||||
is_sliding = layer_types[
|
||||
effective_layer_idx] == "sliding_attention"
|
||||
if is_sliding:
|
||||
sliding_window = config.sliding_window
|
||||
|
||||
|
||||
@@ -199,6 +199,10 @@ class Eagle3LlamaForCausalLM(LlamaForCausalLM):
|
||||
speculative_config.draft_model_config.hf_config
|
||||
target_layer_num = vllm_config.model_config.get_num_layers(
|
||||
vllm_config.parallel_config)
|
||||
|
||||
# Store target layer count in draft config for
|
||||
# proper layer_types indexing in draft models
|
||||
self.config.target_layer_count = target_layer_num
|
||||
self.model = LlamaModel(vllm_config=vllm_config,
|
||||
prefix="model",
|
||||
start_layer_id=target_layer_num)
|
||||
|
||||
@@ -277,8 +277,7 @@ _SPECULATIVE_DECODING_MODELS = {
|
||||
"EagleLlama4ForCausalLM": ("llama4_eagle", "EagleLlama4ForCausalLM"),
|
||||
"EagleMiniCPMForCausalLM": ("minicpm_eagle", "EagleMiniCPMForCausalLM"),
|
||||
"Eagle3LlamaForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
|
||||
# TODO: Re-enable this once tests/models/test_initialization.py is fixed, see PR #22333 #22611 # noqa: E501
|
||||
# "LlamaForCausalLMEagle3": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
|
||||
"LlamaForCausalLMEagle3": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
|
||||
"EagleDeepSeekMTPModel": ("deepseek_eagle", "EagleDeepseekV3ForCausalLM"),
|
||||
"DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"),
|
||||
"ErnieMTPModel": ("ernie_mtp", "ErnieMTP"),
|
||||
|
||||
Reference in New Issue
Block a user