[Bugfix][Spec Decode] Only create draft_id_to_target_id when draft vocab differs (#53458)
Signed-off-by: drslark <slarksblood@qq.com> Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: GLM-5.3 <glm@z.ai> Co-authored-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
GLM-5.3
Isotr0py
Isotr0py
parent
1e47ec00d2
commit
9446ea1680
@@ -305,10 +305,14 @@ class Eagle3DeepseekV2ForCausalLM(LocalArgmaxMixin, DeepseekV2ForCausalLM):
|
||||
self.logits_processor = LogitsProcessor(
|
||||
self.config.draft_vocab_size, scale=logit_scale
|
||||
)
|
||||
self.draft_id_to_target_id = nn.Parameter(
|
||||
torch.zeros(self.config.draft_vocab_size, dtype=torch.long),
|
||||
requires_grad=False,
|
||||
)
|
||||
target_vocab_size = vllm_config.model_config.get_vocab_size()
|
||||
if self.config.draft_vocab_size != target_vocab_size:
|
||||
self.draft_id_to_target_id = nn.Parameter(
|
||||
torch.zeros(self.config.draft_vocab_size, dtype=torch.long),
|
||||
requires_grad=False,
|
||||
)
|
||||
else:
|
||||
self.draft_id_to_target_id = None
|
||||
|
||||
def embed_input_ids(
|
||||
self,
|
||||
|
||||
@@ -335,10 +335,14 @@ class Eagle3LlamaForCausalLM(_Eagle3LlamaForCausalLMBase):
|
||||
self.logits_processor = LogitsProcessor(
|
||||
self.config.draft_vocab_size, scale=logit_scale
|
||||
)
|
||||
self.draft_id_to_target_id = nn.Parameter(
|
||||
torch.zeros(self.config.draft_vocab_size, dtype=torch.long),
|
||||
requires_grad=False,
|
||||
)
|
||||
target_vocab_size = vllm_config.model_config.get_vocab_size()
|
||||
if self.config.draft_vocab_size != target_vocab_size:
|
||||
self.draft_id_to_target_id = nn.Parameter(
|
||||
torch.zeros(self.config.draft_vocab_size, dtype=torch.long),
|
||||
requires_grad=False,
|
||||
)
|
||||
else:
|
||||
self.draft_id_to_target_id = None
|
||||
|
||||
self.use_parallel_drafting = speculative_config.parallel_drafting
|
||||
|
||||
|
||||
@@ -309,10 +309,14 @@ class Eagle3Qwen3ForCausalLM(Qwen3ForCausalLM):
|
||||
self.logits_processor = LogitsProcessor(
|
||||
self.config.draft_vocab_size, scale=logit_scale
|
||||
)
|
||||
self.draft_id_to_target_id = nn.Parameter(
|
||||
torch.zeros(self.config.draft_vocab_size, dtype=torch.long),
|
||||
requires_grad=False,
|
||||
)
|
||||
target_vocab_size = vllm_config.model_config.get_vocab_size()
|
||||
if self.config.draft_vocab_size != target_vocab_size:
|
||||
self.draft_id_to_target_id = nn.Parameter(
|
||||
torch.zeros(self.config.draft_vocab_size, dtype=torch.long),
|
||||
requires_grad=False,
|
||||
)
|
||||
else:
|
||||
self.draft_id_to_target_id = None
|
||||
|
||||
self.use_parallel_drafting = vllm_config.speculative_config.parallel_drafting
|
||||
|
||||
|
||||
Reference in New Issue
Block a user