Added qwen3 vision language moe support for speculative decoding (#32048)
Signed-off-by: shanjiaz <zsjwpianpian@gmail.com> Signed-off-by: shanjiaz <43143795+shanjiaz@users.noreply.github.com>
This commit is contained in:
@@ -112,7 +112,9 @@ class SpecDecodeBaseProposer:
|
||||
self.input_ids = torch.zeros(
|
||||
self.max_num_tokens, dtype=torch.int32, device=device
|
||||
)
|
||||
self.uses_mrope = self.vllm_config.model_config.uses_mrope
|
||||
# Use draft model's M-RoPE setting, not target model's
|
||||
# Draft models may be text-only even if target is multimodal
|
||||
self.uses_mrope = self.draft_model_config.uses_mrope
|
||||
if self.uses_mrope:
|
||||
# NOTE: `mrope_positions` is implemented with one additional dummy
|
||||
# position on purpose to make it non-contiguous so that it can work
|
||||
@@ -221,6 +223,11 @@ class SpecDecodeBaseProposer:
|
||||
if self.uses_mrope:
|
||||
self.mrope_positions[:, :num_tokens] = positions
|
||||
else:
|
||||
# Convert M-RoPE positions if target model uses M-RoPE
|
||||
# but draft doesn't, For text inputs, all M-RoPE
|
||||
# dimensions are identical
|
||||
if self.vllm_config.model_config.uses_mrope:
|
||||
positions = positions[0]
|
||||
self.positions[:num_tokens] = positions
|
||||
|
||||
def initialize_cudagraph_keys(self, cudagraph_mode: CUDAGraphMode) -> None:
|
||||
@@ -1080,6 +1087,7 @@ class SpecDecodeBaseProposer:
|
||||
if self.get_model_name(target_model) in [
|
||||
"Qwen2_5_VLForConditionalGeneration",
|
||||
"Qwen3VLForConditionalGeneration",
|
||||
"Qwen3VLMoeForConditionalGeneration",
|
||||
]:
|
||||
self.model.config.image_token_index = target_model.config.image_token_id
|
||||
elif self.get_model_name(target_model) == "PixtralForConditionalGeneration":
|
||||
|
||||
Reference in New Issue
Block a user