[SpecDecode] Reduce TP communication for large-vocab draft models speculative decoding (#39419)
Signed-off-by: EanWang211123 <wangyiheng@sangfor.com.cn>
This commit is contained in:
@@ -22,6 +22,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.interfaces import LocalArgmaxMixin
|
||||
from vllm.model_executor.models.qwen3_5 import Qwen3_5DecoderLayer, Qwen3_5RMSNorm
|
||||
from vllm.model_executor.models.qwen3_next import QwenNextMixtureOfExperts
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -353,7 +354,7 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
|
||||
"hidden_states": 0,
|
||||
}
|
||||
)
|
||||
class Qwen3_5MTP(nn.Module, SupportsMultiModal):
|
||||
class Qwen3_5MTP(LocalArgmaxMixin, nn.Module, SupportsMultiModal):
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": [
|
||||
"q_proj",
|
||||
|
||||
Reference in New Issue
Block a user