[Model] Add native Dots3 NOTE multimodal support (#51255)
Signed-off-by: KurodaKanbei <mistergalahad@gmail.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
Jee Jee Li
Jee Jee Li
parent
b1b752042f
commit
9035151d6c
@@ -299,6 +299,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
KVCacheSpec,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowMLASpec,
|
||||
get_kv_quant_mode,
|
||||
)
|
||||
|
||||
@@ -406,6 +407,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
indexer: object | None = None,
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
non_causal_multi_token_decode: bool = False,
|
||||
sliding_window: int | None = None,
|
||||
prefill_backend_cls: type[MLAPrefillBackend] | None = None,
|
||||
**extra_impl_args,
|
||||
):
|
||||
super().__init__()
|
||||
@@ -423,6 +426,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
self.layer_name = prefix
|
||||
self.indexer = indexer
|
||||
self.non_causal_multi_token_decode = non_causal_multi_token_decode
|
||||
self.sliding_window = sliding_window
|
||||
self.num_kv_heads = 1
|
||||
self.qk_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim
|
||||
|
||||
@@ -515,7 +519,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
scale=self.scale,
|
||||
num_kv_heads=1,
|
||||
alibi_slopes=None,
|
||||
sliding_window=None,
|
||||
sliding_window=sliding_window,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
logits_soft_cap=None,
|
||||
attn_type=AttentionType.DECODER,
|
||||
@@ -551,7 +555,9 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
self.prefill_backend = None
|
||||
else:
|
||||
try:
|
||||
prefill_backend_cls = get_mla_prefill_backend(vllm_config)
|
||||
prefill_backend_cls = prefill_backend_cls or get_mla_prefill_backend(
|
||||
vllm_config
|
||||
)
|
||||
except ValueError:
|
||||
if (
|
||||
not self.impl.is_sparse
|
||||
@@ -1139,13 +1145,21 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
kv_cache_dtype = kv_cache_dtype_str_to_dtype(
|
||||
self.kv_cache_dtype, vllm_config.model_config
|
||||
)
|
||||
return MLAAttentionSpec(
|
||||
common_kwargs = dict(
|
||||
block_size=vllm_config.cache_config.block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=self.head_size,
|
||||
dtype=kv_cache_dtype,
|
||||
cache_dtype_str=self.kv_cache_dtype,
|
||||
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
|
||||
)
|
||||
if self.sliding_window is not None:
|
||||
return SlidingWindowMLASpec(
|
||||
**common_kwargs,
|
||||
sliding_window=self.sliding_window,
|
||||
)
|
||||
return MLAAttentionSpec(
|
||||
**common_kwargs,
|
||||
non_causal_multi_token_decode=self.non_causal_multi_token_decode,
|
||||
)
|
||||
|
||||
@@ -2056,7 +2070,15 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
self.num_heads = get_num_attention_heads_from_layers(
|
||||
vllm_config, layer_names
|
||||
) or self.model_config.get_num_attention_heads(parallel_config)
|
||||
self.mla_dims = get_mla_dims(self.model_config)
|
||||
# Hybrid MLA models may use different latent dimensions per KV group.
|
||||
layer = self.compilation_config.static_forward_context[layer_names[0]]
|
||||
self.mla_dims = MLADims(
|
||||
q_lora_rank=layer.q_lora_rank,
|
||||
kv_lora_rank=layer.kv_lora_rank,
|
||||
qk_nope_head_dim=layer.qk_nope_head_dim,
|
||||
qk_rope_head_dim=layer.qk_rope_head_dim,
|
||||
v_head_dim=layer.v_head_dim,
|
||||
)
|
||||
self.aot_schedule = current_platform.is_cuda()
|
||||
|
||||
self.kv_cache_spec = kv_cache_spec
|
||||
@@ -2092,7 +2114,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
(
|
||||
self.chunked_prefill_workspace_size
|
||||
+ self.chunked_prefill_workspace_size // self.dcp_world_size,
|
||||
self.model_config.get_head_size(),
|
||||
self.mla_dims.kv_lora_rank + self.mla_dims.qk_rope_head_dim,
|
||||
),
|
||||
dtype=torch.bfloat16
|
||||
if use_packed_fp8_cache
|
||||
@@ -2109,7 +2131,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
self.chunked_prefill_workspace = torch.empty(
|
||||
(
|
||||
self.chunked_prefill_workspace_size,
|
||||
self.model_config.get_head_size(),
|
||||
self.mla_dims.kv_lora_rank + self.mla_dims.qk_rope_head_dim,
|
||||
),
|
||||
dtype=torch.bfloat16 if use_packed_fp8_cache else self.q_data_type,
|
||||
device=device,
|
||||
|
||||
@@ -371,6 +371,10 @@ _MULTIMODAL_MODELS = {
|
||||
"DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
|
||||
"DeepseekOCRForCausalLM": ("deepseek_ocr", "DeepseekOCRForCausalLM"),
|
||||
"DeepseekOCR2ForCausalLM": ("deepseek_ocr2", "DeepseekOCR2ForCausalLM"),
|
||||
"Dots3NoteForCausalLM": (
|
||||
"vllm.models.dots3_note",
|
||||
"Dots3NoteForCausalLM",
|
||||
),
|
||||
"UnlimitedOCRForCausalLM": ("unlimited_ocr", "UnlimitedOCRForCausalLM"),
|
||||
"DotsOCRForCausalLM": ("dots_ocr", "DotsOCRForCausalLM"),
|
||||
"Eagle2_5_VLForConditionalGeneration": (
|
||||
@@ -645,6 +649,7 @@ _SPECULATIVE_DECODING_MODELS = {
|
||||
"Eagle3DeepseekV3ForCausalLM": ("deepseek_eagle3", "Eagle3DeepseekV2ForCausalLM"),
|
||||
"EagleDeepSeekMTPModel": ("deepseek_eagle", "EagleDeepseekV3ForCausalLM"),
|
||||
"DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"),
|
||||
"Dots3NoteMTPModel": ("vllm.models.dots3_note", "Dots3NoteMTP"),
|
||||
"DeepSeekV4MTPModel": ("vllm.models.deepseek_v4", "DeepSeekV4MTP"),
|
||||
"BailingMoeV3MTPModel": ("bailing_moe_v3_mtp", "BailingMoeV3MTPModel"),
|
||||
"MiniMaxM3MTP": ("vllm.models.minimax_m3", "MiniMaxM3MTP"),
|
||||
|
||||
Reference in New Issue
Block a user