[Model] Add native Dots3 NOTE multimodal support (#51255)

Signed-off-by: KurodaKanbei <mistergalahad@gmail.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
范裕达
2026-08-12 23:13:20 +08:00
committed by GitHub
co-authored by OpenAI Codex Jee Jee Li Jee Jee Li
parent b1b752042f
commit 9035151d6c
28 changed files with 6468 additions and 7 deletions
@@ -299,6 +299,7 @@ from vllm.v1.kv_cache_interface import (
AttentionSpec,
KVCacheSpec,
MLAAttentionSpec,
SlidingWindowMLASpec,
get_kv_quant_mode,
)
@@ -406,6 +407,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
indexer: object | None = None,
topk_indices_buffer: torch.Tensor | None = None,
non_causal_multi_token_decode: bool = False,
sliding_window: int | None = None,
prefill_backend_cls: type[MLAPrefillBackend] | None = None,
**extra_impl_args,
):
super().__init__()
@@ -423,6 +426,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
self.layer_name = prefix
self.indexer = indexer
self.non_causal_multi_token_decode = non_causal_multi_token_decode
self.sliding_window = sliding_window
self.num_kv_heads = 1
self.qk_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim
@@ -515,7 +519,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
scale=self.scale,
num_kv_heads=1,
alibi_slopes=None,
sliding_window=None,
sliding_window=sliding_window,
kv_cache_dtype=self.kv_cache_dtype,
logits_soft_cap=None,
attn_type=AttentionType.DECODER,
@@ -551,7 +555,9 @@ class MLAAttention(nn.Module, AttentionLayerBase):
self.prefill_backend = None
else:
try:
prefill_backend_cls = get_mla_prefill_backend(vllm_config)
prefill_backend_cls = prefill_backend_cls or get_mla_prefill_backend(
vllm_config
)
except ValueError:
if (
not self.impl.is_sparse
@@ -1139,13 +1145,21 @@ class MLAAttention(nn.Module, AttentionLayerBase):
kv_cache_dtype = kv_cache_dtype_str_to_dtype(
self.kv_cache_dtype, vllm_config.model_config
)
return MLAAttentionSpec(
common_kwargs = dict(
block_size=vllm_config.cache_config.block_size,
num_kv_heads=1,
head_size=self.head_size,
dtype=kv_cache_dtype,
cache_dtype_str=self.kv_cache_dtype,
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
)
if self.sliding_window is not None:
return SlidingWindowMLASpec(
**common_kwargs,
sliding_window=self.sliding_window,
)
return MLAAttentionSpec(
**common_kwargs,
non_causal_multi_token_decode=self.non_causal_multi_token_decode,
)
@@ -2056,7 +2070,15 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
self.num_heads = get_num_attention_heads_from_layers(
vllm_config, layer_names
) or self.model_config.get_num_attention_heads(parallel_config)
self.mla_dims = get_mla_dims(self.model_config)
# Hybrid MLA models may use different latent dimensions per KV group.
layer = self.compilation_config.static_forward_context[layer_names[0]]
self.mla_dims = MLADims(
q_lora_rank=layer.q_lora_rank,
kv_lora_rank=layer.kv_lora_rank,
qk_nope_head_dim=layer.qk_nope_head_dim,
qk_rope_head_dim=layer.qk_rope_head_dim,
v_head_dim=layer.v_head_dim,
)
self.aot_schedule = current_platform.is_cuda()
self.kv_cache_spec = kv_cache_spec
@@ -2092,7 +2114,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
(
self.chunked_prefill_workspace_size
+ self.chunked_prefill_workspace_size // self.dcp_world_size,
self.model_config.get_head_size(),
self.mla_dims.kv_lora_rank + self.mla_dims.qk_rope_head_dim,
),
dtype=torch.bfloat16
if use_packed_fp8_cache
@@ -2109,7 +2131,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
self.chunked_prefill_workspace = torch.empty(
(
self.chunked_prefill_workspace_size,
self.model_config.get_head_size(),
self.mla_dims.kv_lora_rank + self.mla_dims.qk_rope_head_dim,
),
dtype=torch.bfloat16 if use_packed_fp8_cache else self.q_data_type,
device=device,