[Model] Add MiniMax M3 support (#45381)
Signed-off-by: youkaichao <youkaichao@gmail.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Signed-off-by: functionstackx <47992694+functionstackx@users.noreply.github.com> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: functionstackx <47992694+functionstackx@users.noreply.github.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
co-authored by
OpenAI Codex
Isotr0py
Thien Tran
Bugen Zhao
Jee Jee Li
Roger Wang
functionstackx
Yongye Zhu
Jee Jee Li
parent
a3195fab7b
commit
0a1c5034f5
@@ -7,6 +7,7 @@ import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphWrapper
|
||||
from vllm.config import (
|
||||
CUDAGraphMode,
|
||||
VllmConfig,
|
||||
@@ -250,6 +251,13 @@ class SpecDecodeBaseProposer:
|
||||
DeepseekV4ROCMAiterMLASparseMetadata,
|
||||
DeepseekV4ROCMAiterSparseSWAMetadata,
|
||||
)
|
||||
|
||||
# MiniMax-M3 sparse (lightning-indexer) attention. The multi-step
|
||||
# drafting machinery is shared code at num_speculative_tokens>1.
|
||||
# this just opts the metadata into the ROCm allowlist.
|
||||
from vllm.models.minimax_m3.common.sparse_attention import (
|
||||
MiniMaxM3SparseMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
DeepseekV32IndexerMetadata,
|
||||
)
|
||||
@@ -265,6 +273,7 @@ class SpecDecodeBaseProposer:
|
||||
DeepseekV4ROCMAiterMLASparseMetadata,
|
||||
DeepseekV4ROCMAiterSparseSWAMetadata,
|
||||
DeepseekV32IndexerMetadata,
|
||||
MiniMaxM3SparseMetadata,
|
||||
]
|
||||
# ROCM_AITER_FA is an optional backend
|
||||
# We check is_enabled() here to avoid importing the backend module during
|
||||
@@ -457,8 +466,11 @@ class SpecDecodeBaseProposer:
|
||||
batch_size = common_attn_metadata.batch_size()
|
||||
|
||||
if self.method in ("eagle3", "dflash"):
|
||||
model = self.model
|
||||
if isinstance(model, BreakableCUDAGraphWrapper):
|
||||
model = model.unwrap()
|
||||
assert isinstance(
|
||||
self.model,
|
||||
model,
|
||||
(
|
||||
Eagle3LlamaForCausalLM,
|
||||
Eagle3DeepseekV2ForCausalLM,
|
||||
|
||||
Reference in New Issue
Block a user