Support MLA properly in the Transformers modeling backend (#48250)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-08-04 13:24:11 +01:00
committed by GitHub
parent 24c939c47d
commit 59b2fdfc4e
9 changed files with 704 additions and 57 deletions
+42 -11
View File
@@ -35,6 +35,12 @@ def get_num_fused(model) -> tuple[int, int]:
return glu, qkv
def count_mla_layers(model) -> int:
from vllm.model_executor.layers.attention import MLAAttention
return sum(isinstance(m, MLAAttention) for m in model.attention_instances.values())
def check_implementation(
runner_ref: type[HfRunner | VllmRunner],
runner_test: type[VllmRunner],
@@ -97,17 +103,6 @@ def test_models(
model_impl: str,
num_fused: tuple[int, int],
) -> None:
import transformers
from packaging.version import Version
installed = Version(transformers.__version__)
required = Version("5.0.0")
if model == "allenai/OLMoE-1B-7B-0924" and installed < required:
pytest.skip(
"MoE models with the Transformers modeling backend require "
f"transformers>={required}, but got {installed}"
)
check_implementation(
hf_runner,
vllm_runner,
@@ -132,6 +127,42 @@ def test_hybrid_attention(vllm_runner: type[VllmRunner]) -> None:
)
def test_mla(vllm_runner: type[VllmRunner], example_prompts: list[str]) -> None:
import transformers
from packaging.version import Version
installed = Version(transformers.__version__)
required = Version("5.15.0.dev0")
if installed < required:
pytest.skip(
"MLA models with the Transformers modeling backend require "
f"transformers>={required}, but got {installed}"
)
model = get_model("DeepseekV2ForCausalLM") # DeepSeek-V2-Lite, MLA + MoE
args = (example_prompts, 32, 5)
kwargs: dict[str, Any] = {"max_model_len": 2048, "enforce_eager": True}
with vllm_runner(
model, model_impl="transformers", trust_remote_code=False, **kwargs
) as model_test:
model_config = model_test.llm.llm_engine.model_config
assert model_config.using_transformers_backend()
num_layers = model_config.hf_config.get_text_config().num_hidden_layers
assert model_test.apply_model(count_mla_layers) == [num_layers]
outputs_test = model_test.generate_greedy_logprobs(*args)
with vllm_runner(model, model_impl="auto") as model_ref:
outputs_ref = model_ref.generate_greedy_logprobs(*args)
check_logprobs_close(
outputs_0_lst=outputs_ref,
outputs_1_lst=outputs_test,
name_0="native",
name_1="transformers",
)
@multi_gpu_test(num_gpus=2)
def test_distributed(
hf_runner: type[HfRunner],