[Model] add GLM-5.3-Flash support (#53906)

This commit is contained in:
Jiangyun Zhu
2026-09-03 09:40:35 -07:00
committed by GitHub
parent da8ec28268
commit 98ed0856f3
94 changed files with 17927 additions and 266 deletions
+61 -5
View File
@@ -40,11 +40,13 @@ if not current_platform.is_cuda():
allow_module_level=True,
)
import vllm.v1.attention.backends.mla.flashinfer_mla_sparse as flashinfer_sparse_mod
from vllm.model_executor.layers.attention.mla_attention import (
_canonicalize_sparse_mla_kv_cache_dtype,
)
from vllm.utils.math_utils import cdiv
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
FlashInferMLASparseImpl,
FlashInferMLASparseTRTLLMBackend,
)
from vllm.v1.attention.backends.mla.flashmla_sparse import (
@@ -82,6 +84,60 @@ SPARSE_BACKEND_BATCH_SPECS["large_q_pure_prefill"] = BatchSpec(
DEVICE_TYPE = current_platform.device_type
def test_nope_flashinfer_sparse_mla_uses_model_scale(monkeypatch):
"""Weight absorption must not change the model's attention temperature."""
model_scale = 256**-0.5
kv_lora_rank = 512
topk = torch.zeros((1, 1), dtype=torch.int32)
metadata = SimpleNamespace(
req_id_per_token=torch.zeros(1, dtype=torch.int32),
block_table=torch.zeros((1, 1), dtype=torch.int32),
block_size=1,
)
recorded_scale = None
impl = object.__new__(FlashInferMLASparseImpl)
impl.scale = model_scale
impl.qk_nope_head_dim = 256
impl.kv_lora_rank = kv_lora_rank
impl.qk_rope_head_dim = 0
impl.kv_cache_dtype = "auto"
impl.topk_indices_buffer = topk
impl.dcp_world_size = 1
impl._workspace_buffer = torch.empty(1)
impl.bmm1_scale = None
impl.bmm2_scale = None
impl.is_nope_mla = True
impl.need_to_return_lse_for_decode = False
monkeypatch.setattr(
flashinfer_sparse_mod,
"triton_convert_req_index_to_global_index",
lambda *args, **kwargs: (topk, torch.ones(1, dtype=torch.int32)),
)
import flashinfer.decode
def fake_flashinfer(**kwargs):
nonlocal recorded_scale
recorded_scale = kwargs["bmm1_scale"]
return torch.zeros((1, 1, 1, kv_lora_rank))
monkeypatch.setattr(
flashinfer.decode,
"trtllm_batch_decode_with_kv_cache_mla",
fake_flashinfer,
)
impl.forward_mqa(
torch.zeros(1, 1, kv_lora_rank),
torch.zeros(1, 1, kv_lora_rank),
metadata,
SimpleNamespace(),
)
assert recorded_scale == model_scale
assert recorded_scale != kv_lora_rank**-0.5
def _float_to_e8m0_truncate(f: float) -> float:
"""Simulate SM100's float -> e8m0 -> bf16 scale conversion.
e8m0 format only stores the exponent (power of 2).
@@ -1451,14 +1507,14 @@ def test_triton_convert_returns_valid_counts(num_topk_tokens: int):
return_valid_counts=False,
)
assert isinstance(result_only, torch.Tensor)
torch.testing.assert_close(result_only, result, rtol=0, atol=0)
for row, num_valid in enumerate(expected_valid):
compact_valid = result[row, :num_valid].sort().values
original_valid = result_only[row][result_only[row] >= 0].sort().values
torch.testing.assert_close(compact_valid, original_valid, rtol=0, atol=0)
assert torch.all(result[row, num_valid:] == -1)
def test_flashmla_cache_dtype_aliases_use_ds_layout():
from vllm.model_executor.layers.attention.mla_attention import (
_canonicalize_sparse_mla_kv_cache_dtype,
)
# kv-cache dtype aliases are canonicalized to fp8_ds_mla before the layer
# stores kv_cache_dtype, so they cannot bypass the gate.
for alias in ("fp8", "fp8_e4m3"):