[Model] add GLM-5.3-Flash support (#53906)

This commit is contained in:
Jiangyun Zhu
2026-09-03 09:40:35 -07:00
committed by GitHub
parent da8ec28268
commit 98ed0856f3
94 changed files with 17927 additions and 266 deletions
+12 -4
View File
@@ -143,15 +143,22 @@ def test_reshape_padded_kv_cache_strides_by_padded_page():
@pytest.mark.parametrize(
("kernel_block_sizes", "expected_num_blocks", "expected_num_states"),
(
"kernel_block_sizes",
"storage_block_size",
"expected_num_blocks",
"expected_num_states",
),
[
(None, 4, 64),
([256], 4, 64),
([64], 16, 16),
(None, None, 4, 64),
([256], None, 4, 64),
([64], None, 16, 16),
([64], 256, 4, 64),
],
)
def test_allocate_compressed_mla_cache(
kernel_block_sizes: list[int] | None,
storage_block_size: int | None,
expected_num_blocks: int,
expected_num_states: int,
):
@@ -161,6 +168,7 @@ def test_allocate_compressed_mla_cache(
head_size=128,
dtype=torch.bfloat16,
tokens_per_state=4,
storage_block_size=storage_block_size,
)
num_pages = 4
config = KVCacheConfig(
@@ -76,6 +76,7 @@ def test_packed_dsv4_zeroer_zeroes_only_each_layers_page():
static_forward_context={
f"layer.{i}": SimpleNamespace(kv_cache=views[i]) for i in range(NUM_LAYERS)
},
num_blocks=NUM_BLOCKS,
)
seg_addrs, seg_block_strides, seg_page_sizes, _, _, n_segs = zeroer._meta
@@ -153,6 +154,7 @@ def test_overlaid_zeroer_dedups_segments_with_max_span():
static_forward_context={
name: SimpleNamespace(kv_cache=views[name]) for name in views
},
num_blocks=config.num_blocks,
)
seg_addrs, seg_block_strides, seg_page_sizes, _, _, n_segs = zeroer._meta
+53
View File
@@ -54,6 +54,7 @@ def test_attention_blocks_are_zeroed(spec):
static_forward_context={
layer_name: SimpleNamespace(kv_cache=storage),
},
num_blocks=4,
)
zeroer.zero_block_ids([1])
@@ -64,6 +65,56 @@ def test_attention_blocks_are_zeroed(spec):
assert torch.equal(storage, expected)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
def test_layers_in_one_group_may_use_different_kernel_pages_per_block():
"""Derive kernel pages per block from each layer's allocation."""
device = torch.device("cuda")
num_blocks = 4
spec = SlidingWindowSpec(
block_size=2,
num_kv_heads=1,
head_size=1,
dtype=torch.int32,
sliding_window=2,
)
# Two kernel pages per logical block.
wide = torch.ones((num_blocks * 2, 3), dtype=torch.int32, device=device)
# One kernel page per logical block, carved out of a larger allocation so
# an over-strided write lands in the guard region instead of faulting or
# silently hitting another tensor.
narrow_backing = torch.ones((num_blocks * 3, 5), dtype=torch.int32, device=device)
narrow = narrow_backing[:num_blocks]
zeroer = KVBlockZeroer(
device,
attn_groups_iter=[
AttentionGroup(
None,
["wide", "narrow"],
spec,
0,
)
],
kernel_block_sizes=[1],
static_forward_context={
"wide": SimpleNamespace(kv_cache=wide),
"narrow": SimpleNamespace(kv_cache=narrow),
},
num_blocks=num_blocks,
)
zeroer.zero_block_ids([num_blocks - 1])
torch.accelerator.synchronize()
expected_wide = torch.ones_like(wide)
expected_wide[2 * (num_blocks - 1) :] = 0
assert torch.equal(wide, expected_wide)
expected_narrow = torch.ones_like(narrow_backing)
expected_narrow[num_blocks - 1] = 0
assert torch.equal(narrow_backing, expected_narrow)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
def test_block_ids_are_not_overwritten_while_copy_is_in_flight():
device = torch.device("cuda")
@@ -207,6 +258,7 @@ def test_large_dsv4_launch_geometry(monkeypatch):
name: SimpleNamespace(kv_cache=storage)
for name, storage in storages.items()
},
num_blocks=1,
)
assert zeroer._meta is not None
@@ -332,6 +384,7 @@ def test_zeroes_exactly_one_block_per_layer(layout: KVCacheLayout):
attn_groups_iter=iter(groups),
kernel_block_sizes=[spec.block_size],
static_forward_context=ctx,
num_blocks=num_blocks,
)
zeroer.zero_block_ids([2])
torch.accelerator.synchronize()
@@ -7,15 +7,57 @@ from unittest.mock import Mock
import pytest
import torch
from vllm.config.compilation import CUDAGraphMode
from vllm.platforms import current_platform
from vllm.v1.attention.backends.recoverssm_metadata import (
RecoverSSMMetadata,
RecoverSSMPostprocessMetadata,
)
from vllm.v1.worker.gpu.model_states import mamba_hybrid
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
from vllm.v1.worker.gpu.model_states.recoverssm import RecoverSSMState
def test_prepare_attn_forwards_positions(monkeypatch: pytest.MonkeyPatch) -> None:
state = object.__new__(MambaHybridModelState)
state.vllm_config = SimpleNamespace(num_speculative_tokens=0)
state.max_model_len = 8192
state._align_mode = False
state.recoverssm = None
positions = torch.tensor([1536], dtype=torch.int64)
input_batch = SimpleNamespace(
num_reqs=1,
num_tokens=1,
num_reqs_after_padding=1,
num_tokens_after_padding=1,
query_start_loc_np=torch.tensor([0, 1], dtype=torch.int32).numpy(),
query_start_loc=torch.tensor([0, 1], dtype=torch.int32),
num_scheduled_tokens=torch.tensor([1], dtype=torch.int32),
seq_lens_cpu_upper_bound=torch.tensor([1537], dtype=torch.int32),
seq_lens=torch.tensor([1537], dtype=torch.int32),
is_prefilling_np=torch.tensor([False]).numpy(),
dcp_local_seq_lens=None,
positions=positions,
prompt_lens=torch.tensor([1024], dtype=torch.int32),
)
expected_metadata = {"layer": object()}
build_attn_metadata = Mock(return_value=expected_metadata)
monkeypatch.setattr(mamba_hybrid, "build_attn_metadata", build_attn_metadata)
metadata = state.prepare_attn(
input_batch=input_batch,
cudagraph_mode=CUDAGraphMode.NONE,
block_tables=(),
slot_mappings=torch.empty(0, dtype=torch.int64),
attn_groups=[],
kv_cache_config=Mock(),
)
assert metadata is expected_metadata
assert build_attn_metadata.call_args.kwargs["positions"] is positions
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
@pytest.mark.parametrize(("num_sampled", "expected_value"), [(0, 1), (3, 3)])
def test_postprocess_state_scalar_with_int32_mapping(