[Model] add GLM-5.3-Flash support (#53906)
This commit is contained in:
@@ -143,15 +143,22 @@ def test_reshape_padded_kv_cache_strides_by_padded_page():
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kernel_block_sizes", "expected_num_blocks", "expected_num_states"),
|
||||
(
|
||||
"kernel_block_sizes",
|
||||
"storage_block_size",
|
||||
"expected_num_blocks",
|
||||
"expected_num_states",
|
||||
),
|
||||
[
|
||||
(None, 4, 64),
|
||||
([256], 4, 64),
|
||||
([64], 16, 16),
|
||||
(None, None, 4, 64),
|
||||
([256], None, 4, 64),
|
||||
([64], None, 16, 16),
|
||||
([64], 256, 4, 64),
|
||||
],
|
||||
)
|
||||
def test_allocate_compressed_mla_cache(
|
||||
kernel_block_sizes: list[int] | None,
|
||||
storage_block_size: int | None,
|
||||
expected_num_blocks: int,
|
||||
expected_num_states: int,
|
||||
):
|
||||
@@ -161,6 +168,7 @@ def test_allocate_compressed_mla_cache(
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
tokens_per_state=4,
|
||||
storage_block_size=storage_block_size,
|
||||
)
|
||||
num_pages = 4
|
||||
config = KVCacheConfig(
|
||||
|
||||
@@ -76,6 +76,7 @@ def test_packed_dsv4_zeroer_zeroes_only_each_layers_page():
|
||||
static_forward_context={
|
||||
f"layer.{i}": SimpleNamespace(kv_cache=views[i]) for i in range(NUM_LAYERS)
|
||||
},
|
||||
num_blocks=NUM_BLOCKS,
|
||||
)
|
||||
seg_addrs, seg_block_strides, seg_page_sizes, _, _, n_segs = zeroer._meta
|
||||
|
||||
@@ -153,6 +154,7 @@ def test_overlaid_zeroer_dedups_segments_with_max_span():
|
||||
static_forward_context={
|
||||
name: SimpleNamespace(kv_cache=views[name]) for name in views
|
||||
},
|
||||
num_blocks=config.num_blocks,
|
||||
)
|
||||
seg_addrs, seg_block_strides, seg_page_sizes, _, _, n_segs = zeroer._meta
|
||||
|
||||
|
||||
@@ -54,6 +54,7 @@ def test_attention_blocks_are_zeroed(spec):
|
||||
static_forward_context={
|
||||
layer_name: SimpleNamespace(kv_cache=storage),
|
||||
},
|
||||
num_blocks=4,
|
||||
)
|
||||
|
||||
zeroer.zero_block_ids([1])
|
||||
@@ -64,6 +65,56 @@ def test_attention_blocks_are_zeroed(spec):
|
||||
assert torch.equal(storage, expected)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
def test_layers_in_one_group_may_use_different_kernel_pages_per_block():
|
||||
"""Derive kernel pages per block from each layer's allocation."""
|
||||
device = torch.device("cuda")
|
||||
num_blocks = 4
|
||||
spec = SlidingWindowSpec(
|
||||
block_size=2,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.int32,
|
||||
sliding_window=2,
|
||||
)
|
||||
# Two kernel pages per logical block.
|
||||
wide = torch.ones((num_blocks * 2, 3), dtype=torch.int32, device=device)
|
||||
# One kernel page per logical block, carved out of a larger allocation so
|
||||
# an over-strided write lands in the guard region instead of faulting or
|
||||
# silently hitting another tensor.
|
||||
narrow_backing = torch.ones((num_blocks * 3, 5), dtype=torch.int32, device=device)
|
||||
narrow = narrow_backing[:num_blocks]
|
||||
|
||||
zeroer = KVBlockZeroer(
|
||||
device,
|
||||
attn_groups_iter=[
|
||||
AttentionGroup(
|
||||
None,
|
||||
["wide", "narrow"],
|
||||
spec,
|
||||
0,
|
||||
)
|
||||
],
|
||||
kernel_block_sizes=[1],
|
||||
static_forward_context={
|
||||
"wide": SimpleNamespace(kv_cache=wide),
|
||||
"narrow": SimpleNamespace(kv_cache=narrow),
|
||||
},
|
||||
num_blocks=num_blocks,
|
||||
)
|
||||
|
||||
zeroer.zero_block_ids([num_blocks - 1])
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
expected_wide = torch.ones_like(wide)
|
||||
expected_wide[2 * (num_blocks - 1) :] = 0
|
||||
assert torch.equal(wide, expected_wide)
|
||||
|
||||
expected_narrow = torch.ones_like(narrow_backing)
|
||||
expected_narrow[num_blocks - 1] = 0
|
||||
assert torch.equal(narrow_backing, expected_narrow)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
def test_block_ids_are_not_overwritten_while_copy_is_in_flight():
|
||||
device = torch.device("cuda")
|
||||
@@ -207,6 +258,7 @@ def test_large_dsv4_launch_geometry(monkeypatch):
|
||||
name: SimpleNamespace(kv_cache=storage)
|
||||
for name, storage in storages.items()
|
||||
},
|
||||
num_blocks=1,
|
||||
)
|
||||
|
||||
assert zeroer._meta is not None
|
||||
@@ -332,6 +384,7 @@ def test_zeroes_exactly_one_block_per_layer(layout: KVCacheLayout):
|
||||
attn_groups_iter=iter(groups),
|
||||
kernel_block_sizes=[spec.block_size],
|
||||
static_forward_context=ctx,
|
||||
num_blocks=num_blocks,
|
||||
)
|
||||
zeroer.zero_block_ids([2])
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
@@ -7,15 +7,57 @@ from unittest.mock import Mock
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.recoverssm_metadata import (
|
||||
RecoverSSMMetadata,
|
||||
RecoverSSMPostprocessMetadata,
|
||||
)
|
||||
from vllm.v1.worker.gpu.model_states import mamba_hybrid
|
||||
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
|
||||
from vllm.v1.worker.gpu.model_states.recoverssm import RecoverSSMState
|
||||
|
||||
|
||||
def test_prepare_attn_forwards_positions(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
state = object.__new__(MambaHybridModelState)
|
||||
state.vllm_config = SimpleNamespace(num_speculative_tokens=0)
|
||||
state.max_model_len = 8192
|
||||
state._align_mode = False
|
||||
state.recoverssm = None
|
||||
|
||||
positions = torch.tensor([1536], dtype=torch.int64)
|
||||
input_batch = SimpleNamespace(
|
||||
num_reqs=1,
|
||||
num_tokens=1,
|
||||
num_reqs_after_padding=1,
|
||||
num_tokens_after_padding=1,
|
||||
query_start_loc_np=torch.tensor([0, 1], dtype=torch.int32).numpy(),
|
||||
query_start_loc=torch.tensor([0, 1], dtype=torch.int32),
|
||||
num_scheduled_tokens=torch.tensor([1], dtype=torch.int32),
|
||||
seq_lens_cpu_upper_bound=torch.tensor([1537], dtype=torch.int32),
|
||||
seq_lens=torch.tensor([1537], dtype=torch.int32),
|
||||
is_prefilling_np=torch.tensor([False]).numpy(),
|
||||
dcp_local_seq_lens=None,
|
||||
positions=positions,
|
||||
prompt_lens=torch.tensor([1024], dtype=torch.int32),
|
||||
)
|
||||
expected_metadata = {"layer": object()}
|
||||
build_attn_metadata = Mock(return_value=expected_metadata)
|
||||
monkeypatch.setattr(mamba_hybrid, "build_attn_metadata", build_attn_metadata)
|
||||
|
||||
metadata = state.prepare_attn(
|
||||
input_batch=input_batch,
|
||||
cudagraph_mode=CUDAGraphMode.NONE,
|
||||
block_tables=(),
|
||||
slot_mappings=torch.empty(0, dtype=torch.int64),
|
||||
attn_groups=[],
|
||||
kv_cache_config=Mock(),
|
||||
)
|
||||
|
||||
assert metadata is expected_metadata
|
||||
assert build_attn_metadata.call_args.kwargs["positions"] is positions
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
|
||||
@pytest.mark.parametrize(("num_sampled", "expected_value"), [(0, 1), (3, 3)])
|
||||
def test_postprocess_state_scalar_with_int32_mapping(
|
||||
|
||||
Reference in New Issue
Block a user