[Perf][GDN] Internal prefill checkpoints for the FlashInfer GDN prefill backend (#60050)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
a2fb482732
commit
badfcdbf91
@@ -13,10 +13,19 @@ if current_platform.is_rocm():
|
||||
)
|
||||
|
||||
import flashinfer.gdn_prefill # noqa: E402
|
||||
import torch.nn.functional as F # noqa: E402
|
||||
|
||||
from vllm.model_executor.layers.mamba.checkpoint import ( # noqa: E402
|
||||
MambaPrefillCheckpointExporter,
|
||||
MambaPrefillCheckpointMetadata,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import (
|
||||
fi_chunk_gated_delta_rule,
|
||||
) # noqa: E402
|
||||
from vllm.v1.attention.backends.gdn_attn import ( # noqa: E402
|
||||
plan_flashinfer_checkpoints,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID # noqa: E402
|
||||
|
||||
|
||||
def test_flashinfer_gdn_prefill_uses_int64_cu_seqlens(monkeypatch):
|
||||
@@ -51,3 +60,130 @@ def test_flashinfer_gdn_prefill_uses_int64_cu_seqlens(monkeypatch):
|
||||
assert captured_cu_seqlens.dtype == torch.int64
|
||||
assert output.shape == q.shape
|
||||
assert final_state is None
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not (
|
||||
current_platform.is_device_capability(90)
|
||||
or current_platform.is_device_capability_family(100)
|
||||
or current_platform.is_device_capability_family(120)
|
||||
),
|
||||
reason="FlashInfer GDN prefill checkpoints need SM90, SM100 or SM120.",
|
||||
)
|
||||
@pytest.mark.parametrize("use_cp", [True, False])
|
||||
@pytest.mark.parametrize("ssm_dtype", [torch.float32, torch.bfloat16])
|
||||
def test_flashinfer_checkpoints_export_the_state_at_each_offset(use_cp, ssm_dtype):
|
||||
"""Each exported checkpoint is the state of the prefill stopped there.
|
||||
|
||||
The leading decode row is outside the prefill kernel. The two checkpointed
|
||||
prefills put FlashInfer's stride (their offsets' gcd) below both offsets,
|
||||
so the exporter must pick each state out of the shared emitted buffer;
|
||||
the third prefill takes no checkpoint. Both FlashInfer routes are covered
|
||||
because ``use_cp="auto"`` may take either. The conv cache has MTP's spec
|
||||
slots after the window, which the conv kernels leave alone.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
device = "cuda"
|
||||
num_qk_heads, num_v_heads, head_dim, conv_dim, state_len = 2, 4, 128, 16, 3
|
||||
num_spec = 2
|
||||
offsets = [0, 128, 0, 192]
|
||||
query_lens = [200, 50, 300]
|
||||
state_indices = [NULL_BLOCK_ID, 3, NULL_BLOCK_ID, 5]
|
||||
num_blocks = 8
|
||||
|
||||
num_tokens = sum(query_lens)
|
||||
q = F.normalize(
|
||||
torch.randn(1, num_tokens, num_qk_heads, head_dim, device=device), dim=-1
|
||||
).bfloat16()
|
||||
k = F.normalize(
|
||||
torch.randn(1, num_tokens, num_qk_heads, head_dim, device=device), dim=-1
|
||||
).bfloat16()
|
||||
v = torch.randn(1, num_tokens, num_v_heads, head_dim, device=device).bfloat16()
|
||||
g = F.logsigmoid(torch.randn(1, num_tokens, num_v_heads, device=device))
|
||||
beta = torch.sigmoid(torch.randn(1, num_tokens, num_v_heads, device=device))
|
||||
initial_state = 0.1 * torch.randn(
|
||||
len(query_lens), num_v_heads, head_dim, head_dim, device=device
|
||||
)
|
||||
prefill_cu = [0]
|
||||
for query_len in query_lens:
|
||||
prefill_cu.append(prefill_cu[-1] + query_len)
|
||||
|
||||
every_n_tokens, cu_starts, rows = plan_flashinfer_checkpoints(offsets, query_lens)
|
||||
state_checkpoints = torch.empty(
|
||||
cu_starts[-1], num_v_heads, head_dim, head_dim, device=device
|
||||
)
|
||||
fi_chunk_gated_delta_rule(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g,
|
||||
beta=beta,
|
||||
initial_state=initial_state,
|
||||
output_final_state=True,
|
||||
cu_seqlens=torch.tensor(prefill_cu, device=device),
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
state_checkpoints=state_checkpoints,
|
||||
checkpoint_cu_starts=torch.tensor(cu_starts, device=device),
|
||||
checkpoint_every_n_tokens=every_n_tokens,
|
||||
use_cp=use_cp,
|
||||
)
|
||||
|
||||
# The exporter sees the decode row's token in front of the prefills.
|
||||
raw_qkv = torch.randn(1 + num_tokens, conv_dim, device=device).bfloat16()
|
||||
cu_seqlens = torch.tensor([0] + [1 + n for n in prefill_cu], device=device)
|
||||
conv_state = torch.zeros(
|
||||
num_blocks,
|
||||
state_len + num_spec,
|
||||
conv_dim,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
).transpose(-1, -2)
|
||||
ssm_state = torch.zeros(
|
||||
num_blocks, num_v_heads, head_dim, head_dim, device=device, dtype=ssm_dtype
|
||||
)
|
||||
MambaPrefillCheckpointExporter(state_len=state_len).export(
|
||||
MambaPrefillCheckpointMetadata(
|
||||
checkpoint_offsets=torch.tensor(offsets, device=device, dtype=torch.int32),
|
||||
state_indices=torch.tensor(state_indices, device=device),
|
||||
),
|
||||
raw_qkv=raw_qkv,
|
||||
conv_state=conv_state,
|
||||
recurrent_checkpoint=state_checkpoints,
|
||||
recurrent_state=ssm_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
recurrent_rows=torch.tensor(rows, device=device),
|
||||
)
|
||||
|
||||
written = []
|
||||
for row, (offset, block) in enumerate(zip(offsets, state_indices)):
|
||||
if not offset:
|
||||
continue
|
||||
written.append(block)
|
||||
seq = row - 1
|
||||
start = prefill_cu[seq]
|
||||
_, expected_state = fi_chunk_gated_delta_rule(
|
||||
q=q[:, start : start + offset],
|
||||
k=k[:, start : start + offset],
|
||||
v=v[:, start : start + offset],
|
||||
g=g[:, start : start + offset],
|
||||
beta=beta[:, start : start + offset],
|
||||
initial_state=initial_state[seq : seq + 1],
|
||||
output_final_state=True,
|
||||
cu_seqlens=torch.tensor([0, offset], device=device),
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
ssm_state[block].float(), expected_state[0], atol=2e-2, rtol=2e-2
|
||||
)
|
||||
end = cu_seqlens[row].item() + offset
|
||||
torch.testing.assert_close(
|
||||
conv_state[block, :, :state_len],
|
||||
raw_qkv[end - state_len : end].T,
|
||||
atol=0,
|
||||
rtol=0,
|
||||
)
|
||||
assert not conv_state[block, :, state_len:].any()
|
||||
|
||||
untouched = [i for i in range(num_blocks) if i not in written]
|
||||
assert not ssm_state[untouched].any()
|
||||
assert not conv_state[untouched].any()
|
||||
|
||||
@@ -21,6 +21,7 @@ from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.gdn_attn import (
|
||||
GDNAttentionMetadata,
|
||||
GDNAttentionMetadataBuilder,
|
||||
plan_flashinfer_checkpoints,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import mamba_get_block_table_tensor
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
@@ -488,3 +489,20 @@ def test_update_block_table_regathers_checkpoint(num_spec):
|
||||
assert not torch.equal(
|
||||
actual.checkpoint.state_indices, source.checkpoint.state_indices
|
||||
)
|
||||
|
||||
|
||||
def test_plan_flashinfer_checkpoints_selects_the_state_at_each_offset():
|
||||
"""Each checkpoint row maps to the state FlashInfer emits at its offset.
|
||||
|
||||
The leading decode row is not in the prefill kernel and never checkpoints;
|
||||
the 50-token prefill is shorter than the stride and emits no states.
|
||||
"""
|
||||
every_n_tokens, cu_starts, rows = plan_flashinfer_checkpoints(
|
||||
offsets=[0, 128, 0, 192], query_lens=[200, 50, 300]
|
||||
)
|
||||
|
||||
assert every_n_tokens == 64
|
||||
assert cu_starts == [0, 3, 3, 7]
|
||||
# Row 1: state after 128 tokens of the first prefill. Row 3: after 192
|
||||
# tokens of the third prefill, whose states start at index 3.
|
||||
assert rows == [0, 1, 0, 5]
|
||||
|
||||
@@ -16,7 +16,10 @@ from tests.utils import create_new_process_for_each_test
|
||||
from vllm import LLM, SamplingParams, TokensPrompt
|
||||
from vllm.config import CacheConfig
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateCopyFunc,
|
||||
is_conv_state_dim_first,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
@@ -59,6 +62,10 @@ MODEL = "Qwen/Qwen3-Next-80B-A3B-Instruct-FP8"
|
||||
# where block boundaries fall. XPU rounds GDN block sizes up to a multiple of 64
|
||||
# (XPUPlatform.update_block_size_for_backend), so use one.
|
||||
BLOCK_SIZE = 576
|
||||
# The step actions also encode the split at the prompt's last block boundary,
|
||||
# which a GDN prefill backend with internal checkpoints (FlashInfer) skips; see
|
||||
# get_internal_checkpoint_step_configs.
|
||||
GDN_PREFILL_CONFIG = {"gdn_prefill_backend": "triton"}
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
NUM_HIDDEN_LAYERS = 1
|
||||
cur_step_action_idx = 0
|
||||
@@ -882,6 +889,73 @@ def get_mamba_prefix_cache_step_configs(
|
||||
return tests
|
||||
|
||||
|
||||
def get_internal_checkpoint_step_configs() -> dict[str, TestConfig]:
|
||||
"""Async-scheduling steps when the GDN prefill exports the checkpoint at
|
||||
the prompt's last block boundary itself, so the final chunk runs whole.
|
||||
|
||||
The checkpoint goes in the second-to-last block the chunk reaches, keyed to
|
||||
the boundary: column 1 holds the state at B for a 2B + 10 prompt.
|
||||
"""
|
||||
B = BLOCK_SIZE
|
||||
return {
|
||||
"prompt_2_block_size": TestConfig(
|
||||
num_prompt_tokens=B * 2,
|
||||
num_generated_tokens=10,
|
||||
num_accepted_tokens=4,
|
||||
step_actions=[
|
||||
StepAction(0, B * 2, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(B * 2, 4, [1, 1, 1, 1, 1, 1], (1, 2), (-1, -1)),
|
||||
],
|
||||
),
|
||||
"prompt_2_block_size_10": TestConfig(
|
||||
num_prompt_tokens=B * 2 + 10,
|
||||
num_generated_tokens=10,
|
||||
num_accepted_tokens=4,
|
||||
step_actions=[
|
||||
StepAction(0, B * 2 + 10, [0, 1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(B * 2 + 10, 4, [0, 1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
],
|
||||
),
|
||||
"prompt_10_block_size": TestConfig(
|
||||
num_prompt_tokens=B * 10,
|
||||
num_generated_tokens=10,
|
||||
num_accepted_tokens=4,
|
||||
step_actions=[
|
||||
StepAction(0, B * 5, [0, 0, 0, 0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(
|
||||
B * 5,
|
||||
B * 5,
|
||||
[0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1],
|
||||
(4, 9),
|
||||
(-1, -1),
|
||||
),
|
||||
StepAction(
|
||||
B * 10,
|
||||
4,
|
||||
[0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1],
|
||||
(9, 10),
|
||||
(-1, -1),
|
||||
),
|
||||
],
|
||||
),
|
||||
"prompt_10_block_size_10": TestConfig(
|
||||
num_prompt_tokens=B * 10 + 10,
|
||||
num_generated_tokens=10,
|
||||
num_accepted_tokens=4,
|
||||
step_actions=[
|
||||
StepAction(0, B * 5, [0, 0, 0, 0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(
|
||||
B * 5,
|
||||
B * 5 + 10,
|
||||
[0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1],
|
||||
(4, 10),
|
||||
(-1, -1),
|
||||
),
|
||||
],
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _run_mamba_prefix_cache_mrv1(
|
||||
monkeypatch: pytest.MonkeyPatch, async_scheduling: bool
|
||||
):
|
||||
@@ -902,6 +976,7 @@ def _run_mamba_prefix_cache_mrv1(
|
||||
load_format="dummy",
|
||||
enable_prefix_caching=True,
|
||||
block_size=BLOCK_SIZE,
|
||||
additional_config=GDN_PREFILL_CONFIG,
|
||||
speculative_config={
|
||||
"method": "qwen3_next_mtp",
|
||||
"num_speculative_tokens": num_speculative_tokens,
|
||||
@@ -957,8 +1032,11 @@ def test_mamba_prefix_cache_mrv1_async(monkeypatch: pytest.MonkeyPatch):
|
||||
|
||||
|
||||
def _run_mamba_prefix_cache_mrv2(
|
||||
monkeypatch: pytest.MonkeyPatch, async_scheduling: bool
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
async_scheduling: bool,
|
||||
additional_config: dict[str, Any] = GDN_PREFILL_CONFIG,
|
||||
):
|
||||
internal_checkpoints = additional_config["gdn_prefill_backend"] == "flashinfer"
|
||||
global async_scheduling_mode
|
||||
async_scheduling_mode = async_scheduling
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
@@ -1098,8 +1176,24 @@ def _run_mamba_prefix_cache_mrv2(
|
||||
with gpu_sync_allowed():
|
||||
start_pos = input_batch.positions[input_batch.query_start_loc[0]].item()
|
||||
assert cur_step_action.num_computed_tokens_start == start_pos
|
||||
if scheduler_output.scheduled_new_reqs and "snapshot_col" in captured:
|
||||
captured["snapshot"] = block_states(self, captured.pop("snapshot_col"))
|
||||
return ret
|
||||
|
||||
def block_states(runner: MRV2GPUModelRunner, col: int) -> list[list[torch.Tensor]]:
|
||||
# Each Mamba layer's (conv, temporal) states in block column col.
|
||||
kv_cache_config = captured["kv_cache_config"]
|
||||
forward_context = runner.vllm_config.compilation_config.static_forward_context
|
||||
return [
|
||||
[
|
||||
temporal_block(state, captured["block_tables"][group_id], col).clone()
|
||||
for state in forward_context[layer_name].kv_cache
|
||||
]
|
||||
for group_ids in get_mamba_groups(kv_cache_config).values()
|
||||
for group_id in group_ids
|
||||
for layer_name in kv_cache_config.kv_cache_groups[group_id].layer_names
|
||||
]
|
||||
|
||||
def fake_sample(
|
||||
self: MRV2GPUModelRunner,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -1161,6 +1255,7 @@ def _run_mamba_prefix_cache_mrv2(
|
||||
skip_tokenizer_init=True,
|
||||
enable_prefix_caching=True,
|
||||
block_size=BLOCK_SIZE,
|
||||
additional_config=additional_config,
|
||||
speculative_config={
|
||||
"method": "qwen3_next_mtp",
|
||||
"num_speculative_tokens": num_speculative_tokens,
|
||||
@@ -1173,7 +1268,11 @@ def _run_mamba_prefix_cache_mrv2(
|
||||
)
|
||||
|
||||
try:
|
||||
tests = get_mamba_prefix_cache_step_configs(async_scheduling)
|
||||
tests = (
|
||||
get_internal_checkpoint_step_configs()
|
||||
if internal_checkpoints
|
||||
else get_mamba_prefix_cache_step_configs(async_scheduling)
|
||||
)
|
||||
|
||||
global step_actions
|
||||
global cur_step_action_idx
|
||||
@@ -1219,6 +1318,35 @@ def _run_mamba_prefix_cache_mrv2(
|
||||
f"got events={second_events!r}"
|
||||
)
|
||||
assert engine.llm_engine.engine_core.engine_core.scheduler.reset_prefix_cache()
|
||||
|
||||
if internal_checkpoints:
|
||||
# The checkpoint a 2B + 10 prefill exports at B (column 1) is the
|
||||
# final state of a prefill that stops at B (column 0). Distinct
|
||||
# tokens make a shifted conv window visible.
|
||||
tokens = [(i * 7919) % 30000 + 1 for i in range(BLOCK_SIZE * 2 + 10)]
|
||||
scheduler = engine.llm_engine.engine_core.engine_core.scheduler
|
||||
states = []
|
||||
for num_tokens, col in ((BLOCK_SIZE, 0), (len(tokens), 1)):
|
||||
captured["snapshot_col"] = col
|
||||
_ = engine.generate(
|
||||
[TokensPrompt(prompt_token_ids=tokens[:num_tokens])],
|
||||
sampling_params=sampling_params,
|
||||
)
|
||||
states.append(captured.pop("snapshot"))
|
||||
assert scheduler.reset_prefix_cache()
|
||||
hf_config = engine.llm_engine.model_config.hf_text_config
|
||||
state_len = hf_config.linear_conv_kernel_dim - 1
|
||||
for (ref_conv, ref_ssm), (conv, ssm) in zip(*states):
|
||||
if not is_conv_state_dim_first():
|
||||
ref_conv, conv = ref_conv.mT, conv.mT
|
||||
# Under MTP the conv state has num_spec scratch columns after
|
||||
# the window.
|
||||
for new, ref in (
|
||||
(conv[..., :state_len], ref_conv[..., :state_len]),
|
||||
(ssm, ref_ssm),
|
||||
):
|
||||
rel_err = (new.float() - ref.float()).norm() / ref.float().norm()
|
||||
assert rel_err < 1e-2, rel_err
|
||||
finally:
|
||||
del engine
|
||||
torch.accelerator.empty_cache()
|
||||
@@ -1233,3 +1361,26 @@ def test_mamba_prefix_cache_mrv2(monkeypatch: pytest.MonkeyPatch):
|
||||
@create_new_process_for_each_test()
|
||||
def test_mamba_prefix_cache_mrv2_async(monkeypatch: pytest.MonkeyPatch):
|
||||
_run_mamba_prefix_cache_mrv2(monkeypatch, async_scheduling=True)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda()
|
||||
or not (
|
||||
current_platform.is_device_capability(90)
|
||||
or (
|
||||
(
|
||||
current_platform.is_device_capability_family(100)
|
||||
or current_platform.is_device_capability_family(120)
|
||||
)
|
||||
and current_platform.get_cuda_runtime_major() >= 13
|
||||
)
|
||||
),
|
||||
reason="FlashInfer GDN prefill needs SM90, or SM100/SM120 with CUDA 13.",
|
||||
)
|
||||
@create_new_process_for_each_test()
|
||||
def test_mamba_prefix_cache_mrv2_async_flashinfer(monkeypatch: pytest.MonkeyPatch):
|
||||
_run_mamba_prefix_cache_mrv2(
|
||||
monkeypatch,
|
||||
async_scheduling=True,
|
||||
additional_config={"gdn_prefill_backend": "flashinfer"},
|
||||
)
|
||||
|
||||
@@ -29,6 +29,7 @@ from vllm.distributed.parallel_state import (
|
||||
from vllm.lora.layers import LoRAMappingType
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.mamba.abstract import MambaBase
|
||||
from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange
|
||||
from vllm.platforms import current_platform
|
||||
@@ -42,7 +43,10 @@ from vllm.v1.attention.backends.mla.indexer import DeepseekV32IndexerBackend
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
|
||||
ROCMAiterMLASparseBackend,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.attention.backends.registry import (
|
||||
AttentionBackendEnum,
|
||||
MambaAttentionBackendEnum,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_planning import (
|
||||
estimate_max_model_len,
|
||||
get_kv_cache_configs,
|
||||
@@ -512,6 +516,53 @@ def test_alignment_rejected_by_a_sibling_backend_raises(monkeypatch, cpu):
|
||||
platform.update_block_size_for_backend(vllm_config)
|
||||
|
||||
|
||||
class _CheckpointingMambaLayer(MambaBase):
|
||||
mamba_type = MambaAttentionBackendEnum.GDN_ATTN
|
||||
|
||||
def __init__(self, prefill_checkpoint_alignment: int | None):
|
||||
self._prefill_checkpoint_alignment = prefill_checkpoint_alignment
|
||||
|
||||
def get_prefill_checkpoint_alignment(self) -> int | None:
|
||||
return self._prefill_checkpoint_alignment
|
||||
|
||||
def get_state_shape(self):
|
||||
return ()
|
||||
|
||||
def get_state_dtype(self):
|
||||
return ()
|
||||
|
||||
|
||||
def _align_qwen3_5_blocks(*prefill_checkpoint_alignments: int | None):
|
||||
from tests.v1.attention.utils import create_vllm_config
|
||||
|
||||
vllm_config = create_vllm_config(model_name="Qwen/Qwen3.5-0.8B")
|
||||
cache_config = vllm_config.cache_config
|
||||
cache_config.mamba_cache_mode = "align"
|
||||
forward_context = vllm_config.compilation_config.static_forward_context
|
||||
for i, alignment in enumerate(prefill_checkpoint_alignments):
|
||||
forward_context[f"mamba.{i}"] = _CheckpointingMambaLayer(alignment)
|
||||
Platform._align_hybrid_block_size(vllm_config, _mock_backend([MultipleOf(16)]))
|
||||
return cache_config
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("prefill_checkpoint_alignments", "block_size"),
|
||||
[((64,), 576), ((None,), 544), ((64, None), 544)],
|
||||
)
|
||||
def test_blocks_land_on_the_mamba_checkpoint_grid(
|
||||
prefill_checkpoint_alignments, block_size
|
||||
):
|
||||
"""A layer that checkpoints every 64 tokens needs 64-aligned blocks.
|
||||
|
||||
Unaligned, Qwen3.5-0.8B's block (544) puts every other checkpoint off the
|
||||
grid. A Mamba layer that cannot checkpoint (like Qwen4Exp's PLE short conv)
|
||||
stops the scheduler from taking any, so the block stays.
|
||||
"""
|
||||
cache_config = _align_qwen3_5_blocks(*prefill_checkpoint_alignments)
|
||||
|
||||
assert cache_config.block_size == block_size
|
||||
|
||||
|
||||
def test_xpu_gdn_rounding_rejected_by_a_sibling_backend_raises(monkeypatch):
|
||||
# XPU's GDN rounding runs after super()'s check, so it must check again.
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user