[Spec Decode] Enable Gemma4 DSpark adaptive verification with FlashInfer (#57263)
Signed-off-by: zixi-qi <zixi@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
Claude Opus 5.5
Nick Hill
parent
3b4566c5cf
commit
a4eb3f25d6
@@ -2,12 +2,14 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for v1 attention backends without GPUModelRunner dependency."""
|
||||
|
||||
from dataclasses import replace
|
||||
from functools import partial
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch.nn.attention.flex_attention import create_block_mask, flex_attention
|
||||
from transformers import LlamaConfig
|
||||
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
@@ -28,8 +30,10 @@ from vllm.utils.torch_utils import (
|
||||
)
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionCGSupport,
|
||||
AttentionMetadataBuilder,
|
||||
AttentionType,
|
||||
CommonAttentionMetadata,
|
||||
max_decode_query_len,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheLayout
|
||||
@@ -1066,6 +1070,378 @@ def test_flashinfer_trtllm_gen_padded_decode_uses_varlen_offsets(
|
||||
assert q_cu_seq_lens.tolist() == qo_indptr_values
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
||||
reason="FlashInfer is not available.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"adaptive,decode_kernel,dcp_size,expected_bound",
|
||||
[
|
||||
(None, "TRTLLM_GEN", 1, None),
|
||||
(False, "TRTLLM_GEN", 1, None),
|
||||
(True, "TRTLLM_GEN", 1, 8),
|
||||
(True, "XQA", 1, None),
|
||||
(True, "TRTLLM_GEN", 2, None),
|
||||
],
|
||||
)
|
||||
def test_flashinfer_varlen_cudagraph_capability(
|
||||
monkeypatch, adaptive, decode_kernel, dcp_size, expected_bound
|
||||
):
|
||||
"""Only the trtllm-gen path that adaptive verification enables replays
|
||||
varlen decode graphs, up to the decode width; the uniform level is fixed."""
|
||||
from vllm.model_executor.layers.attention.chunked_local_attention import (
|
||||
create_chunked_local_attention_backend,
|
||||
)
|
||||
from vllm.v1.attention.backends import flashinfer as fi
|
||||
|
||||
config = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(use_non_causal=False),
|
||||
speculative_config=SimpleNamespace(
|
||||
enable_adaptive_verification=adaptive,
|
||||
num_speculative_tokens=7,
|
||||
parallel_drafting=True,
|
||||
)
|
||||
if adaptive is not None
|
||||
else None,
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=dcp_size),
|
||||
model_config=SimpleNamespace(get_num_attention_heads=lambda _: 32),
|
||||
use_v2_model_runner=True,
|
||||
)
|
||||
monkeypatch.setattr(fi, "can_use_trtllm_attention", lambda *_, **__: True)
|
||||
monkeypatch.setattr(
|
||||
fi.FlashInferMetadataBuilder,
|
||||
"_get_flashinfer_trtllm_api_decode_kernel",
|
||||
staticmethod(lambda: fi.FlashInferDecodeKernel[decode_kernel]),
|
||||
)
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=4,
|
||||
head_size=64,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
builder_cls = fi.FlashInferMetadataBuilder
|
||||
assert builder_cls.get_cudagraph_support(config, spec) == (
|
||||
AttentionCGSupport.UNIFORM_BATCH
|
||||
if dcp_size == 1
|
||||
else AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
|
||||
)
|
||||
bound = builder_cls.get_varlen_cudagraph_max_query_len(config, spec)
|
||||
assert bound == expected_bound
|
||||
if bound is not None:
|
||||
# A wrapper forcing NEVER inherits the override but not the bound.
|
||||
chunked_local = create_chunked_local_attention_backend(fi.FlashInferBackend, 16)
|
||||
wrapped_cls = chunked_local.get_builder_cls()
|
||||
assert wrapped_cls.get_varlen_cudagraph_max_query_len(config, spec) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"use_v2_model_runner,parallel_drafting,expected",
|
||||
[
|
||||
(True, None, 1),
|
||||
(True, False, 8),
|
||||
(True, True, 8),
|
||||
(False, False, 8),
|
||||
(False, True, 15),
|
||||
],
|
||||
)
|
||||
def test_spec_as_decode_width_follows_model_runner(
|
||||
use_v2_model_runner, parallel_drafting, expected
|
||||
):
|
||||
"""Only model runner V1's parallel drafter widens decode requests."""
|
||||
|
||||
class Builder(AttentionMetadataBuilder):
|
||||
def __init__(self, vllm_config):
|
||||
self.vllm_config = vllm_config
|
||||
|
||||
def build(self, common_prefix_len, common_attn_metadata, fast_build=False):
|
||||
raise NotImplementedError
|
||||
|
||||
config = SimpleNamespace(
|
||||
speculative_config=None
|
||||
if parallel_drafting is None
|
||||
else SimpleNamespace(
|
||||
num_speculative_tokens=7, parallel_drafting=parallel_drafting
|
||||
),
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
|
||||
use_v2_model_runner=use_v2_model_runner,
|
||||
)
|
||||
assert max_decode_query_len(config) == expected
|
||||
builder = Builder(config)
|
||||
builder._init_reorder_batch_threshold(1, supports_spec_as_decode=True)
|
||||
assert builder.reorder_batch_threshold == expected
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST
|
||||
or not current_platform.is_device_capability_family(100),
|
||||
reason="TRTLLM-GEN varlen decode graphs require FlashInfer on SM100-family GPUs",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"head_size,sliding_window,num_tokens,num_reqs,num_heads,num_kv_heads",
|
||||
[
|
||||
pytest.param(64, None, 8, 8, 32, 4, id="single-token-capture"),
|
||||
pytest.param(
|
||||
256,
|
||||
1024,
|
||||
40,
|
||||
32,
|
||||
16,
|
||||
8,
|
||||
id="gemma4-sliding-gqa",
|
||||
),
|
||||
pytest.param(
|
||||
512,
|
||||
None,
|
||||
40,
|
||||
32,
|
||||
16,
|
||||
1,
|
||||
id="gemma4-global-mqa",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_flashinfer_varlen_decode_graph_replays_changed_layout(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
head_size,
|
||||
sliding_window,
|
||||
num_tokens,
|
||||
num_reqs,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
):
|
||||
"""Replay changes device offsets, active slots, context, and KV writes.
|
||||
|
||||
The graph is captured with evenly spread dummy queries and an explicit
|
||||
bound of eight. CPU offsets then remain stale while device offsets change.
|
||||
Compare real output rows with independent FP32 causal/windowed attention,
|
||||
and verify cache writes including preservation of the null padding block.
|
||||
"""
|
||||
from vllm.v1.attention.backends import flashinfer as fi
|
||||
from vllm.v1.attention.backends.utils import PerLayerParameters
|
||||
|
||||
set_random_seed(0)
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
block_size = 16
|
||||
dtype = torch.bfloat16
|
||||
LlamaConfig(
|
||||
architectures=["LlamaForCausalLM"],
|
||||
hidden_size=num_heads * head_size,
|
||||
head_dim=head_size,
|
||||
num_attention_heads=num_heads,
|
||||
num_key_value_heads=num_kv_heads,
|
||||
num_hidden_layers=1,
|
||||
max_position_embeddings=4096,
|
||||
).save_pretrained(tmp_path)
|
||||
num_blocks = 1 + num_reqs * cdiv(2112, block_size)
|
||||
config = create_vllm_config(
|
||||
model_name=str(tmp_path),
|
||||
max_model_len=4096,
|
||||
dtype=dtype,
|
||||
num_gpu_blocks=num_blocks,
|
||||
block_size=block_size,
|
||||
max_num_seqs=32,
|
||||
max_num_batched_tokens=256,
|
||||
)
|
||||
config.cache_config.kv_cache_layout = "BLHNC"
|
||||
config.speculative_config = SimpleNamespace(
|
||||
num_speculative_tokens=7,
|
||||
enable_adaptive_verification=True,
|
||||
parallel_drafting=False,
|
||||
)
|
||||
spec = create_standard_kv_cache_spec(config)
|
||||
scale = head_size**-0.5
|
||||
monkeypatch.setattr(
|
||||
fi,
|
||||
"get_per_layer_parameters",
|
||||
lambda *_: {
|
||||
"layer": PerLayerParameters(
|
||||
window_left=-1 if sliding_window is None else sliding_window - 1,
|
||||
logits_soft_cap=0.0,
|
||||
sm_scale=scale,
|
||||
has_sinks=False,
|
||||
),
|
||||
},
|
||||
)
|
||||
device = torch.device("cuda")
|
||||
layout = KVCacheLayout.BLHNC
|
||||
query = torch.empty(num_tokens, num_heads, head_size, dtype=dtype, device=device)
|
||||
key = torch.empty(num_tokens, num_kv_heads, head_size, dtype=dtype, device=device)
|
||||
value = torch.empty_like(key)
|
||||
output = torch.empty_like(query)
|
||||
dummy_lens = [
|
||||
num_tokens // num_reqs + (i < num_tokens % num_reqs) for i in range(num_reqs)
|
||||
]
|
||||
common = create_common_attn_metadata(
|
||||
BatchSpec(seq_lens=[2112] * num_reqs, query_lens=dummy_lens),
|
||||
block_size,
|
||||
device,
|
||||
)
|
||||
common.max_query_len = 8
|
||||
cache = None
|
||||
|
||||
def prepare(query_lens):
|
||||
nonlocal cache
|
||||
query_lens = query_lens + [0] * (num_reqs - len(query_lens))
|
||||
query.normal_()
|
||||
key.normal_()
|
||||
value.normal_()
|
||||
# Alternating short prefills, nonempty decode contexts, and contexts
|
||||
# beyond the sliding window; inactive requests have zero sequence length.
|
||||
contexts = [
|
||||
([0, 17, 2079, 39][i % 4] if q else 0) for i, q in enumerate(query_lens)
|
||||
]
|
||||
ks = [
|
||||
torch.randn(c, num_kv_heads, head_size, dtype=dtype, device=device)
|
||||
for c in contexts
|
||||
]
|
||||
vs = [torch.randn_like(k) for k in ks]
|
||||
actual = create_common_attn_metadata(
|
||||
BatchSpec(
|
||||
seq_lens=[c + q for c, q in zip(contexts, query_lens)],
|
||||
query_lens=query_lens,
|
||||
),
|
||||
block_size,
|
||||
device,
|
||||
)
|
||||
fresh_cache = create_and_prepopulate_kv_cache(
|
||||
ks,
|
||||
vs,
|
||||
block_size,
|
||||
num_kv_heads,
|
||||
head_size,
|
||||
dtype,
|
||||
device,
|
||||
num_blocks,
|
||||
actual,
|
||||
layout,
|
||||
randomize_blocks=True,
|
||||
)
|
||||
if cache is None:
|
||||
cache = fresh_cache
|
||||
else:
|
||||
cache.copy_(fresh_cache)
|
||||
common.query_start_loc.copy_(actual.query_start_loc)
|
||||
common.seq_lens.copy_(actual.seq_lens)
|
||||
common.block_table_tensor.zero_()
|
||||
common.block_table_tensor[:, : actual.block_table_tensor.shape[1]].copy_(
|
||||
actual.block_table_tensor
|
||||
)
|
||||
common.slot_mapping.fill_(-1)
|
||||
common.slot_mapping[: sum(query_lens)].copy_(actual.slot_mapping)
|
||||
return contexts, ks, vs
|
||||
|
||||
prepare(dummy_lens)
|
||||
with set_current_vllm_config(config):
|
||||
builder = fi.FlashInferMetadataBuilder(spec, ["layer"], config, device)
|
||||
metadata = builder.build(0, common)
|
||||
assert metadata.num_prefills == 0
|
||||
assert metadata.decode.q_len_per_req == 8
|
||||
impl = fi.FlashInferImpl(
|
||||
num_heads=num_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_size=head_size,
|
||||
scale=scale,
|
||||
alibi_slopes=None,
|
||||
sliding_window=sliding_window,
|
||||
kv_cache_dtype="auto",
|
||||
attn_type=AttentionType.DECODER,
|
||||
)
|
||||
layer = MockAttentionLayer(device)
|
||||
|
||||
def forward(attn_metadata=metadata):
|
||||
impl.do_kv_cache_update(layer, key, value, cache, common.slot_mapping)
|
||||
impl.forward(layer, query, key, value, cache, attn_metadata, output=output)
|
||||
|
||||
for _ in range(3):
|
||||
forward()
|
||||
torch.accelerator.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
forward()
|
||||
|
||||
layouts = [[8], [1] * num_reqs]
|
||||
if num_tokens >= 16:
|
||||
layouts.append([8, 4, 3, 1])
|
||||
if num_tokens >= 40:
|
||||
layouts.append([8] * 5)
|
||||
|
||||
def check(query_lens, run):
|
||||
contexts, ks, vs = prepare(query_lens)
|
||||
assert cache is not None
|
||||
output.fill_(float("nan"))
|
||||
run()
|
||||
start = 0
|
||||
for i, q_len in enumerate(query_lens):
|
||||
if q_len == 0:
|
||||
continue
|
||||
end = start + q_len
|
||||
full_k = (
|
||||
torch.cat((ks[i], key[start:end]))
|
||||
.float()
|
||||
.repeat_interleave(num_heads // num_kv_heads, dim=1)
|
||||
)
|
||||
full_v = (
|
||||
torch.cat((vs[i], value[start:end]))
|
||||
.float()
|
||||
.repeat_interleave(num_heads // num_kv_heads, dim=1)
|
||||
)
|
||||
scores = (
|
||||
torch.einsum("qhd,khd->hqk", query[start:end].float(), full_k) * scale
|
||||
)
|
||||
q_pos = torch.arange(q_len, device=device) + contexts[i]
|
||||
k_pos = torch.arange(full_k.shape[0], device=device)
|
||||
allowed = k_pos[None, :] <= q_pos[:, None]
|
||||
if sliding_window is not None:
|
||||
allowed &= k_pos[None, :] > q_pos[:, None] - sliding_window
|
||||
probs = scores.masked_fill(~allowed[None], float("-inf")).softmax(-1)
|
||||
reference = torch.einsum("hqk,khd->qhd", probs, full_v)
|
||||
torch.testing.assert_close(
|
||||
output[start:end].float(),
|
||||
reference,
|
||||
atol=2e-2,
|
||||
rtol=2e-2,
|
||||
)
|
||||
slots = common.slot_mapping[start:end]
|
||||
written = cache[slots // block_size, :, slots % block_size, :]
|
||||
torch.testing.assert_close(written[..., :head_size], key[start:end])
|
||||
torch.testing.assert_close(written[..., head_size:], value[start:end])
|
||||
start = end
|
||||
assert torch.count_nonzero(cache[0]) == 0
|
||||
|
||||
for query_lens in layouts:
|
||||
check(query_lens, graph.replay)
|
||||
|
||||
if num_tokens == 40:
|
||||
# A longer prefill forces a separate launch in eager/piecewise mode.
|
||||
# Its length must not become the decode kernel's query bound, and
|
||||
# compacted CPU placeholders still need not match the GPU allocation.
|
||||
common.num_reqs = 5
|
||||
common.max_query_len = 24
|
||||
common.query_start_loc_cpu[:6] = torch.tensor([0, 4, 8, 12, 16, 40])
|
||||
common.query_start_loc_cpu[6:] = 40
|
||||
|
||||
def mixed_forward():
|
||||
with set_current_vllm_config(config):
|
||||
metadata = builder.build(
|
||||
0,
|
||||
replace(
|
||||
common,
|
||||
query_start_loc=common.query_start_loc[:6],
|
||||
query_start_loc_cpu=common.query_start_loc_cpu[:6],
|
||||
seq_lens=common.seq_lens[:5],
|
||||
seq_lens_cpu_upper_bound=common.seq_lens_cpu_upper_bound[:5],
|
||||
block_table_tensor=common.block_table_tensor[:5],
|
||||
),
|
||||
)
|
||||
assert metadata.num_decodes == 4
|
||||
assert metadata.num_prefills == 1
|
||||
assert metadata.decode.q_len_per_req == 8
|
||||
forward(metadata)
|
||||
|
||||
check([8, 4, 3, 1, 24], mixed_forward)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
||||
reason="FlashInfer is not available.",
|
||||
|
||||
@@ -294,6 +294,7 @@ def _make_spec_decode_manager(
|
||||
dynamic_spec_schedule: list[tuple[int, int, int]] | None = None,
|
||||
max_num_seqs: int = 8,
|
||||
use_kda_recoverssm: bool = False,
|
||||
varlen_decode: bool = False,
|
||||
) -> gpu_cudagraph_utils.CudaGraphManager:
|
||||
monkeypatch.setattr(
|
||||
gpu_cudagraph_utils,
|
||||
@@ -316,6 +317,7 @@ def _make_spec_decode_manager(
|
||||
device=torch.device("cpu"),
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
decode_query_len=decode_query_len,
|
||||
varlen_decode=varlen_decode,
|
||||
)
|
||||
manager._graphs_captured = True
|
||||
return manager
|
||||
@@ -373,6 +375,28 @@ def test_mixed_batch_never_selects_a_uniform_decode_graph(monkeypatch):
|
||||
assert desc.num_tokens == 16
|
||||
|
||||
|
||||
def test_varlen_decode_graph_takes_bounded_decode_batches_only(monkeypatch):
|
||||
"""A varlen decode graph replays any decode batch within its query-length
|
||||
bound. The runner passes no bound for a batch with a prefill, which must not
|
||||
replay one however short the prefill is."""
|
||||
manager = _make_spec_decode_manager(monkeypatch, varlen_decode=True)
|
||||
|
||||
def dispatch(max_query_len):
|
||||
return manager.dispatch(
|
||||
num_reqs=3,
|
||||
num_tokens=6,
|
||||
uniform_token_count=None,
|
||||
num_active_loras=0,
|
||||
max_query_len=max_query_len,
|
||||
)
|
||||
|
||||
desc = dispatch(_DECODE_QUERY_LEN)
|
||||
assert desc.cg_mode == CUDAGraphMode.FULL
|
||||
assert desc.max_query_len == _DECODE_QUERY_LEN
|
||||
assert dispatch(None).cg_mode == CUDAGraphMode.PIECEWISE
|
||||
assert dispatch(_DECODE_QUERY_LEN + 1).cg_mode == CUDAGraphMode.PIECEWISE
|
||||
|
||||
|
||||
def test_mixed_batch_at_decode_only_token_count_still_gets_a_graph(monkeypatch):
|
||||
"""A mixed batch must not fall to eager where only decode graphs are staged.
|
||||
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.v1.attention.backend import AttentionCGSupport
|
||||
from vllm.v1.attention.backend import AttentionCGSupport, AttentionMetadataBuilder
|
||||
from vllm.v1.worker.gpu.async_utils import StepTimingSample
|
||||
from vllm.v1.worker.gpu.attn_utils import AttentionCGSupportInfo
|
||||
from vllm.v1.worker.gpu.spec_decode import adaptive_verification as adaptive_module
|
||||
from vllm.v1.worker.gpu.spec_decode.adaptive_verification import (
|
||||
AdaptiveVerificationManager,
|
||||
@@ -58,37 +58,90 @@ def test_resolve_adaptive_cudagraph_mode(mode, piecewise_capture_available, expe
|
||||
)
|
||||
|
||||
|
||||
def test_manager_scopes_varlen_check_without_weakening_runner_cg_mode(monkeypatch):
|
||||
class Backend:
|
||||
@classmethod
|
||||
def supports_device_cpu_query_lens_mismatch(cls):
|
||||
return True
|
||||
@pytest.mark.parametrize(
|
||||
"target_support,target_bound,device_offsets,additional,error",
|
||||
[
|
||||
pytest.param(AttentionCGSupport.ALWAYS, None, True, None, None, id="always"),
|
||||
pytest.param(
|
||||
AttentionCGSupport.UNIFORM_BATCH, 8, True, None, None, id="bounded"
|
||||
),
|
||||
pytest.param(
|
||||
AttentionCGSupport.UNIFORM_BATCH,
|
||||
None,
|
||||
True,
|
||||
None,
|
||||
"TargetBackend allows none",
|
||||
id="unsupported",
|
||||
),
|
||||
pytest.param(
|
||||
AttentionCGSupport.UNIFORM_BATCH,
|
||||
7,
|
||||
True,
|
||||
None,
|
||||
"up to 8, but TargetBackend allows at most 7",
|
||||
id="too-narrow",
|
||||
),
|
||||
pytest.param(
|
||||
AttentionCGSupport.UNIFORM_BATCH,
|
||||
8,
|
||||
False,
|
||||
None,
|
||||
"trims verification requests",
|
||||
id="host-query-lens",
|
||||
),
|
||||
pytest.param(
|
||||
AttentionCGSupport.ALWAYS,
|
||||
None,
|
||||
True,
|
||||
(AttentionCGSupport.UNIFORM_BATCH, "EncoderBackend"),
|
||||
"EncoderBackend allows none",
|
||||
id="additional-group",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_manager_checks_target_varlen_cudagraph_bound(
|
||||
monkeypatch, target_support, target_bound, device_offsets, additional, error
|
||||
):
|
||||
"""Target builders must replay varlen decode graphs of the verification
|
||||
width; draft-only groups are not checked."""
|
||||
|
||||
class Builder:
|
||||
def __init__(self, support):
|
||||
self.support = support
|
||||
def group(backend_name, layer_name, support, bound=None, mismatch=True):
|
||||
class Builder(AttentionMetadataBuilder):
|
||||
_cudagraph_support = support
|
||||
|
||||
def get_cudagraph_support(self, *_args):
|
||||
return self.support
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def group(layer_name, support):
|
||||
builder = Builder(support)
|
||||
def build(self, common_prefix_len, common_attn_metadata, fast_build=False):
|
||||
raise NotImplementedError
|
||||
|
||||
if bound is not None:
|
||||
|
||||
@classmethod
|
||||
def get_varlen_cudagraph_max_query_len(cls, *_args):
|
||||
return bound
|
||||
|
||||
backend = type(
|
||||
backend_name,
|
||||
(),
|
||||
{"supports_device_cpu_query_lens_mismatch": staticmethod(lambda: mismatch)},
|
||||
)
|
||||
builder = Builder()
|
||||
return SimpleNamespace(
|
||||
layer_names=[layer_name],
|
||||
backend=Backend,
|
||||
backend=backend,
|
||||
kv_cache_spec=None,
|
||||
get_metadata_builder=lambda _index: builder,
|
||||
)
|
||||
|
||||
groups = [
|
||||
[
|
||||
group("target", AttentionCGSupport.ALWAYS),
|
||||
group("draft", AttentionCGSupport.UNIFORM_BATCH),
|
||||
group(
|
||||
"TargetBackend", "target", target_support, target_bound, device_offsets
|
||||
),
|
||||
group("DraftBackend", "draft", AttentionCGSupport.UNIFORM_BATCH),
|
||||
]
|
||||
]
|
||||
runner_support = AttentionCGSupportInfo(
|
||||
AttentionCGSupport.UNIFORM_BATCH, "DraftBackend"
|
||||
)
|
||||
created = object()
|
||||
monkeypatch.setattr(
|
||||
adaptive_module,
|
||||
@@ -96,20 +149,19 @@ def test_manager_scopes_varlen_check_without_weakening_runner_cg_mode(monkeypatc
|
||||
lambda *_args, **_kwargs: created,
|
||||
)
|
||||
|
||||
manager = maybe_create_adaptive_verification_manager(
|
||||
enable_adaptive_verification=True,
|
||||
attn_groups=groups,
|
||||
attn_cg_support=runner_support,
|
||||
req_states=object(),
|
||||
query_start_loc=object(),
|
||||
num_bonus_tokens=1,
|
||||
max_total_logits=1,
|
||||
vllm_config=None,
|
||||
target_layer_names={"target"},
|
||||
)
|
||||
|
||||
assert manager is created
|
||||
assert runner_support.min_cg_support == AttentionCGSupport.UNIFORM_BATCH
|
||||
with pytest.raises(ValueError, match=error) if error else nullcontext():
|
||||
manager = maybe_create_adaptive_verification_manager(
|
||||
enable_adaptive_verification=True,
|
||||
attn_groups=groups,
|
||||
req_states=SimpleNamespace(num_speculative_steps=7),
|
||||
query_start_loc=object(),
|
||||
num_bonus_tokens=1,
|
||||
max_total_logits=1,
|
||||
vllm_config=None,
|
||||
target_layer_names={"target"},
|
||||
additional_attn_cg_support=additional,
|
||||
)
|
||||
assert manager is created
|
||||
|
||||
|
||||
def test_budget_stops_where_marginal_drafts_stop_paying_for_themselves():
|
||||
|
||||
@@ -8,6 +8,7 @@ never addressed by the logical view.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
@@ -99,12 +100,16 @@ def test_get_kv_cache_spec_resolves_hisparse_block_size(
|
||||
|
||||
|
||||
class _FakeMetadataBuilder:
|
||||
def __init__(self, support: AttentionCGSupport):
|
||||
def __init__(self, support: AttentionCGSupport, varlen_bound: int | None = None):
|
||||
self.support = support
|
||||
self.varlen_bound = varlen_bound
|
||||
|
||||
def get_cudagraph_support(self, *_args):
|
||||
return self.support
|
||||
|
||||
def get_varlen_cudagraph_max_query_len(self, *_args):
|
||||
return self.varlen_bound
|
||||
|
||||
|
||||
class _TargetBackend:
|
||||
@classmethod
|
||||
@@ -183,6 +188,42 @@ def test_attention_checks_preserve_global_and_target_scoped_support():
|
||||
)
|
||||
|
||||
|
||||
def test_varlen_cudagraph_unsupported_backend_checks_scoped_bounds():
|
||||
"""ALWAYS passes without a bound, other builders need one at least as wide as
|
||||
the requested length, and NEVER fails whatever bound a builder reports."""
|
||||
config: Any = SimpleNamespace()
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
def group(
|
||||
backend: Any,
|
||||
layer_name: str,
|
||||
support: AttentionCGSupport,
|
||||
bound: int | None = None,
|
||||
):
|
||||
builder: Any = _FakeMetadataBuilder(support, bound)
|
||||
attn_group = AttentionGroup(backend, [layer_name], spec, 0)
|
||||
attn_group.metadata_builders = [builder]
|
||||
return attn_group
|
||||
|
||||
target = group(_TargetBackend, "target", AttentionCGSupport.ALWAYS)
|
||||
draft = group(_DraftBackend, "draft", AttentionCGSupport.UNIFORM_BATCH, 8)
|
||||
never = group(_DraftBackend, "never", AttentionCGSupport.NEVER, 8)
|
||||
|
||||
unsupported = attn_utils.get_varlen_cudagraph_unsupported_backend
|
||||
assert unsupported([[target, draft]], config, 8) is None
|
||||
assert unsupported([[target, draft]], config, 9) == ("_DraftBackend", 8)
|
||||
assert (
|
||||
unsupported([[target, draft]], config, 9, checked_layer_names={"target"})
|
||||
is None
|
||||
)
|
||||
assert unsupported([[target, never]], config, 1) == ("_DraftBackend", None)
|
||||
|
||||
|
||||
def test_get_kv_sharing_fast_prefill_eligible_layers(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Fast prefill applies to the contiguous suffix of KV-sharing layers.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user