[Bugfix] Generalize KV block zeroing to AttentionSpec (#51749)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
513f83e7ec
commit
e3fe212eaf
@@ -1,10 +1,71 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.v1.worker.utils import KVBlockZeroer, _zero_kv_blocks_kernel
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
ChunkedLocalAttentionSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
from vllm.v1.worker.utils import (
|
||||
AttentionGroup,
|
||||
KVBlockZeroer,
|
||||
_zero_kv_blocks_kernel,
|
||||
)
|
||||
|
||||
|
||||
class _BlockFirstBackend:
|
||||
@staticmethod
|
||||
def get_kv_cache_block_dim(*args, **kwargs):
|
||||
return 0
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
@pytest.mark.parametrize(
|
||||
"spec",
|
||||
[
|
||||
SlidingWindowSpec(
|
||||
block_size=2,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.uint8,
|
||||
sliding_window=4,
|
||||
),
|
||||
ChunkedLocalAttentionSpec(
|
||||
block_size=2,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.uint8,
|
||||
attention_chunk_size=4,
|
||||
),
|
||||
],
|
||||
ids=["sliding-window", "chunked-local"],
|
||||
)
|
||||
def test_attention_blocks_are_zeroed(spec):
|
||||
device = torch.device("cuda")
|
||||
storage = torch.ones((4, 1, 2, 2), dtype=torch.uint8, device=device)
|
||||
layer_name = "draft.self_attn"
|
||||
zeroer = KVBlockZeroer(
|
||||
device,
|
||||
attn_groups_iter=[
|
||||
AttentionGroup(_BlockFirstBackend, [layer_name], spec, 0) # type: ignore[arg-type]
|
||||
],
|
||||
kernel_block_sizes=[2],
|
||||
cache_dtype="fp8",
|
||||
static_forward_context={
|
||||
layer_name: SimpleNamespace(kv_cache=storage),
|
||||
},
|
||||
)
|
||||
|
||||
zeroer.zero_block_ids([1])
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
expected = torch.ones_like(storage)
|
||||
expected[1] = 0
|
||||
assert torch.equal(storage, expected)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
|
||||
Reference in New Issue
Block a user