[NIXL] Generalize block-first backend layouts (FlashInfer-like) (#28282)
This commit is contained in:
@@ -1096,7 +1096,8 @@ def _run_abort_timeout_test(llm: LLM, timeout: int):
|
||||
llm.llm_engine.engine_core.shutdown()
|
||||
|
||||
|
||||
def test_register_kv_caches(dist_init):
|
||||
@pytest.mark.parametrize("attn_backend", ["FLASH_ATTN", "TRITON_ATTN"])
|
||||
def test_register_kv_caches(dist_init, attn_backend, monkeypatch):
|
||||
"""
|
||||
Test that register_kv_caches() properly calls nixl_wrapper methods with
|
||||
correct data.
|
||||
@@ -1108,10 +1109,22 @@ def test_register_kv_caches(dist_init):
|
||||
block layout info
|
||||
"""
|
||||
|
||||
monkeypatch.setenv("VLLM_ATTENTION_BACKEND", attn_backend)
|
||||
|
||||
vllm_config = create_vllm_config()
|
||||
|
||||
# Import the appropriate backend based on the parameter
|
||||
if attn_backend == "FLASH_ATTN":
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
||||
|
||||
backend_cls = FlashAttentionBackend
|
||||
else: # TRITON_ATTN
|
||||
from vllm.v1.attention.backends.triton_attn import TritonAttentionBackend
|
||||
|
||||
backend_cls = TritonAttentionBackend
|
||||
|
||||
# Create test kv cache tensors using proper backend shape
|
||||
kv_cache_shape = FlashAttentionBackend.get_kv_cache_shape(
|
||||
kv_cache_shape = backend_cls.get_kv_cache_shape(
|
||||
num_blocks=2, block_size=16, num_kv_heads=4, head_size=64
|
||||
)
|
||||
shared_tensor = torch.zeros(*kv_cache_shape, dtype=torch.float16)
|
||||
|
||||
Reference in New Issue
Block a user