[XPU] limit FA kernel block size to multipleOf 64 (#57297)

Signed-off-by: Yan Ma <yan.ma@intel.com>
This commit is contained in:
Yan Ma
2026-10-12 10:25:32 +08:00
committed by GitHub
parent 124b89d5b7
commit f147949be8
11 changed files with 269 additions and 126 deletions
+1 -1
View File
@@ -105,7 +105,7 @@ steps:
pytest -v -s v1/test_serial_utils.py &&
pytest -v -s v1/e2e/general/test_correctness_sliding_window.py &&
pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py --ignore=v1/spec_decode/test_speculators_correctness.py --deselect=tests/v1/spec_decode/test_mtp.py::test_glm_mtp_defers_lm_head &&
pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py --ignore=v1/kv_connector/unit/test_nixl_connector_hma.py --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --deselect=tests/v1/kv_connector/unit/test_nixl_connector.py::test_abort_timeout_on_prefiller[ray]'
pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --deselect=tests/v1/kv_connector/unit/test_nixl_connector.py::test_abort_timeout_on_prefiller[ray]'
parallelism: 2
- label: "XPU Qwen3-30B + V1 KV Connector"
key: xpu-qwen3-30b-v1-kv-connector
@@ -51,6 +51,7 @@ from .utils import (
create_scheduler,
create_vllm_config,
make_kv_cache_config,
maybe_update_block_size,
)
pytestmark = pytest.mark.cpu_test
@@ -87,10 +88,11 @@ def _make_connector_with_fake_worker(
hand_shake_latency=0, cycles_before_done=0, do_handshake=True
):
"""Create a NixlConnector with FakeNixlConnectorWorker."""
block_size = maybe_update_block_size(32)
vllm_config = create_vllm_config(
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, block_size=32
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, block_size=block_size
)
kv_cache_config = make_kv_cache_config(block_size=32, num_blocks=2)
kv_cache_config = make_kv_cache_config(block_size=block_size, num_blocks=2)
connector = NixlConnector(vllm_config, KVConnectorRole.WORKER, kv_cache_config)
connector.connector_worker = FakeNixlConnectorWorker(
vllm_config,
@@ -57,7 +57,12 @@ from vllm.v1.kv_cache_interface import (
)
from vllm.v1.request import RequestStatus
from .utils import create_request, create_scheduler, create_vllm_config
from .utils import (
create_request,
create_scheduler,
create_vllm_config,
maybe_update_block_size,
)
@pytest.mark.parametrize(
@@ -148,6 +153,7 @@ def test_region_length_validation_checks_replicated_gqa_heads(
def _make_test_kv_cache_config() -> KVCacheConfig:
block_size = maybe_update_block_size(16)
return KVCacheConfig(
num_blocks=0,
kv_cache_tensors=[],
@@ -160,7 +166,7 @@ def _make_test_kv_cache_config() -> KVCacheConfig:
"model.layers.1.eagle_attn",
],
FullAttentionSpec(
block_size=16,
block_size=block_size,
num_kv_heads=4,
head_size=64,
dtype=torch.float16,
@@ -170,6 +176,14 @@ def _make_test_kv_cache_config() -> KVCacheConfig:
)
def _make_worker_vllm_config(kv_role: str):
return create_vllm_config(
kv_connector="MooncakeConnector",
kv_role=kv_role,
block_size=maybe_update_block_size(16),
)
def _make_packed_mla_kv_cache_config(
num_blocks: int, *, num_groups: int = 1
) -> KVCacheConfig:
@@ -429,9 +443,7 @@ async def test_send_kv_to_decode_aligns_consumer_regions_by_layer_metadata(
):
"""Producer sends its PP layer shard to the matching consumer layer address."""
monkeypatch.setenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "5")
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector", kv_role="kv_producer"
)
vllm_config = _make_worker_vllm_config("kv_producer")
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
prefill_connector = MooncakeConnector(
@@ -1205,7 +1217,7 @@ def _xfer_meta(
@contextlib.contextmanager
def mooncake_register_worker(kv_cache_config, *, kv_role: str = "kv_consumer"):
"""Build a worker whose engine registration and sender thread are patched."""
vllm_config = create_vllm_config(kv_connector="MooncakeConnector", kv_role=kv_role)
vllm_config = _make_worker_vllm_config(kv_role)
connector_mod = (
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector"
)
@@ -1241,9 +1253,7 @@ async def test_receive_kv_selects_remote_pp_workers(
expected_addrs: list[str],
):
"""Decode workers should not hard-code producer pp_rank 0."""
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector", kv_role="kv_consumer"
)
vllm_config = _make_worker_vllm_config("kv_consumer")
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
decode_connector = MooncakeConnector(
@@ -1314,7 +1324,7 @@ def test_resolve_need_send_accounts_for_remote_tp_fanout():
@pytest.mark.asyncio
async def test_heterogeneous_pp_waits_for_consumer_without_shared_layers():
"""P4/D2 must retain KV until both D stages finish, including a no-op pull."""
config = create_vllm_config(kv_connector="MooncakeConnector", kv_role="kv_producer")
config = _make_worker_vllm_config("kv_producer")
with (
set_current_vllm_config(config),
patch_worker_dependencies(),
@@ -1386,9 +1396,7 @@ async def test_kv_producer(monkeypatch):
Verifies memory offset calculation: ptr = base_addr + block_id * block_len.
"""
monkeypatch.setenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "5")
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector", kv_role="kv_producer"
)
vllm_config = _make_worker_vllm_config("kv_producer")
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
prefill_connector = MooncakeConnector(
@@ -1589,9 +1597,7 @@ async def test_kv_consumuer(monkeypatch):
Verifies that MooncakeXferMetadata is correctly serialized and sent via ZMQ.
"""
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector", kv_role="kv_consumer"
)
vllm_config = _make_worker_vllm_config("kv_consumer")
with set_current_vllm_config(vllm_config), patch_worker_dependencies() as mocks:
decode_connector = MooncakeConnector(
@@ -1661,9 +1667,7 @@ async def test_kv_consumuer(monkeypatch):
@pytest.mark.asyncio
async def test_worker_get_finished_timeout(monkeypatch):
"""Tests the cleanup mechanism for requests."""
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector", kv_role="kv_producer"
)
vllm_config = _make_worker_vllm_config("kv_producer")
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
prefill_connector = MooncakeConnector(
vllm_config,
@@ -1713,7 +1717,7 @@ async def test_worker_get_finished_timeout(monkeypatch):
def test_register_kv_caches(layout: KVCacheLayout, separate_kv_head_groups: bool):
"""Tests the memory registration logic with the underlying Mooncake engine."""
spec = FullAttentionSpec(
block_size=16,
block_size=maybe_update_block_size(16),
num_kv_heads=4,
head_size=64,
dtype=torch.float16,
@@ -1794,7 +1798,10 @@ def test_register_noncontiguous_packed_allows_matching_pp():
known. Rejecting every local PP>1 layout at startup is too strict.
"""
spec = FullAttentionSpec(
block_size=16, num_kv_heads=4, head_size=64, dtype=torch.float16
block_size=maybe_update_block_size(16),
num_kv_heads=4,
head_size=64,
dtype=torch.float16,
)
layer_names = ["model.layers.0.self_attn", "model.layers.1.self_attn"]
with mooncake_register_worker(_make_test_kv_cache_config()) as (worker, _reg):
@@ -1814,8 +1821,9 @@ def test_register_kv_caches_supports_mixed_mla_and_eagle_shapes():
with mooncake_register_worker(_make_test_kv_cache_config()) as (worker, reg):
worker.use_mla = True
worker.transfer_topo.is_mla = True
mla_cache = torch.zeros((2, 16, 512), dtype=torch.float16)
eagle_cache = torch.zeros((2, 16, 8, 64), dtype=torch.float16)
block_size = maybe_update_block_size(16)
mla_cache = torch.zeros((2, block_size, 512), dtype=torch.float16)
eagle_cache = torch.zeros((2, block_size, 8, 64), dtype=torch.float16)
kv_caches = {
"model.layers.0.mla_attn": mla_cache,
"model.layers.1.eagle_attn": eagle_cache,
@@ -2240,7 +2248,7 @@ def _register_sliced_packed_mla(
):
"""Register column slices of one MLA packed row. Returns worker fields."""
spec = MLAAttentionSpec(
block_size=16,
block_size=maybe_update_block_size(16),
num_kv_heads=1,
head_size=64,
dtype=torch.uint8,
@@ -2286,7 +2294,7 @@ def test_sliced_packed_mla_registers_each_layer_view():
KVCacheGroupSpec(
[_layer_name(i) for i in layer_idxs],
MLAAttentionSpec(
block_size=16,
block_size=maybe_update_block_size(16),
num_kv_heads=1,
head_size=64,
dtype=torch.uint8,
@@ -2436,9 +2444,7 @@ async def test_kv_producer_heterogeneous_tp(monkeypatch, d_tp_size):
remote_block_len = LOCAL_BLOCK_LEN * P_TP_SIZE // d_tp_size
monkeypatch.setenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "5")
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector", kv_role="kv_producer"
)
vllm_config = _make_worker_vllm_config("kv_producer")
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
prefill_connector = MooncakeConnector(
@@ -30,7 +30,12 @@ from vllm.v1.kv_cache_interface import (
)
from .test_mooncake_connector import FakeMooncakeWrapper, patch_worker_dependencies
from .utils import create_request, create_vllm_config, make_kv_cache_config
from .utils import (
create_request,
create_vllm_config,
make_kv_cache_config,
maybe_update_block_size,
)
# ---------------------------------------------------------------------------
@@ -224,12 +229,11 @@ def test_metadata_hma_block_ids():
async def test_build_transfer_params_multi_group_trimming(monkeypatch):
"""_build_transfer_params trims per-group blocks when local > remote."""
monkeypatch.setenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "5")
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector", kv_role="kv_producer"
)
kv_cache_config = make_kv_cache_config(
block_size=vllm_config.cache_config.block_size, swa_enabled=True
kv_connector="MooncakeConnector", kv_role="kv_producer", block_size=block_size
)
kv_cache_config = make_kv_cache_config(block_size=block_size, swa_enabled=True)
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
connector = MooncakeConnector(
@@ -320,12 +324,11 @@ async def test_build_transfer_params_multi_group_trimming(monkeypatch):
async def test_build_transfer_params_group_count_mismatch(monkeypatch):
"""_build_transfer_params reports an error when group counts differ."""
monkeypatch.setenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "5")
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector", kv_role="kv_producer"
)
kv_cache_config = make_kv_cache_config(
block_size=vllm_config.cache_config.block_size, swa_enabled=True
kv_connector="MooncakeConnector", kv_role="kv_producer", block_size=block_size
)
kv_cache_config = make_kv_cache_config(block_size=block_size, swa_enabled=True)
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
connector = MooncakeConnector(
@@ -449,7 +452,7 @@ def test_request_finished_with_hma_groups():
def _make_kv_consumer_worker(
swa_enabled: bool = False, kv_cache_config: KVCacheConfig | None = None
):
block_size = 16
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector",
kv_role="kv_consumer",
@@ -495,6 +498,7 @@ def test_worker_is_hma_required(swa_enabled, expected_is_hma):
def test_worker_is_hma_required_multiple_full_attention_groups():
"""Two full-attention groups with different block sizes also select
request-level failure reporting, matching the scheduler."""
block_size = maybe_update_block_size(16)
kv_cache_config = KVCacheConfig(
num_blocks=100,
kv_cache_tensors=[],
@@ -502,13 +506,19 @@ def test_worker_is_hma_required_multiple_full_attention_groups():
KVCacheGroupSpec(
["layer0"],
FullAttentionSpec(
block_size=16, num_kv_heads=4, head_size=16, dtype=torch.float16
block_size=block_size,
num_kv_heads=4,
head_size=16,
dtype=torch.float16,
),
),
KVCacheGroupSpec(
["layer1"],
FullAttentionSpec(
block_size=32, num_kv_heads=4, head_size=16, dtype=torch.float16
block_size=2 * block_size,
num_kv_heads=4,
head_size=16,
dtype=torch.float16,
),
),
],
@@ -36,7 +36,7 @@ from vllm.v1.kv_cache_interface import (
)
from .test_mooncake_connector import patch_worker_dependencies
from .utils import create_request, create_vllm_config
from .utils import create_request, create_vllm_config, maybe_update_block_size
def noop_shutdown():
@@ -147,6 +147,7 @@ def test_register_kv_caches_emits_fa_and_gdn_regions(monkeypatch):
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector",
kv_role="kv_consumer",
block_size=maybe_update_block_size(16),
)
kv_cache_config = make_hybrid_gdn_kv_cache_config(
vllm_config.cache_config.block_size
@@ -203,6 +204,7 @@ def test_register_kv_caches_scales_attention_len_to_kernel_block(monkeypatch):
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector",
kv_role="kv_consumer",
block_size=maybe_update_block_size(16),
)
kv_cache_config = make_hybrid_gdn_kv_cache_config(
vllm_config.cache_config.block_size
@@ -392,6 +394,7 @@ def test_register_kv_caches_deduplicates_shared_backing_memory(monkeypatch):
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector",
kv_role="kv_consumer",
block_size=maybe_update_block_size(16),
)
kv_cache_config = make_hybrid_gdn_kv_cache_config(
vllm_config.cache_config.block_size
@@ -405,8 +408,13 @@ def test_register_kv_caches_deduplicates_shared_backing_memory(monkeypatch):
)
worker = connector.connector_worker
backing = torch.empty((4, 64), dtype=torch.float16)
fa_cache = backing[:2, :16]
fa_spec = kv_cache_config.kv_cache_groups[0].kv_cache_spec
fa_cache_width = (
fa_spec.page_size_bytes
// torch.tensor([], dtype=torch.float16).element_size()
)
backing = torch.empty((4, fa_cache_width), dtype=torch.float16)
fa_cache = backing[:2]
gdn_cache = backing[:3]
with patch.object(
@@ -438,6 +446,7 @@ def test_hybrid_gdn_transfer_params_preserve_group_identity(monkeypatch):
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector",
kv_role="kv_producer",
block_size=maybe_update_block_size(16),
)
kv_cache_config = make_hybrid_gdn_kv_cache_config(
vllm_config.cache_config.block_size
@@ -567,6 +576,7 @@ def test_hybrid_gdn_keeps_packed_fa_and_gdn_regions_whole(
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector",
kv_role="kv_producer",
block_size=maybe_update_block_size(16),
)
kv_cache_config = make_hybrid_gdn_kv_cache_config(
vllm_config.cache_config.block_size
@@ -10,7 +10,7 @@ from unittest.mock import MagicMock
import pytest
import torch
from tests.v1.kv_connector.unit.utils import create_vllm_config
from tests.v1.kv_connector.unit.utils import create_vllm_config, maybe_update_block_size
from vllm import LLM, SamplingParams
from vllm.config import KVTransferConfig
from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory
@@ -314,10 +314,13 @@ def test_multi_example_connector_consistency():
},
)
block_size = maybe_update_block_size(16)
expected_num_blocks = 2 if block_size == 64 else 7
expected_num_matched_tokens = (expected_num_blocks - 1) * block_size
llm = LLM(
model=MODEL_NAME,
enforce_eager=True,
block_size=16,
block_size=block_size,
gpu_memory_utilization=0.5,
kv_transfer_config=kv_transfer_config,
async_scheduling=False,
@@ -372,7 +375,7 @@ def test_multi_example_connector_consistency():
"set_xfer_handshake_metadata_pp_aware",
"on_new_request",
"get_num_new_matched_tokens 0",
"update_state_after_alloc num_blocks=[7] 0",
f"update_state_after_alloc num_blocks=[{expected_num_blocks}] 0",
"build_connector_meta",
]
# First three events are from initialization. Layer hooks run before the
@@ -399,7 +402,7 @@ def test_multi_example_connector_consistency():
"set_xfer_handshake_metadata_pp_aware",
"on_new_request",
"get_num_new_matched_tokens 0",
"update_state_after_alloc num_blocks=[7] 0",
f"update_state_after_alloc num_blocks=[{expected_num_blocks}] 0",
"build_connector_meta",
]
# Reset prefix cache or else we'll just get the tokens back from there.
@@ -423,13 +426,14 @@ def test_multi_example_connector_consistency():
assert storage1_scheduler_events[:4] == [
"on_new_request",
"get_num_new_matched_tokens 0",
"update_state_after_alloc num_blocks=[7] 96",
"update_state_after_alloc "
f"num_blocks=[{expected_num_blocks}] {expected_num_matched_tokens}",
"build_connector_meta",
]
assert storage2_scheduler_events[:4] == [
"on_new_request",
"get_num_new_matched_tokens 0",
"update_state_after_alloc num_blocks=[7] 0",
f"update_state_after_alloc num_blocks=[{expected_num_blocks}] 0",
"build_connector_meta",
]
@@ -457,13 +461,14 @@ def test_multi_example_connector_consistency():
assert storage1_scheduler_events[:4] == [
"on_new_request",
"get_num_new_matched_tokens 0",
"update_state_after_alloc num_blocks=[7] 0",
f"update_state_after_alloc num_blocks=[{expected_num_blocks}] 0",
"build_connector_meta",
]
assert storage2_scheduler_events[:4] == [
"on_new_request",
"get_num_new_matched_tokens 0",
"update_state_after_alloc num_blocks=[7] 96",
"update_state_after_alloc "
f"num_blocks=[{expected_num_blocks}] {expected_num_matched_tokens}",
"build_connector_meta",
]
+154 -70
View File
@@ -84,6 +84,7 @@ from .utils import (
create_scheduler,
create_vllm_config,
make_kv_cache_config,
maybe_update_block_size,
)
@@ -418,7 +419,8 @@ def test_kv_transfer_handshake(dist_init):
# Test setup, we creates a scheduler that contains a NixlConnector
# of role SCHEDULER, and expect it to be serving NixlAgentMetadata from
# all workers of the instance.
vllm_config = create_vllm_config()
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(block_size=block_size)
vllm_config.cache_config.kv_cache_layout = "BLHNC"
# in case the test runs on non-GPU machine
vllm_config.kv_transfer_config.kv_buffer_device = "cpu"
@@ -434,7 +436,7 @@ def test_kv_transfer_handshake(dist_init):
KVCacheGroupSpec(
["layer0", "layer1", "layer2"],
FullAttentionSpec(
block_size=16,
block_size=block_size,
num_kv_heads=4,
head_size=16,
dtype=torch.float16,
@@ -544,7 +546,9 @@ class FakeNixlConnectorWorker(NixlConnectorWorker):
**kwargs,
):
if kv_cache_config is None:
kv_cache_config = make_kv_cache_config(block_size=16)
kv_cache_config = make_kv_cache_config(
block_size=args[0].cache_config.block_size
)
super().__init__(*args, kv_cache_config=kv_cache_config, **kwargs)
self._hand_shake_latency = hand_shake_latency
self.kv_cache_layout = kv_cache_layout
@@ -674,7 +678,8 @@ class TestNixlHandshake:
"""Replicated PCP is canonicalized; PCP-DCP publishes every shard."""
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
vllm_config = create_vllm_config(kv_role="kv_producer")
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(kv_role="kv_producer", block_size=block_size)
vllm_config.parallel_config.prefill_context_parallel_size = pcp_size
vllm_config.parallel_config.decode_context_parallel_size = dcp_size
with (
@@ -692,7 +697,7 @@ class TestNixlHandshake:
connector = NixlConnector(
vllm_config,
KVConnectorRole.WORKER,
make_kv_cache_config(block_size=16),
make_kv_cache_config(block_size=block_size),
)
worker = connector.connector_worker
@@ -746,12 +751,13 @@ class TestNixlHandshake:
This test triggers the connector to load remote KV for the same
`request_id`.
"""
vllm_config = create_vllm_config()
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(block_size=block_size)
request_id = "req_id"
# Test worker role in decode server.
kv_cache_config = make_kv_cache_config(block_size=16, num_blocks=10)
kv_cache_config = make_kv_cache_config(block_size=block_size, num_blocks=10)
connector = NixlConnector(vllm_config, KVConnectorRole.WORKER, kv_cache_config)
connector.connector_worker = FakeNixlConnectorWorker(
vllm_config,
@@ -837,12 +843,15 @@ class TestNixlHandshake:
prefill_tp_size,
):
"""Test that NixlConnector's start_load_kv should be non-blocking."""
vllm_config = create_vllm_config()
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(block_size=block_size)
vllm_config.parallel_config.tensor_parallel_size = decode_tp_size
# Test worker role in decode server.
connector = NixlConnector(
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
vllm_config,
KVConnectorRole.WORKER,
make_kv_cache_config(block_size=block_size),
)
connector.connector_worker = FakeNixlConnectorWorker(
vllm_config, connector.engine_id
@@ -899,12 +908,13 @@ class TestNixlHandshake:
lambda: local_tp_size,
)
vllm_config = create_vllm_config()
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(block_size=block_size)
# num_blocks=1 keeps the fake handshake's region_num_blocks consistent
# with the minimal single-block registration below (num_descs == 1);
# _fa_desc_replicated cross-checks the two.
kv_cache_config = make_kv_cache_config(block_size=16, num_blocks=1)
kv_cache_config = make_kv_cache_config(block_size=block_size, num_blocks=1)
connector = NixlConnector(vllm_config, KVConnectorRole.WORKER, kv_cache_config)
connector.connector_worker = FakeNixlConnectorWorker(
vllm_config,
@@ -972,16 +982,21 @@ class TestNixlHandshake:
"""Verify remote TP > local TP handshake succeeds with different
remote configurations for an MLA model.
"""
vllm_config = create_vllm_config()
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(block_size=block_size)
d_tp_size = 1
p_tp_size = 2
# Build two separate connectors/workers to emulate P TP=2 ranks.
conn_p0 = NixlConnector(
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
vllm_config,
KVConnectorRole.WORKER,
make_kv_cache_config(block_size=block_size),
)
conn_p1 = NixlConnector(
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
vllm_config,
KVConnectorRole.WORKER,
make_kv_cache_config(block_size=block_size),
)
conn_p0.connector_worker = FakeNixlConnectorWorker(
vllm_config, conn_p0.engine_id, hand_shake_latency=0
@@ -1075,11 +1090,14 @@ class TestNixlHandshake:
dist_init,
):
"""Test that multiple start_load_kv calls should occur concurrently."""
vllm_config = create_vllm_config()
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(block_size=block_size)
# Test worker role in decode server.
connector = NixlConnector(
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
vllm_config,
KVConnectorRole.WORKER,
make_kv_cache_config(block_size=block_size),
)
connector.connector_worker = FakeNixlConnectorWorker(
vllm_config, connector.engine_id
@@ -1143,8 +1161,10 @@ class TestNixlHandshake:
"""Verify that adding a remote agent fails if kv_cache_layout differs.
This test is only relevant for heterogeneous TP.
"""
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(
enable_permute_local_kv=enable_permute_local_kv
enable_permute_local_kv=enable_permute_local_kv,
block_size=block_size,
)
# Mock TP world size to 2 to force heterogeneous TP when
@@ -1155,7 +1175,9 @@ class TestNixlHandshake:
):
# Initialize connector and worker (with fake NIXL wrapper)
connector = NixlConnector(
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
vllm_config,
KVConnectorRole.WORKER,
make_kv_cache_config(block_size=block_size),
)
connector.connector_worker = FakeNixlConnectorWorker(
vllm_config,
@@ -1205,7 +1227,10 @@ class TestNixlHandshake:
"""Verify that adding a remote agent fails if kv_cache_layout differs.
This test is only relevant for heterogeneous TP.
"""
vllm_config = create_vllm_config(enable_permute_local_kv=True)
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(
enable_permute_local_kv=True, block_size=block_size
)
# Mock TP world size to 2 to force heterogeneous TP when
# remote_tp_size=1
@@ -1215,7 +1240,9 @@ class TestNixlHandshake:
):
# Initialize connector and worker (with fake NIXL wrapper)
connector = NixlConnector(
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
vllm_config,
KVConnectorRole.WORKER,
make_kv_cache_config(block_size=block_size),
)
connector.connector_worker = FakeNixlConnectorWorker(
vllm_config,
@@ -1261,7 +1288,9 @@ class TestNixlHandshake:
self, default_vllm_config, dist_init
):
worker = FakeNixlConnectorWorker(
create_vllm_config(), "engine", hand_shake_latency=0
create_vllm_config(block_size=maybe_update_block_size(16)),
"engine",
hand_shake_latency=0,
)
remote_block_len = 2048
@@ -1316,13 +1345,16 @@ class TestNixlHandshake:
heterogeneous TP must NOT raise (previously a NotImplementedError),
and the per-region gate must still reject a wrong block_len.
"""
vllm_config = create_vllm_config()
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(block_size=block_size)
with patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.get_tensor_model_parallel_world_size", # noqa: E501
return_value=2,
):
connector = NixlConnector(
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
vllm_config,
KVConnectorRole.WORKER,
make_kv_cache_config(block_size=block_size),
)
connector.connector_worker = FakeNixlConnectorWorker(
vllm_config, connector.engine_id, hand_shake_latency=0
@@ -1409,7 +1441,8 @@ class TestNixlHandshake:
expected local_block_len * tp_ratio and rejected the valid
handshake.
"""
vllm_config = create_vllm_config()
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(block_size=block_size)
with patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.get_tensor_model_parallel_world_size", # noqa: E501
@@ -1418,7 +1451,7 @@ class TestNixlHandshake:
connector = NixlConnector(
vllm_config,
KVConnectorRole.WORKER,
make_kv_cache_config(block_size=16),
make_kv_cache_config(block_size=block_size),
)
connector.connector_worker = FakeNixlConnectorWorker(
vllm_config, connector.engine_id, hand_shake_latency=0
@@ -1465,7 +1498,8 @@ class TestNixlHandshake:
block_lens when GQA replication is NOT in effect (32 KV heads,
D_TP=4, P_TP=2: head_ratio=4, both sides have >1 head/rank).
"""
vllm_config = create_vllm_config()
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(block_size=block_size)
with patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.get_tensor_model_parallel_world_size", # noqa: E501
@@ -1474,7 +1508,7 @@ class TestNixlHandshake:
connector = NixlConnector(
vllm_config,
KVConnectorRole.WORKER,
make_kv_cache_config(block_size=16),
make_kv_cache_config(block_size=block_size),
)
connector.connector_worker = FakeNixlConnectorWorker(
vllm_config, connector.engine_id, hand_shake_latency=0
@@ -1522,11 +1556,14 @@ class TestNixlHandshake:
)
def test_kv_connector_stats(default_vllm_config, dist_init):
"""Test that KV transfer stats are properly recorded and retrieved."""
vllm_config = create_vllm_config()
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(block_size=block_size)
# Test worker role in decode server.
connector = NixlConnector(
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
vllm_config,
KVConnectorRole.WORKER,
make_kv_cache_config(block_size=block_size),
)
connector.connector_worker = FakeNixlConnectorWorker(
vllm_config, connector.engine_id, hand_shake_latency=0
@@ -1602,9 +1639,12 @@ def test_reqs_to_send_deadline_rebased_to_worker_clock(default_vllm_config, dist
remote read pulls another request's data (silent accuracy corruption).
The worker must anchor the remaining TTL to its own clock.
"""
vllm_config = create_vllm_config()
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(block_size=block_size)
connector = NixlConnector(
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
vllm_config,
KVConnectorRole.WORKER,
make_kv_cache_config(block_size=block_size),
)
connector.connector_worker = FakeNixlConnectorWorker(
vllm_config, connector.engine_id, hand_shake_latency=0
@@ -1793,9 +1833,12 @@ def test_nixl_prom_metrics_group_handshake_with_transfer_failures():
FakeNixlWrapper,
)
def test_notification_after_expiry_is_counted(default_vllm_config, dist_init):
vllm_config = create_vllm_config()
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(block_size=block_size)
connector = NixlConnector(
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
vllm_config,
KVConnectorRole.WORKER,
make_kv_cache_config(block_size=block_size),
)
connector.connector_worker = FakeNixlConnectorWorker(
vllm_config, connector.engine_id, hand_shake_latency=0
@@ -2323,7 +2366,10 @@ def test_register_kv_caches(
2. nixl_wrapper.get_xfer_descs() is called with blocks_data containing
block layout info
"""
vllm_config = create_vllm_config(attention_backend=attn_backend)
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(
attention_backend=attn_backend, block_size=block_size
)
vllm_config.cache_config.kv_cache_layout = layout
# Import the appropriate backend based on the parameter
@@ -2348,7 +2394,6 @@ def test_register_kv_caches(
patch(f"{nixl_worker}.get_current_attn_backends") as mock_get_attn_backends,
):
mock_get_attn_backends.return_value = [backend_cls]
block_size = 16
num_blocks = 8
num_heads = 4
head_size = 16
@@ -2472,7 +2517,7 @@ def test_register_kv_caches(
}
assert len(blocks_data) == num_blocks * len(base_addrs)
assert connector.connector_worker.block_size == 16
assert connector.connector_worker.block_size == block_size
def test_register_packed_dsv4_mla_cache_as_single_region(
@@ -2573,7 +2618,8 @@ def test_kv_buffer_to_nixl_memory_types(
"""Test that register_kv_caches() passes the correct memory types from the
config to the nixl_wrapper.
"""
vllm_config = create_vllm_config()
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(block_size=block_size)
# Override the default memory types in the config
vllm_config.kv_transfer_config.kv_buffer_device = kv_buffer_device
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import (
@@ -2603,7 +2649,9 @@ def test_kv_buffer_to_nixl_memory_types(
): # noqa: E501
# Create connector and replace its worker with a fake one for isolation
connector = NixlConnector(
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
vllm_config,
KVConnectorRole.WORKER,
make_kv_cache_config(block_size=block_size),
)
# Verify get_reg_descs was called with the correct memory_type
@@ -2617,17 +2665,18 @@ def test_kv_buffer_to_nixl_memory_types(
)
def test_shutdown_cleans_up_resources(default_vllm_config, dist_init):
"""Test that shutdown() properly cleans up all resources."""
vllm_config = create_vllm_config()
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(block_size=block_size)
scheduler = NixlConnectorScheduler(
vllm_config,
vllm_config.kv_transfer_config.engine_id,
make_kv_cache_config(block_size=16),
make_kv_cache_config(block_size=block_size),
)
worker = NixlConnectorWorker(
vllm_config,
vllm_config.kv_transfer_config.engine_id,
make_kv_cache_config(block_size=16),
make_kv_cache_config(block_size=block_size),
)
nixl_wrapper = worker.nixl_wrapper
@@ -2643,7 +2692,7 @@ def test_shutdown_cleans_up_resources(default_vllm_config, dist_init):
# Mock register_kv_cache which registers local handle
worker.src_xfer_handles_by_block_size = {worker.block_size: 455}
# P TP = 2 * D TP case, we should register 2 local handles
worker.src_xfer_handles_by_tp_ratio = {(-2, 16): [456, 457]}
worker.src_xfer_handles_by_tp_ratio = {(-2, block_size): [456, 457]}
worker.dst_xfer_side_handles = {"engine1": {0: 789}}
worker._remote_agents = {"engine1": {(0, 0): "agent1"}}
# _cleanup_remote_engine (called by shutdown) also clears these:
@@ -2688,13 +2737,15 @@ def _setup_worker_with_remote_engine(
engine_ttl: float = 10.0,
) -> tuple[Any, str]:
"""Create a worker with one remote engine registered."""
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(
block_size=block_size,
kv_connector_extra_config={"engine_ttl": engine_ttl},
)
worker = NixlConnectorWorker(
vllm_config,
vllm_config.kv_transfer_config.engine_id,
make_kv_cache_config(block_size=16),
make_kv_cache_config(block_size=block_size),
)
engine_id = "remote-engine-1"
@@ -2857,7 +2908,11 @@ class TestPeerReplacement:
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker as bw
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
config = create_vllm_config(kv_connector_extra_config={"engine_ttl": 0})
block_size = maybe_update_block_size(16)
config = create_vllm_config(
kv_connector_extra_config={"engine_ttl": 0}, block_size=block_size
)
config.kv_transfer_config.kv_buffer_device = "cpu"
platform = SimpleNamespace(
device_type="cpu",
@@ -3258,12 +3313,15 @@ def test_aborted_request_removed_from_worker_in_batch(default_vllm_config, dist_
the real scheduler, then simulate an abort (request not in next scheduler
iteration) and verify the worker no longer tracks it as in-batch.
"""
vllm_config = create_vllm_config()
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(block_size=block_size)
scheduler = create_scheduler(vllm_config)
# KVConnector Worker in P
connector = NixlConnector(
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
vllm_config,
KVConnectorRole.WORKER,
make_kv_cache_config(block_size=block_size),
)
connector.connector_worker = FakeNixlConnectorWorker(
vllm_config, connector.engine_id, hand_shake_latency=0
@@ -3398,9 +3456,12 @@ def test_empty_recv_is_reported_only_when_awaited(
A failed notification changes neither: the KV is local either way, and the
producer frees its own blocks on a timeout.
"""
vllm_config = create_vllm_config()
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(block_size=block_size)
connector = NixlConnector(
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
vllm_config,
KVConnectorRole.WORKER,
make_kv_cache_config(block_size=block_size),
)
connector.connector_worker = FakeNixlConnectorWorker(
vllm_config,
@@ -3518,12 +3579,13 @@ def test_transfer_failure_logging(
"""
import logging
vllm_config = create_vllm_config()
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(block_size=block_size)
connector = NixlConnector(
vllm_config,
KVConnectorRole.WORKER,
make_kv_cache_config(block_size=16, swa_enabled=enable_hma),
make_kv_cache_config(block_size=block_size, swa_enabled=enable_hma),
)
connector.connector_worker = FakeNixlConnectorWorker(
vllm_config,
@@ -3648,10 +3710,13 @@ def test_transfer_failure_logging(
)
def test_handshake_failure_returns_finished(default_vllm_config, dist_init):
"""Test that handshake failures mark blocks invalid and return via get_finished."""
vllm_config = create_vllm_config()
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(block_size=block_size)
connector = NixlConnector(
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
vllm_config,
KVConnectorRole.WORKER,
make_kv_cache_config(block_size=block_size),
)
connector.connector_worker = FakeNixlConnectorWorker(
vllm_config, connector.engine_id, hand_shake_latency=0.1
@@ -3704,10 +3769,13 @@ def test_transfer_setup_failure_returns_finished(
default_vllm_config, dist_init, is_hma
):
"""Setup failures report the request; only non-HMA reports block IDs."""
vllm_config = create_vllm_config()
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(block_size=block_size)
connector = NixlConnector(
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
vllm_config,
KVConnectorRole.WORKER,
make_kv_cache_config(block_size=block_size),
)
connector.connector_worker = FakeNixlConnectorWorker(
vllm_config, connector.engine_id, hand_shake_latency=0
@@ -3776,7 +3844,9 @@ class _ScriptedXferWrapper(FakeNixlWrapper):
def _make_split_read_connector(vllm_config, request_id, states):
"""Seed a request with scripted in-flight xfer handles + recv metadata."""
connector = NixlConnector(
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
vllm_config,
KVConnectorRole.WORKER,
make_kv_cache_config(block_size=vllm_config.cache_config.block_size),
)
connector.connector_worker = FakeNixlConnectorWorker(
vllm_config, connector.engine_id, hand_shake_latency=0
@@ -3818,7 +3888,7 @@ def test_split_read_failure_defers_report_until_last_handle(
request_id = "split_read_partial_failure"
err_handle, live_handle = 11, 22
connector, worker, wrapper = _make_split_read_connector(
create_vllm_config(),
create_vllm_config(block_size=maybe_update_block_size(16)),
request_id,
{err_handle: ["ERR"], live_handle: ["PROC", "PROC", "DONE"]},
)
@@ -3894,10 +3964,15 @@ def test_failed_request_skips_kv_postprocessing(
# Use enable_permute_local_kv=True so that
# post_process_device_kv_on_receive would be called on the success path,
# making the assertion meaningful (not trivially true).
vllm_config = create_vllm_config(enable_permute_local_kv=True)
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(
enable_permute_local_kv=True, block_size=block_size
)
connector = NixlConnector(
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
vllm_config,
KVConnectorRole.WORKER,
make_kv_cache_config(block_size=block_size),
)
connector.connector_worker = FakeNixlConnectorWorker(
vllm_config,
@@ -4220,14 +4295,15 @@ def test_compatibility_hash_validation(
enforce_handshake_compat: whether to enforce compatibility checking
"""
block_size = maybe_update_block_size(16)
local_vllm_config = create_vllm_config(
model="facebook/opt-125m",
block_size=16,
block_size=block_size,
kv_connector_extra_config={
"enforce_handshake_compat": enforce_handshake_compat
},
)
kv_cache_config = make_kv_cache_config(block_size=16, num_blocks=2)
kv_cache_config = make_kv_cache_config(block_size=block_size, num_blocks=2)
decode_connector = NixlConnector(
local_vllm_config, KVConnectorRole.WORKER, kv_cache_config
)
@@ -4253,7 +4329,7 @@ def test_compatibility_hash_validation(
remote_config_params: dict[str, Any] = {
"model": "facebook/opt-125m",
"block_size": 16,
"block_size": block_size,
**config_overrides,
}
remote_vllm_config = create_vllm_config(**remote_config_params)
@@ -4276,7 +4352,7 @@ def test_compatibility_hash_validation(
decode_worker.backend_name,
)
prefill_block_size = config_overrides.get("block_size", 16)
prefill_block_size = remote_vllm_config.cache_config.block_size
prefill_block_lens = [4096 * prefill_block_size]
prefill_metadata = NixlAgentMetadata(
engine_id=FakeNixlConnectorWorker.REMOTE_ENGINE_ID,
@@ -4352,12 +4428,15 @@ def test_handshake_decode_errors(default_vllm_config, dist_init, error_scenario)
- NixlHandshakePayload decoder
- NixlAgentMetadata decoder
"""
block_size = maybe_update_block_size(16)
local_vllm_config = create_vllm_config(
model="facebook/opt-125m",
block_size=16,
block_size=block_size,
)
decode_connector = NixlConnector(
local_vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
local_vllm_config,
KVConnectorRole.WORKER,
make_kv_cache_config(block_size=block_size),
)
decode_worker = decode_connector.connector_worker
@@ -4441,11 +4520,14 @@ def test_handshake_decode_errors(default_vllm_config, dist_init, error_scenario)
decode_tp_size = 1
prefill_tp_size = 4
vllm_config = create_vllm_config()
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(block_size=block_size)
vllm_config.parallel_config.tensor_parallel_size = decode_tp_size
connector = NixlConnector(
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
vllm_config,
KVConnectorRole.WORKER,
make_kv_cache_config(block_size=block_size),
)
connector.connector_worker = FakeNixlConnectorWorker(
vllm_config, connector.engine_id, hand_shake_latency=0
@@ -4550,7 +4632,8 @@ def test_kv_both_deprecation_warning(default_vllm_config, dist_init):
_print_warning_once.cache_clear()
vllm_config = create_vllm_config(kv_role="kv_both")
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(kv_role="kv_both", block_size=block_size)
with patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.connector.logger"
@@ -4559,7 +4642,7 @@ def test_kv_both_deprecation_warning(default_vllm_config, dist_init):
NixlConnector(
vllm_config,
KVConnectorRole.WORKER,
make_kv_cache_config(block_size=16),
make_kv_cache_config(block_size=block_size),
)
mock_logger.warning_once.assert_called_once()
@@ -4575,14 +4658,15 @@ def test_kv_both_deprecation_warning(default_vllm_config, dist_init):
def test_explicit_kv_role_no_deprecation_warning(default_vllm_config, dist_init):
"""kv_role='kv_consumer' or 'kv_producer' should NOT emit a warning."""
for role in ("kv_consumer", "kv_producer"):
vllm_config = create_vllm_config(kv_role=role)
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(kv_role=role, block_size=block_size)
with patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.connector.logger"
) as mock_logger:
NixlConnector(
vllm_config,
KVConnectorRole.WORKER,
make_kv_cache_config(block_size=16),
make_kv_cache_config(block_size=block_size),
)
(
@@ -39,6 +39,7 @@ from .utils import (
create_vllm_config,
make_kv_cache_config,
make_nixl_scheduler,
maybe_update_block_size,
)
@@ -1027,7 +1028,7 @@ def test_fewer_blocks_with_hma(monkeypatch, model_name, sw_size):
kv_connector="NixlConnector",
kv_role="kv_consumer",
)
block_size = 16
block_size = maybe_update_block_size(16)
llm_kwargs = {
"model": model_name,
"enforce_eager": True,
+5
View File
@@ -31,6 +31,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
from vllm.distributed.kv_transfer.kv_connector.v1.example_connector import ( # noqa
ExampleConnector,
)
from vllm.platforms import current_platform
from vllm.utils.hashing import sha256
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash
@@ -59,6 +60,10 @@ def create_mock_connector() -> Mock:
return connector
def maybe_update_block_size(block_size: int) -> int:
return 64 if current_platform.is_xpu() else block_size
def assert_scheduler_empty(scheduler: Scheduler):
"""Confirm the scheduler is "empty" - i.e. no leaks."""
# Scheduler Metadata.
+2
View File
@@ -355,6 +355,8 @@ class FlashAttentionBackend(AttentionBackend):
def get_supported_kernel_block_sizes(
cls, kv_cache_spec: KVCacheSpec | None = None
) -> list[int | MultipleOf]:
if current_platform.is_xpu():
return [MultipleOf(64)]
if block_size := cls._get_sm90_fa4_fp8_kv_block_size(kv_cache_spec):
return [block_size]
if block_size := cls._get_fa4_hd256_block_size(kv_cache_spec):
+19 -1
View File
@@ -14,9 +14,11 @@ from vllm.config import (
get_layers_from_vllm_config,
)
from vllm.config.compilation import CUDAGraphMode
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.platforms import current_platform
from vllm.v1.attention.backend import (
AttentionCGSupport,
CommonAttentionMetadata,
@@ -45,6 +47,8 @@ from vllm.v1.worker.utils import (
prepare_kernel_block_sizes,
)
logger = init_logger(__name__)
if TYPE_CHECKING:
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.cudagraph_utils import (
@@ -263,10 +267,24 @@ def init_attn_backend(
# by all backends within that group.
kernel_block_sizes = prepare_kernel_block_sizes(kv_cache_config, attn_groups)
if current_platform.is_xpu():
for kv_cache_group_id, selected_kernel_block_size in enumerate(
kernel_block_sizes
):
if selected_kernel_block_size % 64 != 0:
for attn_group in attn_groups[kv_cache_group_id]:
logger.warning_once(
"On XPU platform, current selected kernel block size is "
"not fully optimized: kernel_block_size=%s, backend=%s, "
"kv_cache_spec=%s",
selected_kernel_block_size,
attn_group.backend,
repr(attn_group.kv_cache_spec),
)
# Phase 3: create metadata builders and determine cudagraph support.
attn_backend_workspace: torch.Tensor | None = None
for kv_cache_group_id, groups in enumerate(attn_groups):
kernel_block_size = None
kernel_block_size: int | None = None
if kv_cache_group_id < len(kernel_block_sizes):
kernel_block_size = kernel_block_sizes[kv_cache_group_id]
for group in groups: