diff --git a/.buildkite/intel_jobs/test-intel.yaml b/.buildkite/intel_jobs/test-intel.yaml index a2a0e14be1..1567c0eba7 100644 --- a/.buildkite/intel_jobs/test-intel.yaml +++ b/.buildkite/intel_jobs/test-intel.yaml @@ -105,7 +105,7 @@ steps: pytest -v -s v1/test_serial_utils.py && pytest -v -s v1/e2e/general/test_correctness_sliding_window.py && pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py --ignore=v1/spec_decode/test_speculators_correctness.py --deselect=tests/v1/spec_decode/test_mtp.py::test_glm_mtp_defers_lm_head && - pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py --ignore=v1/kv_connector/unit/test_nixl_connector_hma.py --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --deselect=tests/v1/kv_connector/unit/test_nixl_connector.py::test_abort_timeout_on_prefiller[ray]' + pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --deselect=tests/v1/kv_connector/unit/test_nixl_connector.py::test_abort_timeout_on_prefiller[ray]' parallelism: 2 - label: "XPU Qwen3-30B + V1 KV Connector" key: xpu-qwen3-30b-v1-kv-connector diff --git a/tests/v1/kv_connector/unit/test_bidirectional_kv_transfer.py b/tests/v1/kv_connector/unit/test_bidirectional_kv_transfer.py index 3d539f97c8..2aae7d71f5 100644 --- a/tests/v1/kv_connector/unit/test_bidirectional_kv_transfer.py +++ b/tests/v1/kv_connector/unit/test_bidirectional_kv_transfer.py @@ -51,6 +51,7 @@ from .utils import ( create_scheduler, create_vllm_config, make_kv_cache_config, + maybe_update_block_size, ) pytestmark = pytest.mark.cpu_test @@ -87,10 +88,11 @@ def _make_connector_with_fake_worker( hand_shake_latency=0, cycles_before_done=0, do_handshake=True ): """Create a NixlConnector with FakeNixlConnectorWorker.""" + block_size = maybe_update_block_size(32) vllm_config = create_vllm_config( - kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, block_size=32 + kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, block_size=block_size ) - kv_cache_config = make_kv_cache_config(block_size=32, num_blocks=2) + kv_cache_config = make_kv_cache_config(block_size=block_size, num_blocks=2) connector = NixlConnector(vllm_config, KVConnectorRole.WORKER, kv_cache_config) connector.connector_worker = FakeNixlConnectorWorker( vllm_config, diff --git a/tests/v1/kv_connector/unit/test_mooncake_connector.py b/tests/v1/kv_connector/unit/test_mooncake_connector.py index d326615aad..f79d7aa316 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_connector.py +++ b/tests/v1/kv_connector/unit/test_mooncake_connector.py @@ -57,7 +57,12 @@ from vllm.v1.kv_cache_interface import ( ) from vllm.v1.request import RequestStatus -from .utils import create_request, create_scheduler, create_vllm_config +from .utils import ( + create_request, + create_scheduler, + create_vllm_config, + maybe_update_block_size, +) @pytest.mark.parametrize( @@ -148,6 +153,7 @@ def test_region_length_validation_checks_replicated_gqa_heads( def _make_test_kv_cache_config() -> KVCacheConfig: + block_size = maybe_update_block_size(16) return KVCacheConfig( num_blocks=0, kv_cache_tensors=[], @@ -160,7 +166,7 @@ def _make_test_kv_cache_config() -> KVCacheConfig: "model.layers.1.eagle_attn", ], FullAttentionSpec( - block_size=16, + block_size=block_size, num_kv_heads=4, head_size=64, dtype=torch.float16, @@ -170,6 +176,14 @@ def _make_test_kv_cache_config() -> KVCacheConfig: ) +def _make_worker_vllm_config(kv_role: str): + return create_vllm_config( + kv_connector="MooncakeConnector", + kv_role=kv_role, + block_size=maybe_update_block_size(16), + ) + + def _make_packed_mla_kv_cache_config( num_blocks: int, *, num_groups: int = 1 ) -> KVCacheConfig: @@ -429,9 +443,7 @@ async def test_send_kv_to_decode_aligns_consumer_regions_by_layer_metadata( ): """Producer sends its PP layer shard to the matching consumer layer address.""" monkeypatch.setenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "5") - vllm_config = create_vllm_config( - kv_connector="MooncakeConnector", kv_role="kv_producer" - ) + vllm_config = _make_worker_vllm_config("kv_producer") with set_current_vllm_config(vllm_config), patch_worker_dependencies(): prefill_connector = MooncakeConnector( @@ -1205,7 +1217,7 @@ def _xfer_meta( @contextlib.contextmanager def mooncake_register_worker(kv_cache_config, *, kv_role: str = "kv_consumer"): """Build a worker whose engine registration and sender thread are patched.""" - vllm_config = create_vllm_config(kv_connector="MooncakeConnector", kv_role=kv_role) + vllm_config = _make_worker_vllm_config(kv_role) connector_mod = ( "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector" ) @@ -1241,9 +1253,7 @@ async def test_receive_kv_selects_remote_pp_workers( expected_addrs: list[str], ): """Decode workers should not hard-code producer pp_rank 0.""" - vllm_config = create_vllm_config( - kv_connector="MooncakeConnector", kv_role="kv_consumer" - ) + vllm_config = _make_worker_vllm_config("kv_consumer") with set_current_vllm_config(vllm_config), patch_worker_dependencies(): decode_connector = MooncakeConnector( @@ -1314,7 +1324,7 @@ def test_resolve_need_send_accounts_for_remote_tp_fanout(): @pytest.mark.asyncio async def test_heterogeneous_pp_waits_for_consumer_without_shared_layers(): """P4/D2 must retain KV until both D stages finish, including a no-op pull.""" - config = create_vllm_config(kv_connector="MooncakeConnector", kv_role="kv_producer") + config = _make_worker_vllm_config("kv_producer") with ( set_current_vllm_config(config), patch_worker_dependencies(), @@ -1386,9 +1396,7 @@ async def test_kv_producer(monkeypatch): Verifies memory offset calculation: ptr = base_addr + block_id * block_len. """ monkeypatch.setenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "5") - vllm_config = create_vllm_config( - kv_connector="MooncakeConnector", kv_role="kv_producer" - ) + vllm_config = _make_worker_vllm_config("kv_producer") with set_current_vllm_config(vllm_config), patch_worker_dependencies(): prefill_connector = MooncakeConnector( @@ -1589,9 +1597,7 @@ async def test_kv_consumuer(monkeypatch): Verifies that MooncakeXferMetadata is correctly serialized and sent via ZMQ. """ - vllm_config = create_vllm_config( - kv_connector="MooncakeConnector", kv_role="kv_consumer" - ) + vllm_config = _make_worker_vllm_config("kv_consumer") with set_current_vllm_config(vllm_config), patch_worker_dependencies() as mocks: decode_connector = MooncakeConnector( @@ -1661,9 +1667,7 @@ async def test_kv_consumuer(monkeypatch): @pytest.mark.asyncio async def test_worker_get_finished_timeout(monkeypatch): """Tests the cleanup mechanism for requests.""" - vllm_config = create_vllm_config( - kv_connector="MooncakeConnector", kv_role="kv_producer" - ) + vllm_config = _make_worker_vllm_config("kv_producer") with set_current_vllm_config(vllm_config), patch_worker_dependencies(): prefill_connector = MooncakeConnector( vllm_config, @@ -1713,7 +1717,7 @@ async def test_worker_get_finished_timeout(monkeypatch): def test_register_kv_caches(layout: KVCacheLayout, separate_kv_head_groups: bool): """Tests the memory registration logic with the underlying Mooncake engine.""" spec = FullAttentionSpec( - block_size=16, + block_size=maybe_update_block_size(16), num_kv_heads=4, head_size=64, dtype=torch.float16, @@ -1794,7 +1798,10 @@ def test_register_noncontiguous_packed_allows_matching_pp(): known. Rejecting every local PP>1 layout at startup is too strict. """ spec = FullAttentionSpec( - block_size=16, num_kv_heads=4, head_size=64, dtype=torch.float16 + block_size=maybe_update_block_size(16), + num_kv_heads=4, + head_size=64, + dtype=torch.float16, ) layer_names = ["model.layers.0.self_attn", "model.layers.1.self_attn"] with mooncake_register_worker(_make_test_kv_cache_config()) as (worker, _reg): @@ -1814,8 +1821,9 @@ def test_register_kv_caches_supports_mixed_mla_and_eagle_shapes(): with mooncake_register_worker(_make_test_kv_cache_config()) as (worker, reg): worker.use_mla = True worker.transfer_topo.is_mla = True - mla_cache = torch.zeros((2, 16, 512), dtype=torch.float16) - eagle_cache = torch.zeros((2, 16, 8, 64), dtype=torch.float16) + block_size = maybe_update_block_size(16) + mla_cache = torch.zeros((2, block_size, 512), dtype=torch.float16) + eagle_cache = torch.zeros((2, block_size, 8, 64), dtype=torch.float16) kv_caches = { "model.layers.0.mla_attn": mla_cache, "model.layers.1.eagle_attn": eagle_cache, @@ -2240,7 +2248,7 @@ def _register_sliced_packed_mla( ): """Register column slices of one MLA packed row. Returns worker fields.""" spec = MLAAttentionSpec( - block_size=16, + block_size=maybe_update_block_size(16), num_kv_heads=1, head_size=64, dtype=torch.uint8, @@ -2286,7 +2294,7 @@ def test_sliced_packed_mla_registers_each_layer_view(): KVCacheGroupSpec( [_layer_name(i) for i in layer_idxs], MLAAttentionSpec( - block_size=16, + block_size=maybe_update_block_size(16), num_kv_heads=1, head_size=64, dtype=torch.uint8, @@ -2436,9 +2444,7 @@ async def test_kv_producer_heterogeneous_tp(monkeypatch, d_tp_size): remote_block_len = LOCAL_BLOCK_LEN * P_TP_SIZE // d_tp_size monkeypatch.setenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "5") - vllm_config = create_vllm_config( - kv_connector="MooncakeConnector", kv_role="kv_producer" - ) + vllm_config = _make_worker_vllm_config("kv_producer") with set_current_vllm_config(vllm_config), patch_worker_dependencies(): prefill_connector = MooncakeConnector( diff --git a/tests/v1/kv_connector/unit/test_mooncake_connector_hma.py b/tests/v1/kv_connector/unit/test_mooncake_connector_hma.py index a7fb2be023..87dafebb4c 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_connector_hma.py +++ b/tests/v1/kv_connector/unit/test_mooncake_connector_hma.py @@ -30,7 +30,12 @@ from vllm.v1.kv_cache_interface import ( ) from .test_mooncake_connector import FakeMooncakeWrapper, patch_worker_dependencies -from .utils import create_request, create_vllm_config, make_kv_cache_config +from .utils import ( + create_request, + create_vllm_config, + make_kv_cache_config, + maybe_update_block_size, +) # --------------------------------------------------------------------------- @@ -224,12 +229,11 @@ def test_metadata_hma_block_ids(): async def test_build_transfer_params_multi_group_trimming(monkeypatch): """_build_transfer_params trims per-group blocks when local > remote.""" monkeypatch.setenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "5") + block_size = maybe_update_block_size(16) vllm_config = create_vllm_config( - kv_connector="MooncakeConnector", kv_role="kv_producer" - ) - kv_cache_config = make_kv_cache_config( - block_size=vllm_config.cache_config.block_size, swa_enabled=True + kv_connector="MooncakeConnector", kv_role="kv_producer", block_size=block_size ) + kv_cache_config = make_kv_cache_config(block_size=block_size, swa_enabled=True) with set_current_vllm_config(vllm_config), patch_worker_dependencies(): connector = MooncakeConnector( @@ -320,12 +324,11 @@ async def test_build_transfer_params_multi_group_trimming(monkeypatch): async def test_build_transfer_params_group_count_mismatch(monkeypatch): """_build_transfer_params reports an error when group counts differ.""" monkeypatch.setenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "5") + block_size = maybe_update_block_size(16) vllm_config = create_vllm_config( - kv_connector="MooncakeConnector", kv_role="kv_producer" - ) - kv_cache_config = make_kv_cache_config( - block_size=vllm_config.cache_config.block_size, swa_enabled=True + kv_connector="MooncakeConnector", kv_role="kv_producer", block_size=block_size ) + kv_cache_config = make_kv_cache_config(block_size=block_size, swa_enabled=True) with set_current_vllm_config(vllm_config), patch_worker_dependencies(): connector = MooncakeConnector( @@ -449,7 +452,7 @@ def test_request_finished_with_hma_groups(): def _make_kv_consumer_worker( swa_enabled: bool = False, kv_cache_config: KVCacheConfig | None = None ): - block_size = 16 + block_size = maybe_update_block_size(16) vllm_config = create_vllm_config( kv_connector="MooncakeConnector", kv_role="kv_consumer", @@ -495,6 +498,7 @@ def test_worker_is_hma_required(swa_enabled, expected_is_hma): def test_worker_is_hma_required_multiple_full_attention_groups(): """Two full-attention groups with different block sizes also select request-level failure reporting, matching the scheduler.""" + block_size = maybe_update_block_size(16) kv_cache_config = KVCacheConfig( num_blocks=100, kv_cache_tensors=[], @@ -502,13 +506,19 @@ def test_worker_is_hma_required_multiple_full_attention_groups(): KVCacheGroupSpec( ["layer0"], FullAttentionSpec( - block_size=16, num_kv_heads=4, head_size=16, dtype=torch.float16 + block_size=block_size, + num_kv_heads=4, + head_size=16, + dtype=torch.float16, ), ), KVCacheGroupSpec( ["layer1"], FullAttentionSpec( - block_size=32, num_kv_heads=4, head_size=16, dtype=torch.float16 + block_size=2 * block_size, + num_kv_heads=4, + head_size=16, + dtype=torch.float16, ), ), ], diff --git a/tests/v1/kv_connector/unit/test_mooncake_connector_hybrid_mamba.py b/tests/v1/kv_connector/unit/test_mooncake_connector_hybrid_mamba.py index 969bf65f8c..65abd910bb 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_connector_hybrid_mamba.py +++ b/tests/v1/kv_connector/unit/test_mooncake_connector_hybrid_mamba.py @@ -36,7 +36,7 @@ from vllm.v1.kv_cache_interface import ( ) from .test_mooncake_connector import patch_worker_dependencies -from .utils import create_request, create_vllm_config +from .utils import create_request, create_vllm_config, maybe_update_block_size def noop_shutdown(): @@ -147,6 +147,7 @@ def test_register_kv_caches_emits_fa_and_gdn_regions(monkeypatch): vllm_config = create_vllm_config( kv_connector="MooncakeConnector", kv_role="kv_consumer", + block_size=maybe_update_block_size(16), ) kv_cache_config = make_hybrid_gdn_kv_cache_config( vllm_config.cache_config.block_size @@ -203,6 +204,7 @@ def test_register_kv_caches_scales_attention_len_to_kernel_block(monkeypatch): vllm_config = create_vllm_config( kv_connector="MooncakeConnector", kv_role="kv_consumer", + block_size=maybe_update_block_size(16), ) kv_cache_config = make_hybrid_gdn_kv_cache_config( vllm_config.cache_config.block_size @@ -392,6 +394,7 @@ def test_register_kv_caches_deduplicates_shared_backing_memory(monkeypatch): vllm_config = create_vllm_config( kv_connector="MooncakeConnector", kv_role="kv_consumer", + block_size=maybe_update_block_size(16), ) kv_cache_config = make_hybrid_gdn_kv_cache_config( vllm_config.cache_config.block_size @@ -405,8 +408,13 @@ def test_register_kv_caches_deduplicates_shared_backing_memory(monkeypatch): ) worker = connector.connector_worker - backing = torch.empty((4, 64), dtype=torch.float16) - fa_cache = backing[:2, :16] + fa_spec = kv_cache_config.kv_cache_groups[0].kv_cache_spec + fa_cache_width = ( + fa_spec.page_size_bytes + // torch.tensor([], dtype=torch.float16).element_size() + ) + backing = torch.empty((4, fa_cache_width), dtype=torch.float16) + fa_cache = backing[:2] gdn_cache = backing[:3] with patch.object( @@ -438,6 +446,7 @@ def test_hybrid_gdn_transfer_params_preserve_group_identity(monkeypatch): vllm_config = create_vllm_config( kv_connector="MooncakeConnector", kv_role="kv_producer", + block_size=maybe_update_block_size(16), ) kv_cache_config = make_hybrid_gdn_kv_cache_config( vllm_config.cache_config.block_size @@ -567,6 +576,7 @@ def test_hybrid_gdn_keeps_packed_fa_and_gdn_regions_whole( vllm_config = create_vllm_config( kv_connector="MooncakeConnector", kv_role="kv_producer", + block_size=maybe_update_block_size(16), ) kv_cache_config = make_hybrid_gdn_kv_cache_config( vllm_config.cache_config.block_size diff --git a/tests/v1/kv_connector/unit/test_multi_connector.py b/tests/v1/kv_connector/unit/test_multi_connector.py index e576e3d0d9..4ac3eaf7c5 100644 --- a/tests/v1/kv_connector/unit/test_multi_connector.py +++ b/tests/v1/kv_connector/unit/test_multi_connector.py @@ -10,7 +10,7 @@ from unittest.mock import MagicMock import pytest import torch -from tests.v1.kv_connector.unit.utils import create_vllm_config +from tests.v1.kv_connector.unit.utils import create_vllm_config, maybe_update_block_size from vllm import LLM, SamplingParams from vllm.config import KVTransferConfig from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory @@ -314,10 +314,13 @@ def test_multi_example_connector_consistency(): }, ) + block_size = maybe_update_block_size(16) + expected_num_blocks = 2 if block_size == 64 else 7 + expected_num_matched_tokens = (expected_num_blocks - 1) * block_size llm = LLM( model=MODEL_NAME, enforce_eager=True, - block_size=16, + block_size=block_size, gpu_memory_utilization=0.5, kv_transfer_config=kv_transfer_config, async_scheduling=False, @@ -372,7 +375,7 @@ def test_multi_example_connector_consistency(): "set_xfer_handshake_metadata_pp_aware", "on_new_request", "get_num_new_matched_tokens 0", - "update_state_after_alloc num_blocks=[7] 0", + f"update_state_after_alloc num_blocks=[{expected_num_blocks}] 0", "build_connector_meta", ] # First three events are from initialization. Layer hooks run before the @@ -399,7 +402,7 @@ def test_multi_example_connector_consistency(): "set_xfer_handshake_metadata_pp_aware", "on_new_request", "get_num_new_matched_tokens 0", - "update_state_after_alloc num_blocks=[7] 0", + f"update_state_after_alloc num_blocks=[{expected_num_blocks}] 0", "build_connector_meta", ] # Reset prefix cache or else we'll just get the tokens back from there. @@ -423,13 +426,14 @@ def test_multi_example_connector_consistency(): assert storage1_scheduler_events[:4] == [ "on_new_request", "get_num_new_matched_tokens 0", - "update_state_after_alloc num_blocks=[7] 96", + "update_state_after_alloc " + f"num_blocks=[{expected_num_blocks}] {expected_num_matched_tokens}", "build_connector_meta", ] assert storage2_scheduler_events[:4] == [ "on_new_request", "get_num_new_matched_tokens 0", - "update_state_after_alloc num_blocks=[7] 0", + f"update_state_after_alloc num_blocks=[{expected_num_blocks}] 0", "build_connector_meta", ] @@ -457,13 +461,14 @@ def test_multi_example_connector_consistency(): assert storage1_scheduler_events[:4] == [ "on_new_request", "get_num_new_matched_tokens 0", - "update_state_after_alloc num_blocks=[7] 0", + f"update_state_after_alloc num_blocks=[{expected_num_blocks}] 0", "build_connector_meta", ] assert storage2_scheduler_events[:4] == [ "on_new_request", "get_num_new_matched_tokens 0", - "update_state_after_alloc num_blocks=[7] 96", + "update_state_after_alloc " + f"num_blocks=[{expected_num_blocks}] {expected_num_matched_tokens}", "build_connector_meta", ] diff --git a/tests/v1/kv_connector/unit/test_nixl_connector.py b/tests/v1/kv_connector/unit/test_nixl_connector.py index d918c38fbf..607f5ff3cd 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector.py @@ -84,6 +84,7 @@ from .utils import ( create_scheduler, create_vllm_config, make_kv_cache_config, + maybe_update_block_size, ) @@ -418,7 +419,8 @@ def test_kv_transfer_handshake(dist_init): # Test setup, we creates a scheduler that contains a NixlConnector # of role SCHEDULER, and expect it to be serving NixlAgentMetadata from # all workers of the instance. - vllm_config = create_vllm_config() + block_size = maybe_update_block_size(16) + vllm_config = create_vllm_config(block_size=block_size) vllm_config.cache_config.kv_cache_layout = "BLHNC" # in case the test runs on non-GPU machine vllm_config.kv_transfer_config.kv_buffer_device = "cpu" @@ -434,7 +436,7 @@ def test_kv_transfer_handshake(dist_init): KVCacheGroupSpec( ["layer0", "layer1", "layer2"], FullAttentionSpec( - block_size=16, + block_size=block_size, num_kv_heads=4, head_size=16, dtype=torch.float16, @@ -544,7 +546,9 @@ class FakeNixlConnectorWorker(NixlConnectorWorker): **kwargs, ): if kv_cache_config is None: - kv_cache_config = make_kv_cache_config(block_size=16) + kv_cache_config = make_kv_cache_config( + block_size=args[0].cache_config.block_size + ) super().__init__(*args, kv_cache_config=kv_cache_config, **kwargs) self._hand_shake_latency = hand_shake_latency self.kv_cache_layout = kv_cache_layout @@ -674,7 +678,8 @@ class TestNixlHandshake: """Replicated PCP is canonicalized; PCP-DCP publishes every shard.""" from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend - vllm_config = create_vllm_config(kv_role="kv_producer") + block_size = maybe_update_block_size(16) + vllm_config = create_vllm_config(kv_role="kv_producer", block_size=block_size) vllm_config.parallel_config.prefill_context_parallel_size = pcp_size vllm_config.parallel_config.decode_context_parallel_size = dcp_size with ( @@ -692,7 +697,7 @@ class TestNixlHandshake: connector = NixlConnector( vllm_config, KVConnectorRole.WORKER, - make_kv_cache_config(block_size=16), + make_kv_cache_config(block_size=block_size), ) worker = connector.connector_worker @@ -746,12 +751,13 @@ class TestNixlHandshake: This test triggers the connector to load remote KV for the same `request_id`. """ - vllm_config = create_vllm_config() + block_size = maybe_update_block_size(16) + vllm_config = create_vllm_config(block_size=block_size) request_id = "req_id" # Test worker role in decode server. - kv_cache_config = make_kv_cache_config(block_size=16, num_blocks=10) + kv_cache_config = make_kv_cache_config(block_size=block_size, num_blocks=10) connector = NixlConnector(vllm_config, KVConnectorRole.WORKER, kv_cache_config) connector.connector_worker = FakeNixlConnectorWorker( vllm_config, @@ -837,12 +843,15 @@ class TestNixlHandshake: prefill_tp_size, ): """Test that NixlConnector's start_load_kv should be non-blocking.""" - vllm_config = create_vllm_config() + block_size = maybe_update_block_size(16) + vllm_config = create_vllm_config(block_size=block_size) vllm_config.parallel_config.tensor_parallel_size = decode_tp_size # Test worker role in decode server. connector = NixlConnector( - vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16) + vllm_config, + KVConnectorRole.WORKER, + make_kv_cache_config(block_size=block_size), ) connector.connector_worker = FakeNixlConnectorWorker( vllm_config, connector.engine_id @@ -899,12 +908,13 @@ class TestNixlHandshake: lambda: local_tp_size, ) - vllm_config = create_vllm_config() + block_size = maybe_update_block_size(16) + vllm_config = create_vllm_config(block_size=block_size) # num_blocks=1 keeps the fake handshake's region_num_blocks consistent # with the minimal single-block registration below (num_descs == 1); # _fa_desc_replicated cross-checks the two. - kv_cache_config = make_kv_cache_config(block_size=16, num_blocks=1) + kv_cache_config = make_kv_cache_config(block_size=block_size, num_blocks=1) connector = NixlConnector(vllm_config, KVConnectorRole.WORKER, kv_cache_config) connector.connector_worker = FakeNixlConnectorWorker( vllm_config, @@ -972,16 +982,21 @@ class TestNixlHandshake: """Verify remote TP > local TP handshake succeeds with different remote configurations for an MLA model. """ - vllm_config = create_vllm_config() + block_size = maybe_update_block_size(16) + vllm_config = create_vllm_config(block_size=block_size) d_tp_size = 1 p_tp_size = 2 # Build two separate connectors/workers to emulate P TP=2 ranks. conn_p0 = NixlConnector( - vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16) + vllm_config, + KVConnectorRole.WORKER, + make_kv_cache_config(block_size=block_size), ) conn_p1 = NixlConnector( - vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16) + vllm_config, + KVConnectorRole.WORKER, + make_kv_cache_config(block_size=block_size), ) conn_p0.connector_worker = FakeNixlConnectorWorker( vllm_config, conn_p0.engine_id, hand_shake_latency=0 @@ -1075,11 +1090,14 @@ class TestNixlHandshake: dist_init, ): """Test that multiple start_load_kv calls should occur concurrently.""" - vllm_config = create_vllm_config() + block_size = maybe_update_block_size(16) + vllm_config = create_vllm_config(block_size=block_size) # Test worker role in decode server. connector = NixlConnector( - vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16) + vllm_config, + KVConnectorRole.WORKER, + make_kv_cache_config(block_size=block_size), ) connector.connector_worker = FakeNixlConnectorWorker( vllm_config, connector.engine_id @@ -1143,8 +1161,10 @@ class TestNixlHandshake: """Verify that adding a remote agent fails if kv_cache_layout differs. This test is only relevant for heterogeneous TP. """ + block_size = maybe_update_block_size(16) vllm_config = create_vllm_config( - enable_permute_local_kv=enable_permute_local_kv + enable_permute_local_kv=enable_permute_local_kv, + block_size=block_size, ) # Mock TP world size to 2 to force heterogeneous TP when @@ -1155,7 +1175,9 @@ class TestNixlHandshake: ): # Initialize connector and worker (with fake NIXL wrapper) connector = NixlConnector( - vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16) + vllm_config, + KVConnectorRole.WORKER, + make_kv_cache_config(block_size=block_size), ) connector.connector_worker = FakeNixlConnectorWorker( vllm_config, @@ -1205,7 +1227,10 @@ class TestNixlHandshake: """Verify that adding a remote agent fails if kv_cache_layout differs. This test is only relevant for heterogeneous TP. """ - vllm_config = create_vllm_config(enable_permute_local_kv=True) + block_size = maybe_update_block_size(16) + vllm_config = create_vllm_config( + enable_permute_local_kv=True, block_size=block_size + ) # Mock TP world size to 2 to force heterogeneous TP when # remote_tp_size=1 @@ -1215,7 +1240,9 @@ class TestNixlHandshake: ): # Initialize connector and worker (with fake NIXL wrapper) connector = NixlConnector( - vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16) + vllm_config, + KVConnectorRole.WORKER, + make_kv_cache_config(block_size=block_size), ) connector.connector_worker = FakeNixlConnectorWorker( vllm_config, @@ -1261,7 +1288,9 @@ class TestNixlHandshake: self, default_vllm_config, dist_init ): worker = FakeNixlConnectorWorker( - create_vllm_config(), "engine", hand_shake_latency=0 + create_vllm_config(block_size=maybe_update_block_size(16)), + "engine", + hand_shake_latency=0, ) remote_block_len = 2048 @@ -1316,13 +1345,16 @@ class TestNixlHandshake: heterogeneous TP must NOT raise (previously a NotImplementedError), and the per-region gate must still reject a wrong block_len. """ - vllm_config = create_vllm_config() + block_size = maybe_update_block_size(16) + vllm_config = create_vllm_config(block_size=block_size) with patch( "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.get_tensor_model_parallel_world_size", # noqa: E501 return_value=2, ): connector = NixlConnector( - vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16) + vllm_config, + KVConnectorRole.WORKER, + make_kv_cache_config(block_size=block_size), ) connector.connector_worker = FakeNixlConnectorWorker( vllm_config, connector.engine_id, hand_shake_latency=0 @@ -1409,7 +1441,8 @@ class TestNixlHandshake: expected local_block_len * tp_ratio and rejected the valid handshake. """ - vllm_config = create_vllm_config() + block_size = maybe_update_block_size(16) + vllm_config = create_vllm_config(block_size=block_size) with patch( "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.get_tensor_model_parallel_world_size", # noqa: E501 @@ -1418,7 +1451,7 @@ class TestNixlHandshake: connector = NixlConnector( vllm_config, KVConnectorRole.WORKER, - make_kv_cache_config(block_size=16), + make_kv_cache_config(block_size=block_size), ) connector.connector_worker = FakeNixlConnectorWorker( vllm_config, connector.engine_id, hand_shake_latency=0 @@ -1465,7 +1498,8 @@ class TestNixlHandshake: block_lens when GQA replication is NOT in effect (32 KV heads, D_TP=4, P_TP=2: head_ratio=4, both sides have >1 head/rank). """ - vllm_config = create_vllm_config() + block_size = maybe_update_block_size(16) + vllm_config = create_vllm_config(block_size=block_size) with patch( "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.get_tensor_model_parallel_world_size", # noqa: E501 @@ -1474,7 +1508,7 @@ class TestNixlHandshake: connector = NixlConnector( vllm_config, KVConnectorRole.WORKER, - make_kv_cache_config(block_size=16), + make_kv_cache_config(block_size=block_size), ) connector.connector_worker = FakeNixlConnectorWorker( vllm_config, connector.engine_id, hand_shake_latency=0 @@ -1522,11 +1556,14 @@ class TestNixlHandshake: ) def test_kv_connector_stats(default_vllm_config, dist_init): """Test that KV transfer stats are properly recorded and retrieved.""" - vllm_config = create_vllm_config() + block_size = maybe_update_block_size(16) + vllm_config = create_vllm_config(block_size=block_size) # Test worker role in decode server. connector = NixlConnector( - vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16) + vllm_config, + KVConnectorRole.WORKER, + make_kv_cache_config(block_size=block_size), ) connector.connector_worker = FakeNixlConnectorWorker( vllm_config, connector.engine_id, hand_shake_latency=0 @@ -1602,9 +1639,12 @@ def test_reqs_to_send_deadline_rebased_to_worker_clock(default_vllm_config, dist remote read pulls another request's data (silent accuracy corruption). The worker must anchor the remaining TTL to its own clock. """ - vllm_config = create_vllm_config() + block_size = maybe_update_block_size(16) + vllm_config = create_vllm_config(block_size=block_size) connector = NixlConnector( - vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16) + vllm_config, + KVConnectorRole.WORKER, + make_kv_cache_config(block_size=block_size), ) connector.connector_worker = FakeNixlConnectorWorker( vllm_config, connector.engine_id, hand_shake_latency=0 @@ -1793,9 +1833,12 @@ def test_nixl_prom_metrics_group_handshake_with_transfer_failures(): FakeNixlWrapper, ) def test_notification_after_expiry_is_counted(default_vllm_config, dist_init): - vllm_config = create_vllm_config() + block_size = maybe_update_block_size(16) + vllm_config = create_vllm_config(block_size=block_size) connector = NixlConnector( - vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16) + vllm_config, + KVConnectorRole.WORKER, + make_kv_cache_config(block_size=block_size), ) connector.connector_worker = FakeNixlConnectorWorker( vllm_config, connector.engine_id, hand_shake_latency=0 @@ -2323,7 +2366,10 @@ def test_register_kv_caches( 2. nixl_wrapper.get_xfer_descs() is called with blocks_data containing block layout info """ - vllm_config = create_vllm_config(attention_backend=attn_backend) + block_size = maybe_update_block_size(16) + vllm_config = create_vllm_config( + attention_backend=attn_backend, block_size=block_size + ) vllm_config.cache_config.kv_cache_layout = layout # Import the appropriate backend based on the parameter @@ -2348,7 +2394,6 @@ def test_register_kv_caches( patch(f"{nixl_worker}.get_current_attn_backends") as mock_get_attn_backends, ): mock_get_attn_backends.return_value = [backend_cls] - block_size = 16 num_blocks = 8 num_heads = 4 head_size = 16 @@ -2472,7 +2517,7 @@ def test_register_kv_caches( } assert len(blocks_data) == num_blocks * len(base_addrs) - assert connector.connector_worker.block_size == 16 + assert connector.connector_worker.block_size == block_size def test_register_packed_dsv4_mla_cache_as_single_region( @@ -2573,7 +2618,8 @@ def test_kv_buffer_to_nixl_memory_types( """Test that register_kv_caches() passes the correct memory types from the config to the nixl_wrapper. """ - vllm_config = create_vllm_config() + block_size = maybe_update_block_size(16) + vllm_config = create_vllm_config(block_size=block_size) # Override the default memory types in the config vllm_config.kv_transfer_config.kv_buffer_device = kv_buffer_device from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import ( @@ -2603,7 +2649,9 @@ def test_kv_buffer_to_nixl_memory_types( ): # noqa: E501 # Create connector and replace its worker with a fake one for isolation connector = NixlConnector( - vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16) + vllm_config, + KVConnectorRole.WORKER, + make_kv_cache_config(block_size=block_size), ) # Verify get_reg_descs was called with the correct memory_type @@ -2617,17 +2665,18 @@ def test_kv_buffer_to_nixl_memory_types( ) def test_shutdown_cleans_up_resources(default_vllm_config, dist_init): """Test that shutdown() properly cleans up all resources.""" - vllm_config = create_vllm_config() + block_size = maybe_update_block_size(16) + vllm_config = create_vllm_config(block_size=block_size) scheduler = NixlConnectorScheduler( vllm_config, vllm_config.kv_transfer_config.engine_id, - make_kv_cache_config(block_size=16), + make_kv_cache_config(block_size=block_size), ) worker = NixlConnectorWorker( vllm_config, vllm_config.kv_transfer_config.engine_id, - make_kv_cache_config(block_size=16), + make_kv_cache_config(block_size=block_size), ) nixl_wrapper = worker.nixl_wrapper @@ -2643,7 +2692,7 @@ def test_shutdown_cleans_up_resources(default_vllm_config, dist_init): # Mock register_kv_cache which registers local handle worker.src_xfer_handles_by_block_size = {worker.block_size: 455} # P TP = 2 * D TP case, we should register 2 local handles - worker.src_xfer_handles_by_tp_ratio = {(-2, 16): [456, 457]} + worker.src_xfer_handles_by_tp_ratio = {(-2, block_size): [456, 457]} worker.dst_xfer_side_handles = {"engine1": {0: 789}} worker._remote_agents = {"engine1": {(0, 0): "agent1"}} # _cleanup_remote_engine (called by shutdown) also clears these: @@ -2688,13 +2737,15 @@ def _setup_worker_with_remote_engine( engine_ttl: float = 10.0, ) -> tuple[Any, str]: """Create a worker with one remote engine registered.""" + block_size = maybe_update_block_size(16) vllm_config = create_vllm_config( + block_size=block_size, kv_connector_extra_config={"engine_ttl": engine_ttl}, ) worker = NixlConnectorWorker( vllm_config, vllm_config.kv_transfer_config.engine_id, - make_kv_cache_config(block_size=16), + make_kv_cache_config(block_size=block_size), ) engine_id = "remote-engine-1" @@ -2857,7 +2908,11 @@ class TestPeerReplacement: from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker as bw from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend - config = create_vllm_config(kv_connector_extra_config={"engine_ttl": 0}) + block_size = maybe_update_block_size(16) + + config = create_vllm_config( + kv_connector_extra_config={"engine_ttl": 0}, block_size=block_size + ) config.kv_transfer_config.kv_buffer_device = "cpu" platform = SimpleNamespace( device_type="cpu", @@ -3258,12 +3313,15 @@ def test_aborted_request_removed_from_worker_in_batch(default_vllm_config, dist_ the real scheduler, then simulate an abort (request not in next scheduler iteration) and verify the worker no longer tracks it as in-batch. """ - vllm_config = create_vllm_config() + block_size = maybe_update_block_size(16) + vllm_config = create_vllm_config(block_size=block_size) scheduler = create_scheduler(vllm_config) # KVConnector Worker in P connector = NixlConnector( - vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16) + vllm_config, + KVConnectorRole.WORKER, + make_kv_cache_config(block_size=block_size), ) connector.connector_worker = FakeNixlConnectorWorker( vllm_config, connector.engine_id, hand_shake_latency=0 @@ -3398,9 +3456,12 @@ def test_empty_recv_is_reported_only_when_awaited( A failed notification changes neither: the KV is local either way, and the producer frees its own blocks on a timeout. """ - vllm_config = create_vllm_config() + block_size = maybe_update_block_size(16) + vllm_config = create_vllm_config(block_size=block_size) connector = NixlConnector( - vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16) + vllm_config, + KVConnectorRole.WORKER, + make_kv_cache_config(block_size=block_size), ) connector.connector_worker = FakeNixlConnectorWorker( vllm_config, @@ -3518,12 +3579,13 @@ def test_transfer_failure_logging( """ import logging - vllm_config = create_vllm_config() + block_size = maybe_update_block_size(16) + vllm_config = create_vllm_config(block_size=block_size) connector = NixlConnector( vllm_config, KVConnectorRole.WORKER, - make_kv_cache_config(block_size=16, swa_enabled=enable_hma), + make_kv_cache_config(block_size=block_size, swa_enabled=enable_hma), ) connector.connector_worker = FakeNixlConnectorWorker( vllm_config, @@ -3648,10 +3710,13 @@ def test_transfer_failure_logging( ) def test_handshake_failure_returns_finished(default_vllm_config, dist_init): """Test that handshake failures mark blocks invalid and return via get_finished.""" - vllm_config = create_vllm_config() + block_size = maybe_update_block_size(16) + vllm_config = create_vllm_config(block_size=block_size) connector = NixlConnector( - vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16) + vllm_config, + KVConnectorRole.WORKER, + make_kv_cache_config(block_size=block_size), ) connector.connector_worker = FakeNixlConnectorWorker( vllm_config, connector.engine_id, hand_shake_latency=0.1 @@ -3704,10 +3769,13 @@ def test_transfer_setup_failure_returns_finished( default_vllm_config, dist_init, is_hma ): """Setup failures report the request; only non-HMA reports block IDs.""" - vllm_config = create_vllm_config() + block_size = maybe_update_block_size(16) + vllm_config = create_vllm_config(block_size=block_size) connector = NixlConnector( - vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16) + vllm_config, + KVConnectorRole.WORKER, + make_kv_cache_config(block_size=block_size), ) connector.connector_worker = FakeNixlConnectorWorker( vllm_config, connector.engine_id, hand_shake_latency=0 @@ -3776,7 +3844,9 @@ class _ScriptedXferWrapper(FakeNixlWrapper): def _make_split_read_connector(vllm_config, request_id, states): """Seed a request with scripted in-flight xfer handles + recv metadata.""" connector = NixlConnector( - vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16) + vllm_config, + KVConnectorRole.WORKER, + make_kv_cache_config(block_size=vllm_config.cache_config.block_size), ) connector.connector_worker = FakeNixlConnectorWorker( vllm_config, connector.engine_id, hand_shake_latency=0 @@ -3818,7 +3888,7 @@ def test_split_read_failure_defers_report_until_last_handle( request_id = "split_read_partial_failure" err_handle, live_handle = 11, 22 connector, worker, wrapper = _make_split_read_connector( - create_vllm_config(), + create_vllm_config(block_size=maybe_update_block_size(16)), request_id, {err_handle: ["ERR"], live_handle: ["PROC", "PROC", "DONE"]}, ) @@ -3894,10 +3964,15 @@ def test_failed_request_skips_kv_postprocessing( # Use enable_permute_local_kv=True so that # post_process_device_kv_on_receive would be called on the success path, # making the assertion meaningful (not trivially true). - vllm_config = create_vllm_config(enable_permute_local_kv=True) + block_size = maybe_update_block_size(16) + vllm_config = create_vllm_config( + enable_permute_local_kv=True, block_size=block_size + ) connector = NixlConnector( - vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16) + vllm_config, + KVConnectorRole.WORKER, + make_kv_cache_config(block_size=block_size), ) connector.connector_worker = FakeNixlConnectorWorker( vllm_config, @@ -4220,14 +4295,15 @@ def test_compatibility_hash_validation( enforce_handshake_compat: whether to enforce compatibility checking """ + block_size = maybe_update_block_size(16) local_vllm_config = create_vllm_config( model="facebook/opt-125m", - block_size=16, + block_size=block_size, kv_connector_extra_config={ "enforce_handshake_compat": enforce_handshake_compat }, ) - kv_cache_config = make_kv_cache_config(block_size=16, num_blocks=2) + kv_cache_config = make_kv_cache_config(block_size=block_size, num_blocks=2) decode_connector = NixlConnector( local_vllm_config, KVConnectorRole.WORKER, kv_cache_config ) @@ -4253,7 +4329,7 @@ def test_compatibility_hash_validation( remote_config_params: dict[str, Any] = { "model": "facebook/opt-125m", - "block_size": 16, + "block_size": block_size, **config_overrides, } remote_vllm_config = create_vllm_config(**remote_config_params) @@ -4276,7 +4352,7 @@ def test_compatibility_hash_validation( decode_worker.backend_name, ) - prefill_block_size = config_overrides.get("block_size", 16) + prefill_block_size = remote_vllm_config.cache_config.block_size prefill_block_lens = [4096 * prefill_block_size] prefill_metadata = NixlAgentMetadata( engine_id=FakeNixlConnectorWorker.REMOTE_ENGINE_ID, @@ -4352,12 +4428,15 @@ def test_handshake_decode_errors(default_vllm_config, dist_init, error_scenario) - NixlHandshakePayload decoder - NixlAgentMetadata decoder """ + block_size = maybe_update_block_size(16) local_vllm_config = create_vllm_config( model="facebook/opt-125m", - block_size=16, + block_size=block_size, ) decode_connector = NixlConnector( - local_vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16) + local_vllm_config, + KVConnectorRole.WORKER, + make_kv_cache_config(block_size=block_size), ) decode_worker = decode_connector.connector_worker @@ -4441,11 +4520,14 @@ def test_handshake_decode_errors(default_vllm_config, dist_init, error_scenario) decode_tp_size = 1 prefill_tp_size = 4 - vllm_config = create_vllm_config() + block_size = maybe_update_block_size(16) + vllm_config = create_vllm_config(block_size=block_size) vllm_config.parallel_config.tensor_parallel_size = decode_tp_size connector = NixlConnector( - vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16) + vllm_config, + KVConnectorRole.WORKER, + make_kv_cache_config(block_size=block_size), ) connector.connector_worker = FakeNixlConnectorWorker( vllm_config, connector.engine_id, hand_shake_latency=0 @@ -4550,7 +4632,8 @@ def test_kv_both_deprecation_warning(default_vllm_config, dist_init): _print_warning_once.cache_clear() - vllm_config = create_vllm_config(kv_role="kv_both") + block_size = maybe_update_block_size(16) + vllm_config = create_vllm_config(kv_role="kv_both", block_size=block_size) with patch( "vllm.distributed.kv_transfer.kv_connector.v1.nixl.connector.logger" @@ -4559,7 +4642,7 @@ def test_kv_both_deprecation_warning(default_vllm_config, dist_init): NixlConnector( vllm_config, KVConnectorRole.WORKER, - make_kv_cache_config(block_size=16), + make_kv_cache_config(block_size=block_size), ) mock_logger.warning_once.assert_called_once() @@ -4575,14 +4658,15 @@ def test_kv_both_deprecation_warning(default_vllm_config, dist_init): def test_explicit_kv_role_no_deprecation_warning(default_vllm_config, dist_init): """kv_role='kv_consumer' or 'kv_producer' should NOT emit a warning.""" for role in ("kv_consumer", "kv_producer"): - vllm_config = create_vllm_config(kv_role=role) + block_size = maybe_update_block_size(16) + vllm_config = create_vllm_config(kv_role=role, block_size=block_size) with patch( "vllm.distributed.kv_transfer.kv_connector.v1.nixl.connector.logger" ) as mock_logger: NixlConnector( vllm_config, KVConnectorRole.WORKER, - make_kv_cache_config(block_size=16), + make_kv_cache_config(block_size=block_size), ) ( diff --git a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py index c665ce0c5b..469e0f5d82 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py @@ -39,6 +39,7 @@ from .utils import ( create_vllm_config, make_kv_cache_config, make_nixl_scheduler, + maybe_update_block_size, ) @@ -1027,7 +1028,7 @@ def test_fewer_blocks_with_hma(monkeypatch, model_name, sw_size): kv_connector="NixlConnector", kv_role="kv_consumer", ) - block_size = 16 + block_size = maybe_update_block_size(16) llm_kwargs = { "model": model_name, "enforce_eager": True, diff --git a/tests/v1/kv_connector/unit/utils.py b/tests/v1/kv_connector/unit/utils.py index 28bfda419a..59b2e7ad03 100644 --- a/tests/v1/kv_connector/unit/utils.py +++ b/tests/v1/kv_connector/unit/utils.py @@ -31,6 +31,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import ( from vllm.distributed.kv_transfer.kv_connector.v1.example_connector import ( # noqa ExampleConnector, ) +from vllm.platforms import current_platform from vllm.utils.hashing import sha256 from vllm.v1.core.kv_cache_manager import KVCacheBlocks from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash @@ -59,6 +60,10 @@ def create_mock_connector() -> Mock: return connector +def maybe_update_block_size(block_size: int) -> int: + return 64 if current_platform.is_xpu() else block_size + + def assert_scheduler_empty(scheduler: Scheduler): """Confirm the scheduler is "empty" - i.e. no leaks.""" # Scheduler Metadata. diff --git a/vllm/v1/attention/backends/flash_attn.py b/vllm/v1/attention/backends/flash_attn.py index 6179b27bb6..4a336586ac 100755 --- a/vllm/v1/attention/backends/flash_attn.py +++ b/vllm/v1/attention/backends/flash_attn.py @@ -355,6 +355,8 @@ class FlashAttentionBackend(AttentionBackend): def get_supported_kernel_block_sizes( cls, kv_cache_spec: KVCacheSpec | None = None ) -> list[int | MultipleOf]: + if current_platform.is_xpu(): + return [MultipleOf(64)] if block_size := cls._get_sm90_fa4_fp8_kv_block_size(kv_cache_spec): return [block_size] if block_size := cls._get_fa4_hd256_block_size(kv_cache_spec): diff --git a/vllm/v1/worker/gpu/attn_utils.py b/vllm/v1/worker/gpu/attn_utils.py index d1feaef031..8964c660a1 100644 --- a/vllm/v1/worker/gpu/attn_utils.py +++ b/vllm/v1/worker/gpu/attn_utils.py @@ -14,9 +14,11 @@ from vllm.config import ( get_layers_from_vllm_config, ) from vllm.config.compilation import CUDAGraphMode +from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase from vllm.multimodal.inputs import MultiModalFeatureSpec +from vllm.platforms import current_platform from vllm.v1.attention.backend import ( AttentionCGSupport, CommonAttentionMetadata, @@ -45,6 +47,8 @@ from vllm.v1.worker.utils import ( prepare_kernel_block_sizes, ) +logger = init_logger(__name__) + if TYPE_CHECKING: from vllm.v1.worker.gpu.block_table import BlockTables from vllm.v1.worker.gpu.cudagraph_utils import ( @@ -263,10 +267,24 @@ def init_attn_backend( # by all backends within that group. kernel_block_sizes = prepare_kernel_block_sizes(kv_cache_config, attn_groups) + if current_platform.is_xpu(): + for kv_cache_group_id, selected_kernel_block_size in enumerate( + kernel_block_sizes + ): + if selected_kernel_block_size % 64 != 0: + for attn_group in attn_groups[kv_cache_group_id]: + logger.warning_once( + "On XPU platform, current selected kernel block size is " + "not fully optimized: kernel_block_size=%s, backend=%s, " + "kv_cache_spec=%s", + selected_kernel_block_size, + attn_group.backend, + repr(attn_group.kv_cache_spec), + ) # Phase 3: create metadata builders and determine cudagraph support. attn_backend_workspace: torch.Tensor | None = None for kv_cache_group_id, groups in enumerate(attn_groups): - kernel_block_size = None + kernel_block_size: int | None = None if kv_cache_group_id < len(kernel_block_sizes): kernel_block_size = kernel_block_sizes[kv_cache_group_id] for group in groups: