[XPU] limit FA kernel block size to multipleOf 64 (#57297)
Signed-off-by: Yan Ma <yan.ma@intel.com>
This commit is contained in:
@@ -105,7 +105,7 @@ steps:
|
||||
pytest -v -s v1/test_serial_utils.py &&
|
||||
pytest -v -s v1/e2e/general/test_correctness_sliding_window.py &&
|
||||
pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py --ignore=v1/spec_decode/test_speculators_correctness.py --deselect=tests/v1/spec_decode/test_mtp.py::test_glm_mtp_defers_lm_head &&
|
||||
pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py --ignore=v1/kv_connector/unit/test_nixl_connector_hma.py --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --deselect=tests/v1/kv_connector/unit/test_nixl_connector.py::test_abort_timeout_on_prefiller[ray]'
|
||||
pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --deselect=tests/v1/kv_connector/unit/test_nixl_connector.py::test_abort_timeout_on_prefiller[ray]'
|
||||
parallelism: 2
|
||||
- label: "XPU Qwen3-30B + V1 KV Connector"
|
||||
key: xpu-qwen3-30b-v1-kv-connector
|
||||
|
||||
@@ -51,6 +51,7 @@ from .utils import (
|
||||
create_scheduler,
|
||||
create_vllm_config,
|
||||
make_kv_cache_config,
|
||||
maybe_update_block_size,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
@@ -87,10 +88,11 @@ def _make_connector_with_fake_worker(
|
||||
hand_shake_latency=0, cycles_before_done=0, do_handshake=True
|
||||
):
|
||||
"""Create a NixlConnector with FakeNixlConnectorWorker."""
|
||||
block_size = maybe_update_block_size(32)
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, block_size=32
|
||||
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, block_size=block_size
|
||||
)
|
||||
kv_cache_config = make_kv_cache_config(block_size=32, num_blocks=2)
|
||||
kv_cache_config = make_kv_cache_config(block_size=block_size, num_blocks=2)
|
||||
connector = NixlConnector(vllm_config, KVConnectorRole.WORKER, kv_cache_config)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config,
|
||||
|
||||
@@ -57,7 +57,12 @@ from vllm.v1.kv_cache_interface import (
|
||||
)
|
||||
from vllm.v1.request import RequestStatus
|
||||
|
||||
from .utils import create_request, create_scheduler, create_vllm_config
|
||||
from .utils import (
|
||||
create_request,
|
||||
create_scheduler,
|
||||
create_vllm_config,
|
||||
maybe_update_block_size,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -148,6 +153,7 @@ def test_region_length_validation_checks_replicated_gqa_heads(
|
||||
|
||||
|
||||
def _make_test_kv_cache_config() -> KVCacheConfig:
|
||||
block_size = maybe_update_block_size(16)
|
||||
return KVCacheConfig(
|
||||
num_blocks=0,
|
||||
kv_cache_tensors=[],
|
||||
@@ -160,7 +166,7 @@ def _make_test_kv_cache_config() -> KVCacheConfig:
|
||||
"model.layers.1.eagle_attn",
|
||||
],
|
||||
FullAttentionSpec(
|
||||
block_size=16,
|
||||
block_size=block_size,
|
||||
num_kv_heads=4,
|
||||
head_size=64,
|
||||
dtype=torch.float16,
|
||||
@@ -170,6 +176,14 @@ def _make_test_kv_cache_config() -> KVCacheConfig:
|
||||
)
|
||||
|
||||
|
||||
def _make_worker_vllm_config(kv_role: str):
|
||||
return create_vllm_config(
|
||||
kv_connector="MooncakeConnector",
|
||||
kv_role=kv_role,
|
||||
block_size=maybe_update_block_size(16),
|
||||
)
|
||||
|
||||
|
||||
def _make_packed_mla_kv_cache_config(
|
||||
num_blocks: int, *, num_groups: int = 1
|
||||
) -> KVCacheConfig:
|
||||
@@ -429,9 +443,7 @@ async def test_send_kv_to_decode_aligns_consumer_regions_by_layer_metadata(
|
||||
):
|
||||
"""Producer sends its PP layer shard to the matching consumer layer address."""
|
||||
monkeypatch.setenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "5")
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeConnector", kv_role="kv_producer"
|
||||
)
|
||||
vllm_config = _make_worker_vllm_config("kv_producer")
|
||||
|
||||
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
|
||||
prefill_connector = MooncakeConnector(
|
||||
@@ -1205,7 +1217,7 @@ def _xfer_meta(
|
||||
@contextlib.contextmanager
|
||||
def mooncake_register_worker(kv_cache_config, *, kv_role: str = "kv_consumer"):
|
||||
"""Build a worker whose engine registration and sender thread are patched."""
|
||||
vllm_config = create_vllm_config(kv_connector="MooncakeConnector", kv_role=kv_role)
|
||||
vllm_config = _make_worker_vllm_config(kv_role)
|
||||
connector_mod = (
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector"
|
||||
)
|
||||
@@ -1241,9 +1253,7 @@ async def test_receive_kv_selects_remote_pp_workers(
|
||||
expected_addrs: list[str],
|
||||
):
|
||||
"""Decode workers should not hard-code producer pp_rank 0."""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeConnector", kv_role="kv_consumer"
|
||||
)
|
||||
vllm_config = _make_worker_vllm_config("kv_consumer")
|
||||
|
||||
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
|
||||
decode_connector = MooncakeConnector(
|
||||
@@ -1314,7 +1324,7 @@ def test_resolve_need_send_accounts_for_remote_tp_fanout():
|
||||
@pytest.mark.asyncio
|
||||
async def test_heterogeneous_pp_waits_for_consumer_without_shared_layers():
|
||||
"""P4/D2 must retain KV until both D stages finish, including a no-op pull."""
|
||||
config = create_vllm_config(kv_connector="MooncakeConnector", kv_role="kv_producer")
|
||||
config = _make_worker_vllm_config("kv_producer")
|
||||
with (
|
||||
set_current_vllm_config(config),
|
||||
patch_worker_dependencies(),
|
||||
@@ -1386,9 +1396,7 @@ async def test_kv_producer(monkeypatch):
|
||||
Verifies memory offset calculation: ptr = base_addr + block_id * block_len.
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "5")
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeConnector", kv_role="kv_producer"
|
||||
)
|
||||
vllm_config = _make_worker_vllm_config("kv_producer")
|
||||
|
||||
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
|
||||
prefill_connector = MooncakeConnector(
|
||||
@@ -1589,9 +1597,7 @@ async def test_kv_consumuer(monkeypatch):
|
||||
|
||||
Verifies that MooncakeXferMetadata is correctly serialized and sent via ZMQ.
|
||||
"""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeConnector", kv_role="kv_consumer"
|
||||
)
|
||||
vllm_config = _make_worker_vllm_config("kv_consumer")
|
||||
|
||||
with set_current_vllm_config(vllm_config), patch_worker_dependencies() as mocks:
|
||||
decode_connector = MooncakeConnector(
|
||||
@@ -1661,9 +1667,7 @@ async def test_kv_consumuer(monkeypatch):
|
||||
@pytest.mark.asyncio
|
||||
async def test_worker_get_finished_timeout(monkeypatch):
|
||||
"""Tests the cleanup mechanism for requests."""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeConnector", kv_role="kv_producer"
|
||||
)
|
||||
vllm_config = _make_worker_vllm_config("kv_producer")
|
||||
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
|
||||
prefill_connector = MooncakeConnector(
|
||||
vllm_config,
|
||||
@@ -1713,7 +1717,7 @@ async def test_worker_get_finished_timeout(monkeypatch):
|
||||
def test_register_kv_caches(layout: KVCacheLayout, separate_kv_head_groups: bool):
|
||||
"""Tests the memory registration logic with the underlying Mooncake engine."""
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16,
|
||||
block_size=maybe_update_block_size(16),
|
||||
num_kv_heads=4,
|
||||
head_size=64,
|
||||
dtype=torch.float16,
|
||||
@@ -1794,7 +1798,10 @@ def test_register_noncontiguous_packed_allows_matching_pp():
|
||||
known. Rejecting every local PP>1 layout at startup is too strict.
|
||||
"""
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=4, head_size=64, dtype=torch.float16
|
||||
block_size=maybe_update_block_size(16),
|
||||
num_kv_heads=4,
|
||||
head_size=64,
|
||||
dtype=torch.float16,
|
||||
)
|
||||
layer_names = ["model.layers.0.self_attn", "model.layers.1.self_attn"]
|
||||
with mooncake_register_worker(_make_test_kv_cache_config()) as (worker, _reg):
|
||||
@@ -1814,8 +1821,9 @@ def test_register_kv_caches_supports_mixed_mla_and_eagle_shapes():
|
||||
with mooncake_register_worker(_make_test_kv_cache_config()) as (worker, reg):
|
||||
worker.use_mla = True
|
||||
worker.transfer_topo.is_mla = True
|
||||
mla_cache = torch.zeros((2, 16, 512), dtype=torch.float16)
|
||||
eagle_cache = torch.zeros((2, 16, 8, 64), dtype=torch.float16)
|
||||
block_size = maybe_update_block_size(16)
|
||||
mla_cache = torch.zeros((2, block_size, 512), dtype=torch.float16)
|
||||
eagle_cache = torch.zeros((2, block_size, 8, 64), dtype=torch.float16)
|
||||
kv_caches = {
|
||||
"model.layers.0.mla_attn": mla_cache,
|
||||
"model.layers.1.eagle_attn": eagle_cache,
|
||||
@@ -2240,7 +2248,7 @@ def _register_sliced_packed_mla(
|
||||
):
|
||||
"""Register column slices of one MLA packed row. Returns worker fields."""
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=16,
|
||||
block_size=maybe_update_block_size(16),
|
||||
num_kv_heads=1,
|
||||
head_size=64,
|
||||
dtype=torch.uint8,
|
||||
@@ -2286,7 +2294,7 @@ def test_sliced_packed_mla_registers_each_layer_view():
|
||||
KVCacheGroupSpec(
|
||||
[_layer_name(i) for i in layer_idxs],
|
||||
MLAAttentionSpec(
|
||||
block_size=16,
|
||||
block_size=maybe_update_block_size(16),
|
||||
num_kv_heads=1,
|
||||
head_size=64,
|
||||
dtype=torch.uint8,
|
||||
@@ -2436,9 +2444,7 @@ async def test_kv_producer_heterogeneous_tp(monkeypatch, d_tp_size):
|
||||
remote_block_len = LOCAL_BLOCK_LEN * P_TP_SIZE // d_tp_size
|
||||
|
||||
monkeypatch.setenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "5")
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeConnector", kv_role="kv_producer"
|
||||
)
|
||||
vllm_config = _make_worker_vllm_config("kv_producer")
|
||||
|
||||
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
|
||||
prefill_connector = MooncakeConnector(
|
||||
|
||||
@@ -30,7 +30,12 @@ from vllm.v1.kv_cache_interface import (
|
||||
)
|
||||
|
||||
from .test_mooncake_connector import FakeMooncakeWrapper, patch_worker_dependencies
|
||||
from .utils import create_request, create_vllm_config, make_kv_cache_config
|
||||
from .utils import (
|
||||
create_request,
|
||||
create_vllm_config,
|
||||
make_kv_cache_config,
|
||||
maybe_update_block_size,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -224,12 +229,11 @@ def test_metadata_hma_block_ids():
|
||||
async def test_build_transfer_params_multi_group_trimming(monkeypatch):
|
||||
"""_build_transfer_params trims per-group blocks when local > remote."""
|
||||
monkeypatch.setenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "5")
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeConnector", kv_role="kv_producer"
|
||||
)
|
||||
kv_cache_config = make_kv_cache_config(
|
||||
block_size=vllm_config.cache_config.block_size, swa_enabled=True
|
||||
kv_connector="MooncakeConnector", kv_role="kv_producer", block_size=block_size
|
||||
)
|
||||
kv_cache_config = make_kv_cache_config(block_size=block_size, swa_enabled=True)
|
||||
|
||||
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
|
||||
connector = MooncakeConnector(
|
||||
@@ -320,12 +324,11 @@ async def test_build_transfer_params_multi_group_trimming(monkeypatch):
|
||||
async def test_build_transfer_params_group_count_mismatch(monkeypatch):
|
||||
"""_build_transfer_params reports an error when group counts differ."""
|
||||
monkeypatch.setenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "5")
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeConnector", kv_role="kv_producer"
|
||||
)
|
||||
kv_cache_config = make_kv_cache_config(
|
||||
block_size=vllm_config.cache_config.block_size, swa_enabled=True
|
||||
kv_connector="MooncakeConnector", kv_role="kv_producer", block_size=block_size
|
||||
)
|
||||
kv_cache_config = make_kv_cache_config(block_size=block_size, swa_enabled=True)
|
||||
|
||||
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
|
||||
connector = MooncakeConnector(
|
||||
@@ -449,7 +452,7 @@ def test_request_finished_with_hma_groups():
|
||||
def _make_kv_consumer_worker(
|
||||
swa_enabled: bool = False, kv_cache_config: KVCacheConfig | None = None
|
||||
):
|
||||
block_size = 16
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeConnector",
|
||||
kv_role="kv_consumer",
|
||||
@@ -495,6 +498,7 @@ def test_worker_is_hma_required(swa_enabled, expected_is_hma):
|
||||
def test_worker_is_hma_required_multiple_full_attention_groups():
|
||||
"""Two full-attention groups with different block sizes also select
|
||||
request-level failure reporting, matching the scheduler."""
|
||||
block_size = maybe_update_block_size(16)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=100,
|
||||
kv_cache_tensors=[],
|
||||
@@ -502,13 +506,19 @@ def test_worker_is_hma_required_multiple_full_attention_groups():
|
||||
KVCacheGroupSpec(
|
||||
["layer0"],
|
||||
FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=4, head_size=16, dtype=torch.float16
|
||||
block_size=block_size,
|
||||
num_kv_heads=4,
|
||||
head_size=16,
|
||||
dtype=torch.float16,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["layer1"],
|
||||
FullAttentionSpec(
|
||||
block_size=32, num_kv_heads=4, head_size=16, dtype=torch.float16
|
||||
block_size=2 * block_size,
|
||||
num_kv_heads=4,
|
||||
head_size=16,
|
||||
dtype=torch.float16,
|
||||
),
|
||||
),
|
||||
],
|
||||
|
||||
@@ -36,7 +36,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
)
|
||||
|
||||
from .test_mooncake_connector import patch_worker_dependencies
|
||||
from .utils import create_request, create_vllm_config
|
||||
from .utils import create_request, create_vllm_config, maybe_update_block_size
|
||||
|
||||
|
||||
def noop_shutdown():
|
||||
@@ -147,6 +147,7 @@ def test_register_kv_caches_emits_fa_and_gdn_regions(monkeypatch):
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeConnector",
|
||||
kv_role="kv_consumer",
|
||||
block_size=maybe_update_block_size(16),
|
||||
)
|
||||
kv_cache_config = make_hybrid_gdn_kv_cache_config(
|
||||
vllm_config.cache_config.block_size
|
||||
@@ -203,6 +204,7 @@ def test_register_kv_caches_scales_attention_len_to_kernel_block(monkeypatch):
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeConnector",
|
||||
kv_role="kv_consumer",
|
||||
block_size=maybe_update_block_size(16),
|
||||
)
|
||||
kv_cache_config = make_hybrid_gdn_kv_cache_config(
|
||||
vllm_config.cache_config.block_size
|
||||
@@ -392,6 +394,7 @@ def test_register_kv_caches_deduplicates_shared_backing_memory(monkeypatch):
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeConnector",
|
||||
kv_role="kv_consumer",
|
||||
block_size=maybe_update_block_size(16),
|
||||
)
|
||||
kv_cache_config = make_hybrid_gdn_kv_cache_config(
|
||||
vllm_config.cache_config.block_size
|
||||
@@ -405,8 +408,13 @@ def test_register_kv_caches_deduplicates_shared_backing_memory(monkeypatch):
|
||||
)
|
||||
worker = connector.connector_worker
|
||||
|
||||
backing = torch.empty((4, 64), dtype=torch.float16)
|
||||
fa_cache = backing[:2, :16]
|
||||
fa_spec = kv_cache_config.kv_cache_groups[0].kv_cache_spec
|
||||
fa_cache_width = (
|
||||
fa_spec.page_size_bytes
|
||||
// torch.tensor([], dtype=torch.float16).element_size()
|
||||
)
|
||||
backing = torch.empty((4, fa_cache_width), dtype=torch.float16)
|
||||
fa_cache = backing[:2]
|
||||
gdn_cache = backing[:3]
|
||||
|
||||
with patch.object(
|
||||
@@ -438,6 +446,7 @@ def test_hybrid_gdn_transfer_params_preserve_group_identity(monkeypatch):
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeConnector",
|
||||
kv_role="kv_producer",
|
||||
block_size=maybe_update_block_size(16),
|
||||
)
|
||||
kv_cache_config = make_hybrid_gdn_kv_cache_config(
|
||||
vllm_config.cache_config.block_size
|
||||
@@ -567,6 +576,7 @@ def test_hybrid_gdn_keeps_packed_fa_and_gdn_regions_whole(
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeConnector",
|
||||
kv_role="kv_producer",
|
||||
block_size=maybe_update_block_size(16),
|
||||
)
|
||||
kv_cache_config = make_hybrid_gdn_kv_cache_config(
|
||||
vllm_config.cache_config.block_size
|
||||
|
||||
@@ -10,7 +10,7 @@ from unittest.mock import MagicMock
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.v1.kv_connector.unit.utils import create_vllm_config
|
||||
from tests.v1.kv_connector.unit.utils import create_vllm_config, maybe_update_block_size
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.config import KVTransferConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory
|
||||
@@ -314,10 +314,13 @@ def test_multi_example_connector_consistency():
|
||||
},
|
||||
)
|
||||
|
||||
block_size = maybe_update_block_size(16)
|
||||
expected_num_blocks = 2 if block_size == 64 else 7
|
||||
expected_num_matched_tokens = (expected_num_blocks - 1) * block_size
|
||||
llm = LLM(
|
||||
model=MODEL_NAME,
|
||||
enforce_eager=True,
|
||||
block_size=16,
|
||||
block_size=block_size,
|
||||
gpu_memory_utilization=0.5,
|
||||
kv_transfer_config=kv_transfer_config,
|
||||
async_scheduling=False,
|
||||
@@ -372,7 +375,7 @@ def test_multi_example_connector_consistency():
|
||||
"set_xfer_handshake_metadata_pp_aware",
|
||||
"on_new_request",
|
||||
"get_num_new_matched_tokens 0",
|
||||
"update_state_after_alloc num_blocks=[7] 0",
|
||||
f"update_state_after_alloc num_blocks=[{expected_num_blocks}] 0",
|
||||
"build_connector_meta",
|
||||
]
|
||||
# First three events are from initialization. Layer hooks run before the
|
||||
@@ -399,7 +402,7 @@ def test_multi_example_connector_consistency():
|
||||
"set_xfer_handshake_metadata_pp_aware",
|
||||
"on_new_request",
|
||||
"get_num_new_matched_tokens 0",
|
||||
"update_state_after_alloc num_blocks=[7] 0",
|
||||
f"update_state_after_alloc num_blocks=[{expected_num_blocks}] 0",
|
||||
"build_connector_meta",
|
||||
]
|
||||
# Reset prefix cache or else we'll just get the tokens back from there.
|
||||
@@ -423,13 +426,14 @@ def test_multi_example_connector_consistency():
|
||||
assert storage1_scheduler_events[:4] == [
|
||||
"on_new_request",
|
||||
"get_num_new_matched_tokens 0",
|
||||
"update_state_after_alloc num_blocks=[7] 96",
|
||||
"update_state_after_alloc "
|
||||
f"num_blocks=[{expected_num_blocks}] {expected_num_matched_tokens}",
|
||||
"build_connector_meta",
|
||||
]
|
||||
assert storage2_scheduler_events[:4] == [
|
||||
"on_new_request",
|
||||
"get_num_new_matched_tokens 0",
|
||||
"update_state_after_alloc num_blocks=[7] 0",
|
||||
f"update_state_after_alloc num_blocks=[{expected_num_blocks}] 0",
|
||||
"build_connector_meta",
|
||||
]
|
||||
|
||||
@@ -457,13 +461,14 @@ def test_multi_example_connector_consistency():
|
||||
assert storage1_scheduler_events[:4] == [
|
||||
"on_new_request",
|
||||
"get_num_new_matched_tokens 0",
|
||||
"update_state_after_alloc num_blocks=[7] 0",
|
||||
f"update_state_after_alloc num_blocks=[{expected_num_blocks}] 0",
|
||||
"build_connector_meta",
|
||||
]
|
||||
assert storage2_scheduler_events[:4] == [
|
||||
"on_new_request",
|
||||
"get_num_new_matched_tokens 0",
|
||||
"update_state_after_alloc num_blocks=[7] 96",
|
||||
"update_state_after_alloc "
|
||||
f"num_blocks=[{expected_num_blocks}] {expected_num_matched_tokens}",
|
||||
"build_connector_meta",
|
||||
]
|
||||
|
||||
|
||||
@@ -84,6 +84,7 @@ from .utils import (
|
||||
create_scheduler,
|
||||
create_vllm_config,
|
||||
make_kv_cache_config,
|
||||
maybe_update_block_size,
|
||||
)
|
||||
|
||||
|
||||
@@ -418,7 +419,8 @@ def test_kv_transfer_handshake(dist_init):
|
||||
# Test setup, we creates a scheduler that contains a NixlConnector
|
||||
# of role SCHEDULER, and expect it to be serving NixlAgentMetadata from
|
||||
# all workers of the instance.
|
||||
vllm_config = create_vllm_config()
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(block_size=block_size)
|
||||
vllm_config.cache_config.kv_cache_layout = "BLHNC"
|
||||
# in case the test runs on non-GPU machine
|
||||
vllm_config.kv_transfer_config.kv_buffer_device = "cpu"
|
||||
@@ -434,7 +436,7 @@ def test_kv_transfer_handshake(dist_init):
|
||||
KVCacheGroupSpec(
|
||||
["layer0", "layer1", "layer2"],
|
||||
FullAttentionSpec(
|
||||
block_size=16,
|
||||
block_size=block_size,
|
||||
num_kv_heads=4,
|
||||
head_size=16,
|
||||
dtype=torch.float16,
|
||||
@@ -544,7 +546,9 @@ class FakeNixlConnectorWorker(NixlConnectorWorker):
|
||||
**kwargs,
|
||||
):
|
||||
if kv_cache_config is None:
|
||||
kv_cache_config = make_kv_cache_config(block_size=16)
|
||||
kv_cache_config = make_kv_cache_config(
|
||||
block_size=args[0].cache_config.block_size
|
||||
)
|
||||
super().__init__(*args, kv_cache_config=kv_cache_config, **kwargs)
|
||||
self._hand_shake_latency = hand_shake_latency
|
||||
self.kv_cache_layout = kv_cache_layout
|
||||
@@ -674,7 +678,8 @@ class TestNixlHandshake:
|
||||
"""Replicated PCP is canonicalized; PCP-DCP publishes every shard."""
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
||||
|
||||
vllm_config = create_vllm_config(kv_role="kv_producer")
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(kv_role="kv_producer", block_size=block_size)
|
||||
vllm_config.parallel_config.prefill_context_parallel_size = pcp_size
|
||||
vllm_config.parallel_config.decode_context_parallel_size = dcp_size
|
||||
with (
|
||||
@@ -692,7 +697,7 @@ class TestNixlHandshake:
|
||||
connector = NixlConnector(
|
||||
vllm_config,
|
||||
KVConnectorRole.WORKER,
|
||||
make_kv_cache_config(block_size=16),
|
||||
make_kv_cache_config(block_size=block_size),
|
||||
)
|
||||
|
||||
worker = connector.connector_worker
|
||||
@@ -746,12 +751,13 @@ class TestNixlHandshake:
|
||||
This test triggers the connector to load remote KV for the same
|
||||
`request_id`.
|
||||
"""
|
||||
vllm_config = create_vllm_config()
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(block_size=block_size)
|
||||
|
||||
request_id = "req_id"
|
||||
|
||||
# Test worker role in decode server.
|
||||
kv_cache_config = make_kv_cache_config(block_size=16, num_blocks=10)
|
||||
kv_cache_config = make_kv_cache_config(block_size=block_size, num_blocks=10)
|
||||
connector = NixlConnector(vllm_config, KVConnectorRole.WORKER, kv_cache_config)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config,
|
||||
@@ -837,12 +843,15 @@ class TestNixlHandshake:
|
||||
prefill_tp_size,
|
||||
):
|
||||
"""Test that NixlConnector's start_load_kv should be non-blocking."""
|
||||
vllm_config = create_vllm_config()
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(block_size=block_size)
|
||||
vllm_config.parallel_config.tensor_parallel_size = decode_tp_size
|
||||
|
||||
# Test worker role in decode server.
|
||||
connector = NixlConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
vllm_config,
|
||||
KVConnectorRole.WORKER,
|
||||
make_kv_cache_config(block_size=block_size),
|
||||
)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config, connector.engine_id
|
||||
@@ -899,12 +908,13 @@ class TestNixlHandshake:
|
||||
lambda: local_tp_size,
|
||||
)
|
||||
|
||||
vllm_config = create_vllm_config()
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(block_size=block_size)
|
||||
|
||||
# num_blocks=1 keeps the fake handshake's region_num_blocks consistent
|
||||
# with the minimal single-block registration below (num_descs == 1);
|
||||
# _fa_desc_replicated cross-checks the two.
|
||||
kv_cache_config = make_kv_cache_config(block_size=16, num_blocks=1)
|
||||
kv_cache_config = make_kv_cache_config(block_size=block_size, num_blocks=1)
|
||||
connector = NixlConnector(vllm_config, KVConnectorRole.WORKER, kv_cache_config)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config,
|
||||
@@ -972,16 +982,21 @@ class TestNixlHandshake:
|
||||
"""Verify remote TP > local TP handshake succeeds with different
|
||||
remote configurations for an MLA model.
|
||||
"""
|
||||
vllm_config = create_vllm_config()
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(block_size=block_size)
|
||||
d_tp_size = 1
|
||||
p_tp_size = 2
|
||||
|
||||
# Build two separate connectors/workers to emulate P TP=2 ranks.
|
||||
conn_p0 = NixlConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
vllm_config,
|
||||
KVConnectorRole.WORKER,
|
||||
make_kv_cache_config(block_size=block_size),
|
||||
)
|
||||
conn_p1 = NixlConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
vllm_config,
|
||||
KVConnectorRole.WORKER,
|
||||
make_kv_cache_config(block_size=block_size),
|
||||
)
|
||||
conn_p0.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config, conn_p0.engine_id, hand_shake_latency=0
|
||||
@@ -1075,11 +1090,14 @@ class TestNixlHandshake:
|
||||
dist_init,
|
||||
):
|
||||
"""Test that multiple start_load_kv calls should occur concurrently."""
|
||||
vllm_config = create_vllm_config()
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(block_size=block_size)
|
||||
|
||||
# Test worker role in decode server.
|
||||
connector = NixlConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
vllm_config,
|
||||
KVConnectorRole.WORKER,
|
||||
make_kv_cache_config(block_size=block_size),
|
||||
)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config, connector.engine_id
|
||||
@@ -1143,8 +1161,10 @@ class TestNixlHandshake:
|
||||
"""Verify that adding a remote agent fails if kv_cache_layout differs.
|
||||
This test is only relevant for heterogeneous TP.
|
||||
"""
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(
|
||||
enable_permute_local_kv=enable_permute_local_kv
|
||||
enable_permute_local_kv=enable_permute_local_kv,
|
||||
block_size=block_size,
|
||||
)
|
||||
|
||||
# Mock TP world size to 2 to force heterogeneous TP when
|
||||
@@ -1155,7 +1175,9 @@ class TestNixlHandshake:
|
||||
):
|
||||
# Initialize connector and worker (with fake NIXL wrapper)
|
||||
connector = NixlConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
vllm_config,
|
||||
KVConnectorRole.WORKER,
|
||||
make_kv_cache_config(block_size=block_size),
|
||||
)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config,
|
||||
@@ -1205,7 +1227,10 @@ class TestNixlHandshake:
|
||||
"""Verify that adding a remote agent fails if kv_cache_layout differs.
|
||||
This test is only relevant for heterogeneous TP.
|
||||
"""
|
||||
vllm_config = create_vllm_config(enable_permute_local_kv=True)
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(
|
||||
enable_permute_local_kv=True, block_size=block_size
|
||||
)
|
||||
|
||||
# Mock TP world size to 2 to force heterogeneous TP when
|
||||
# remote_tp_size=1
|
||||
@@ -1215,7 +1240,9 @@ class TestNixlHandshake:
|
||||
):
|
||||
# Initialize connector and worker (with fake NIXL wrapper)
|
||||
connector = NixlConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
vllm_config,
|
||||
KVConnectorRole.WORKER,
|
||||
make_kv_cache_config(block_size=block_size),
|
||||
)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config,
|
||||
@@ -1261,7 +1288,9 @@ class TestNixlHandshake:
|
||||
self, default_vllm_config, dist_init
|
||||
):
|
||||
worker = FakeNixlConnectorWorker(
|
||||
create_vllm_config(), "engine", hand_shake_latency=0
|
||||
create_vllm_config(block_size=maybe_update_block_size(16)),
|
||||
"engine",
|
||||
hand_shake_latency=0,
|
||||
)
|
||||
|
||||
remote_block_len = 2048
|
||||
@@ -1316,13 +1345,16 @@ class TestNixlHandshake:
|
||||
heterogeneous TP must NOT raise (previously a NotImplementedError),
|
||||
and the per-region gate must still reject a wrong block_len.
|
||||
"""
|
||||
vllm_config = create_vllm_config()
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(block_size=block_size)
|
||||
with patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.get_tensor_model_parallel_world_size", # noqa: E501
|
||||
return_value=2,
|
||||
):
|
||||
connector = NixlConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
vllm_config,
|
||||
KVConnectorRole.WORKER,
|
||||
make_kv_cache_config(block_size=block_size),
|
||||
)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config, connector.engine_id, hand_shake_latency=0
|
||||
@@ -1409,7 +1441,8 @@ class TestNixlHandshake:
|
||||
expected local_block_len * tp_ratio and rejected the valid
|
||||
handshake.
|
||||
"""
|
||||
vllm_config = create_vllm_config()
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(block_size=block_size)
|
||||
|
||||
with patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.get_tensor_model_parallel_world_size", # noqa: E501
|
||||
@@ -1418,7 +1451,7 @@ class TestNixlHandshake:
|
||||
connector = NixlConnector(
|
||||
vllm_config,
|
||||
KVConnectorRole.WORKER,
|
||||
make_kv_cache_config(block_size=16),
|
||||
make_kv_cache_config(block_size=block_size),
|
||||
)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config, connector.engine_id, hand_shake_latency=0
|
||||
@@ -1465,7 +1498,8 @@ class TestNixlHandshake:
|
||||
block_lens when GQA replication is NOT in effect (32 KV heads,
|
||||
D_TP=4, P_TP=2: head_ratio=4, both sides have >1 head/rank).
|
||||
"""
|
||||
vllm_config = create_vllm_config()
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(block_size=block_size)
|
||||
|
||||
with patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.get_tensor_model_parallel_world_size", # noqa: E501
|
||||
@@ -1474,7 +1508,7 @@ class TestNixlHandshake:
|
||||
connector = NixlConnector(
|
||||
vllm_config,
|
||||
KVConnectorRole.WORKER,
|
||||
make_kv_cache_config(block_size=16),
|
||||
make_kv_cache_config(block_size=block_size),
|
||||
)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config, connector.engine_id, hand_shake_latency=0
|
||||
@@ -1522,11 +1556,14 @@ class TestNixlHandshake:
|
||||
)
|
||||
def test_kv_connector_stats(default_vllm_config, dist_init):
|
||||
"""Test that KV transfer stats are properly recorded and retrieved."""
|
||||
vllm_config = create_vllm_config()
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(block_size=block_size)
|
||||
|
||||
# Test worker role in decode server.
|
||||
connector = NixlConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
vllm_config,
|
||||
KVConnectorRole.WORKER,
|
||||
make_kv_cache_config(block_size=block_size),
|
||||
)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config, connector.engine_id, hand_shake_latency=0
|
||||
@@ -1602,9 +1639,12 @@ def test_reqs_to_send_deadline_rebased_to_worker_clock(default_vllm_config, dist
|
||||
remote read pulls another request's data (silent accuracy corruption).
|
||||
The worker must anchor the remaining TTL to its own clock.
|
||||
"""
|
||||
vllm_config = create_vllm_config()
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(block_size=block_size)
|
||||
connector = NixlConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
vllm_config,
|
||||
KVConnectorRole.WORKER,
|
||||
make_kv_cache_config(block_size=block_size),
|
||||
)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config, connector.engine_id, hand_shake_latency=0
|
||||
@@ -1793,9 +1833,12 @@ def test_nixl_prom_metrics_group_handshake_with_transfer_failures():
|
||||
FakeNixlWrapper,
|
||||
)
|
||||
def test_notification_after_expiry_is_counted(default_vllm_config, dist_init):
|
||||
vllm_config = create_vllm_config()
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(block_size=block_size)
|
||||
connector = NixlConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
vllm_config,
|
||||
KVConnectorRole.WORKER,
|
||||
make_kv_cache_config(block_size=block_size),
|
||||
)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config, connector.engine_id, hand_shake_latency=0
|
||||
@@ -2323,7 +2366,10 @@ def test_register_kv_caches(
|
||||
2. nixl_wrapper.get_xfer_descs() is called with blocks_data containing
|
||||
block layout info
|
||||
"""
|
||||
vllm_config = create_vllm_config(attention_backend=attn_backend)
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(
|
||||
attention_backend=attn_backend, block_size=block_size
|
||||
)
|
||||
vllm_config.cache_config.kv_cache_layout = layout
|
||||
|
||||
# Import the appropriate backend based on the parameter
|
||||
@@ -2348,7 +2394,6 @@ def test_register_kv_caches(
|
||||
patch(f"{nixl_worker}.get_current_attn_backends") as mock_get_attn_backends,
|
||||
):
|
||||
mock_get_attn_backends.return_value = [backend_cls]
|
||||
block_size = 16
|
||||
num_blocks = 8
|
||||
num_heads = 4
|
||||
head_size = 16
|
||||
@@ -2472,7 +2517,7 @@ def test_register_kv_caches(
|
||||
}
|
||||
assert len(blocks_data) == num_blocks * len(base_addrs)
|
||||
|
||||
assert connector.connector_worker.block_size == 16
|
||||
assert connector.connector_worker.block_size == block_size
|
||||
|
||||
|
||||
def test_register_packed_dsv4_mla_cache_as_single_region(
|
||||
@@ -2573,7 +2618,8 @@ def test_kv_buffer_to_nixl_memory_types(
|
||||
"""Test that register_kv_caches() passes the correct memory types from the
|
||||
config to the nixl_wrapper.
|
||||
"""
|
||||
vllm_config = create_vllm_config()
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(block_size=block_size)
|
||||
# Override the default memory types in the config
|
||||
vllm_config.kv_transfer_config.kv_buffer_device = kv_buffer_device
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import (
|
||||
@@ -2603,7 +2649,9 @@ def test_kv_buffer_to_nixl_memory_types(
|
||||
): # noqa: E501
|
||||
# Create connector and replace its worker with a fake one for isolation
|
||||
connector = NixlConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
vllm_config,
|
||||
KVConnectorRole.WORKER,
|
||||
make_kv_cache_config(block_size=block_size),
|
||||
)
|
||||
|
||||
# Verify get_reg_descs was called with the correct memory_type
|
||||
@@ -2617,17 +2665,18 @@ def test_kv_buffer_to_nixl_memory_types(
|
||||
)
|
||||
def test_shutdown_cleans_up_resources(default_vllm_config, dist_init):
|
||||
"""Test that shutdown() properly cleans up all resources."""
|
||||
vllm_config = create_vllm_config()
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(block_size=block_size)
|
||||
|
||||
scheduler = NixlConnectorScheduler(
|
||||
vllm_config,
|
||||
vllm_config.kv_transfer_config.engine_id,
|
||||
make_kv_cache_config(block_size=16),
|
||||
make_kv_cache_config(block_size=block_size),
|
||||
)
|
||||
worker = NixlConnectorWorker(
|
||||
vllm_config,
|
||||
vllm_config.kv_transfer_config.engine_id,
|
||||
make_kv_cache_config(block_size=16),
|
||||
make_kv_cache_config(block_size=block_size),
|
||||
)
|
||||
nixl_wrapper = worker.nixl_wrapper
|
||||
|
||||
@@ -2643,7 +2692,7 @@ def test_shutdown_cleans_up_resources(default_vllm_config, dist_init):
|
||||
# Mock register_kv_cache which registers local handle
|
||||
worker.src_xfer_handles_by_block_size = {worker.block_size: 455}
|
||||
# P TP = 2 * D TP case, we should register 2 local handles
|
||||
worker.src_xfer_handles_by_tp_ratio = {(-2, 16): [456, 457]}
|
||||
worker.src_xfer_handles_by_tp_ratio = {(-2, block_size): [456, 457]}
|
||||
worker.dst_xfer_side_handles = {"engine1": {0: 789}}
|
||||
worker._remote_agents = {"engine1": {(0, 0): "agent1"}}
|
||||
# _cleanup_remote_engine (called by shutdown) also clears these:
|
||||
@@ -2688,13 +2737,15 @@ def _setup_worker_with_remote_engine(
|
||||
engine_ttl: float = 10.0,
|
||||
) -> tuple[Any, str]:
|
||||
"""Create a worker with one remote engine registered."""
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(
|
||||
block_size=block_size,
|
||||
kv_connector_extra_config={"engine_ttl": engine_ttl},
|
||||
)
|
||||
worker = NixlConnectorWorker(
|
||||
vllm_config,
|
||||
vllm_config.kv_transfer_config.engine_id,
|
||||
make_kv_cache_config(block_size=16),
|
||||
make_kv_cache_config(block_size=block_size),
|
||||
)
|
||||
|
||||
engine_id = "remote-engine-1"
|
||||
@@ -2857,7 +2908,11 @@ class TestPeerReplacement:
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker as bw
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
||||
|
||||
config = create_vllm_config(kv_connector_extra_config={"engine_ttl": 0})
|
||||
block_size = maybe_update_block_size(16)
|
||||
|
||||
config = create_vllm_config(
|
||||
kv_connector_extra_config={"engine_ttl": 0}, block_size=block_size
|
||||
)
|
||||
config.kv_transfer_config.kv_buffer_device = "cpu"
|
||||
platform = SimpleNamespace(
|
||||
device_type="cpu",
|
||||
@@ -3258,12 +3313,15 @@ def test_aborted_request_removed_from_worker_in_batch(default_vllm_config, dist_
|
||||
the real scheduler, then simulate an abort (request not in next scheduler
|
||||
iteration) and verify the worker no longer tracks it as in-batch.
|
||||
"""
|
||||
vllm_config = create_vllm_config()
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(block_size=block_size)
|
||||
|
||||
scheduler = create_scheduler(vllm_config)
|
||||
# KVConnector Worker in P
|
||||
connector = NixlConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
vllm_config,
|
||||
KVConnectorRole.WORKER,
|
||||
make_kv_cache_config(block_size=block_size),
|
||||
)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config, connector.engine_id, hand_shake_latency=0
|
||||
@@ -3398,9 +3456,12 @@ def test_empty_recv_is_reported_only_when_awaited(
|
||||
A failed notification changes neither: the KV is local either way, and the
|
||||
producer frees its own blocks on a timeout.
|
||||
"""
|
||||
vllm_config = create_vllm_config()
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(block_size=block_size)
|
||||
connector = NixlConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
vllm_config,
|
||||
KVConnectorRole.WORKER,
|
||||
make_kv_cache_config(block_size=block_size),
|
||||
)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config,
|
||||
@@ -3518,12 +3579,13 @@ def test_transfer_failure_logging(
|
||||
"""
|
||||
import logging
|
||||
|
||||
vllm_config = create_vllm_config()
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(block_size=block_size)
|
||||
|
||||
connector = NixlConnector(
|
||||
vllm_config,
|
||||
KVConnectorRole.WORKER,
|
||||
make_kv_cache_config(block_size=16, swa_enabled=enable_hma),
|
||||
make_kv_cache_config(block_size=block_size, swa_enabled=enable_hma),
|
||||
)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config,
|
||||
@@ -3648,10 +3710,13 @@ def test_transfer_failure_logging(
|
||||
)
|
||||
def test_handshake_failure_returns_finished(default_vllm_config, dist_init):
|
||||
"""Test that handshake failures mark blocks invalid and return via get_finished."""
|
||||
vllm_config = create_vllm_config()
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(block_size=block_size)
|
||||
|
||||
connector = NixlConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
vllm_config,
|
||||
KVConnectorRole.WORKER,
|
||||
make_kv_cache_config(block_size=block_size),
|
||||
)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config, connector.engine_id, hand_shake_latency=0.1
|
||||
@@ -3704,10 +3769,13 @@ def test_transfer_setup_failure_returns_finished(
|
||||
default_vllm_config, dist_init, is_hma
|
||||
):
|
||||
"""Setup failures report the request; only non-HMA reports block IDs."""
|
||||
vllm_config = create_vllm_config()
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(block_size=block_size)
|
||||
|
||||
connector = NixlConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
vllm_config,
|
||||
KVConnectorRole.WORKER,
|
||||
make_kv_cache_config(block_size=block_size),
|
||||
)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config, connector.engine_id, hand_shake_latency=0
|
||||
@@ -3776,7 +3844,9 @@ class _ScriptedXferWrapper(FakeNixlWrapper):
|
||||
def _make_split_read_connector(vllm_config, request_id, states):
|
||||
"""Seed a request with scripted in-flight xfer handles + recv metadata."""
|
||||
connector = NixlConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
vllm_config,
|
||||
KVConnectorRole.WORKER,
|
||||
make_kv_cache_config(block_size=vllm_config.cache_config.block_size),
|
||||
)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config, connector.engine_id, hand_shake_latency=0
|
||||
@@ -3818,7 +3888,7 @@ def test_split_read_failure_defers_report_until_last_handle(
|
||||
request_id = "split_read_partial_failure"
|
||||
err_handle, live_handle = 11, 22
|
||||
connector, worker, wrapper = _make_split_read_connector(
|
||||
create_vllm_config(),
|
||||
create_vllm_config(block_size=maybe_update_block_size(16)),
|
||||
request_id,
|
||||
{err_handle: ["ERR"], live_handle: ["PROC", "PROC", "DONE"]},
|
||||
)
|
||||
@@ -3894,10 +3964,15 @@ def test_failed_request_skips_kv_postprocessing(
|
||||
# Use enable_permute_local_kv=True so that
|
||||
# post_process_device_kv_on_receive would be called on the success path,
|
||||
# making the assertion meaningful (not trivially true).
|
||||
vllm_config = create_vllm_config(enable_permute_local_kv=True)
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(
|
||||
enable_permute_local_kv=True, block_size=block_size
|
||||
)
|
||||
|
||||
connector = NixlConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
vllm_config,
|
||||
KVConnectorRole.WORKER,
|
||||
make_kv_cache_config(block_size=block_size),
|
||||
)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config,
|
||||
@@ -4220,14 +4295,15 @@ def test_compatibility_hash_validation(
|
||||
enforce_handshake_compat: whether to enforce compatibility checking
|
||||
|
||||
"""
|
||||
block_size = maybe_update_block_size(16)
|
||||
local_vllm_config = create_vllm_config(
|
||||
model="facebook/opt-125m",
|
||||
block_size=16,
|
||||
block_size=block_size,
|
||||
kv_connector_extra_config={
|
||||
"enforce_handshake_compat": enforce_handshake_compat
|
||||
},
|
||||
)
|
||||
kv_cache_config = make_kv_cache_config(block_size=16, num_blocks=2)
|
||||
kv_cache_config = make_kv_cache_config(block_size=block_size, num_blocks=2)
|
||||
decode_connector = NixlConnector(
|
||||
local_vllm_config, KVConnectorRole.WORKER, kv_cache_config
|
||||
)
|
||||
@@ -4253,7 +4329,7 @@ def test_compatibility_hash_validation(
|
||||
|
||||
remote_config_params: dict[str, Any] = {
|
||||
"model": "facebook/opt-125m",
|
||||
"block_size": 16,
|
||||
"block_size": block_size,
|
||||
**config_overrides,
|
||||
}
|
||||
remote_vllm_config = create_vllm_config(**remote_config_params)
|
||||
@@ -4276,7 +4352,7 @@ def test_compatibility_hash_validation(
|
||||
decode_worker.backend_name,
|
||||
)
|
||||
|
||||
prefill_block_size = config_overrides.get("block_size", 16)
|
||||
prefill_block_size = remote_vllm_config.cache_config.block_size
|
||||
prefill_block_lens = [4096 * prefill_block_size]
|
||||
prefill_metadata = NixlAgentMetadata(
|
||||
engine_id=FakeNixlConnectorWorker.REMOTE_ENGINE_ID,
|
||||
@@ -4352,12 +4428,15 @@ def test_handshake_decode_errors(default_vllm_config, dist_init, error_scenario)
|
||||
- NixlHandshakePayload decoder
|
||||
- NixlAgentMetadata decoder
|
||||
"""
|
||||
block_size = maybe_update_block_size(16)
|
||||
local_vllm_config = create_vllm_config(
|
||||
model="facebook/opt-125m",
|
||||
block_size=16,
|
||||
block_size=block_size,
|
||||
)
|
||||
decode_connector = NixlConnector(
|
||||
local_vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
local_vllm_config,
|
||||
KVConnectorRole.WORKER,
|
||||
make_kv_cache_config(block_size=block_size),
|
||||
)
|
||||
decode_worker = decode_connector.connector_worker
|
||||
|
||||
@@ -4441,11 +4520,14 @@ def test_handshake_decode_errors(default_vllm_config, dist_init, error_scenario)
|
||||
decode_tp_size = 1
|
||||
prefill_tp_size = 4
|
||||
|
||||
vllm_config = create_vllm_config()
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(block_size=block_size)
|
||||
vllm_config.parallel_config.tensor_parallel_size = decode_tp_size
|
||||
|
||||
connector = NixlConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
vllm_config,
|
||||
KVConnectorRole.WORKER,
|
||||
make_kv_cache_config(block_size=block_size),
|
||||
)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config, connector.engine_id, hand_shake_latency=0
|
||||
@@ -4550,7 +4632,8 @@ def test_kv_both_deprecation_warning(default_vllm_config, dist_init):
|
||||
|
||||
_print_warning_once.cache_clear()
|
||||
|
||||
vllm_config = create_vllm_config(kv_role="kv_both")
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(kv_role="kv_both", block_size=block_size)
|
||||
|
||||
with patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.connector.logger"
|
||||
@@ -4559,7 +4642,7 @@ def test_kv_both_deprecation_warning(default_vllm_config, dist_init):
|
||||
NixlConnector(
|
||||
vllm_config,
|
||||
KVConnectorRole.WORKER,
|
||||
make_kv_cache_config(block_size=16),
|
||||
make_kv_cache_config(block_size=block_size),
|
||||
)
|
||||
|
||||
mock_logger.warning_once.assert_called_once()
|
||||
@@ -4575,14 +4658,15 @@ def test_kv_both_deprecation_warning(default_vllm_config, dist_init):
|
||||
def test_explicit_kv_role_no_deprecation_warning(default_vllm_config, dist_init):
|
||||
"""kv_role='kv_consumer' or 'kv_producer' should NOT emit a warning."""
|
||||
for role in ("kv_consumer", "kv_producer"):
|
||||
vllm_config = create_vllm_config(kv_role=role)
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(kv_role=role, block_size=block_size)
|
||||
with patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.connector.logger"
|
||||
) as mock_logger:
|
||||
NixlConnector(
|
||||
vllm_config,
|
||||
KVConnectorRole.WORKER,
|
||||
make_kv_cache_config(block_size=16),
|
||||
make_kv_cache_config(block_size=block_size),
|
||||
)
|
||||
|
||||
(
|
||||
|
||||
@@ -39,6 +39,7 @@ from .utils import (
|
||||
create_vllm_config,
|
||||
make_kv_cache_config,
|
||||
make_nixl_scheduler,
|
||||
maybe_update_block_size,
|
||||
)
|
||||
|
||||
|
||||
@@ -1027,7 +1028,7 @@ def test_fewer_blocks_with_hma(monkeypatch, model_name, sw_size):
|
||||
kv_connector="NixlConnector",
|
||||
kv_role="kv_consumer",
|
||||
)
|
||||
block_size = 16
|
||||
block_size = maybe_update_block_size(16)
|
||||
llm_kwargs = {
|
||||
"model": model_name,
|
||||
"enforce_eager": True,
|
||||
|
||||
@@ -31,6 +31,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.example_connector import ( # noqa
|
||||
ExampleConnector,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.hashing import sha256
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash
|
||||
@@ -59,6 +60,10 @@ def create_mock_connector() -> Mock:
|
||||
return connector
|
||||
|
||||
|
||||
def maybe_update_block_size(block_size: int) -> int:
|
||||
return 64 if current_platform.is_xpu() else block_size
|
||||
|
||||
|
||||
def assert_scheduler_empty(scheduler: Scheduler):
|
||||
"""Confirm the scheduler is "empty" - i.e. no leaks."""
|
||||
# Scheduler Metadata.
|
||||
|
||||
@@ -355,6 +355,8 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
def get_supported_kernel_block_sizes(
|
||||
cls, kv_cache_spec: KVCacheSpec | None = None
|
||||
) -> list[int | MultipleOf]:
|
||||
if current_platform.is_xpu():
|
||||
return [MultipleOf(64)]
|
||||
if block_size := cls._get_sm90_fa4_fp8_kv_block_size(kv_cache_spec):
|
||||
return [block_size]
|
||||
if block_size := cls._get_fa4_hd256_block_size(kv_cache_spec):
|
||||
|
||||
@@ -14,9 +14,11 @@ from vllm.config import (
|
||||
get_layers_from_vllm_config,
|
||||
)
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionCGSupport,
|
||||
CommonAttentionMetadata,
|
||||
@@ -45,6 +47,8 @@ from vllm.v1.worker.utils import (
|
||||
prepare_kernel_block_sizes,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
@@ -263,10 +267,24 @@ def init_attn_backend(
|
||||
# by all backends within that group.
|
||||
kernel_block_sizes = prepare_kernel_block_sizes(kv_cache_config, attn_groups)
|
||||
|
||||
if current_platform.is_xpu():
|
||||
for kv_cache_group_id, selected_kernel_block_size in enumerate(
|
||||
kernel_block_sizes
|
||||
):
|
||||
if selected_kernel_block_size % 64 != 0:
|
||||
for attn_group in attn_groups[kv_cache_group_id]:
|
||||
logger.warning_once(
|
||||
"On XPU platform, current selected kernel block size is "
|
||||
"not fully optimized: kernel_block_size=%s, backend=%s, "
|
||||
"kv_cache_spec=%s",
|
||||
selected_kernel_block_size,
|
||||
attn_group.backend,
|
||||
repr(attn_group.kv_cache_spec),
|
||||
)
|
||||
# Phase 3: create metadata builders and determine cudagraph support.
|
||||
attn_backend_workspace: torch.Tensor | None = None
|
||||
for kv_cache_group_id, groups in enumerate(attn_groups):
|
||||
kernel_block_size = None
|
||||
kernel_block_size: int | None = None
|
||||
if kv_cache_group_id < len(kernel_block_sizes):
|
||||
kernel_block_size = kernel_block_sizes[kv_cache_group_id]
|
||||
for group in groups:
|
||||
|
||||
Reference in New Issue
Block a user