Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai> Co-authored-by: Yifan Qiao <17067717+ivanium@users.noreply.github.com>
This commit is contained in:
co-authored by
Yifan Qiao
parent
153242a314
commit
9488318c6a
@@ -10,6 +10,7 @@ import asyncio
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector import (
|
||||
@@ -22,6 +23,11 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector im
|
||||
SendBlockMeta,
|
||||
TransferRegion,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
)
|
||||
|
||||
from .test_mooncake_connector import FakeMooncakeWrapper, patch_worker_dependencies
|
||||
from .utils import create_request, create_vllm_config, make_kv_cache_config
|
||||
@@ -440,17 +446,19 @@ def test_request_finished_with_hma_groups():
|
||||
# ---------------------------------------------------------------------------
|
||||
# Worker-side load-failure reporting (HMA vs non-HMA)
|
||||
# ---------------------------------------------------------------------------
|
||||
def _make_kv_consumer_worker(swa_enabled: bool, disable_hma: bool = False):
|
||||
def _make_kv_consumer_worker(
|
||||
swa_enabled: bool = False, kv_cache_config: KVCacheConfig | None = None
|
||||
):
|
||||
block_size = 16
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeConnector",
|
||||
kv_role="kv_consumer",
|
||||
block_size=block_size,
|
||||
)
|
||||
vllm_config.scheduler_config.disable_hybrid_kv_cache_manager = disable_hma
|
||||
kv_cache_config = make_kv_cache_config(
|
||||
block_size=block_size, swa_enabled=swa_enabled
|
||||
)
|
||||
if kv_cache_config is None:
|
||||
kv_cache_config = make_kv_cache_config(
|
||||
block_size=block_size, swa_enabled=swa_enabled
|
||||
)
|
||||
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
|
||||
connector = MooncakeConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, kv_cache_config
|
||||
@@ -472,19 +480,43 @@ def _make_pull_meta(d_req_id: str, local_block_ids: list[list[int]]) -> PullReqM
|
||||
# requires an accelerator device, so (like the worker tests in
|
||||
# test_mooncake_connector.py) they run on the GPU KV-connectors job only.
|
||||
@pytest.mark.parametrize(
|
||||
"swa_enabled,disable_hma,expected_is_hma",
|
||||
"swa_enabled,expected_is_hma",
|
||||
[
|
||||
(True, False, True), # SWA group present, HMA enabled
|
||||
(True, True, False), # SWA group present, but HMA disabled
|
||||
(False, False, False), # FA only, HMA not needed
|
||||
(True, True), # multiple groups (FA + SWA)
|
||||
(False, False), # single group, block-level reporting is unambiguous
|
||||
],
|
||||
)
|
||||
def test_worker_is_hma_required(swa_enabled, disable_hma, expected_is_hma):
|
||||
"""Worker-side _is_hma_required mirrors the scheduler's derivation."""
|
||||
worker = _make_kv_consumer_worker(swa_enabled, disable_hma)
|
||||
def test_worker_is_hma_required(swa_enabled, expected_is_hma):
|
||||
"""Worker-side _is_hma_required mirrors the scheduler's condition."""
|
||||
worker = _make_kv_consumer_worker(swa_enabled=swa_enabled)
|
||||
assert worker._is_hma_required is expected_is_hma
|
||||
|
||||
|
||||
def test_worker_is_hma_required_multiple_full_attention_groups():
|
||||
"""Two full-attention groups with different block sizes also select
|
||||
request-level failure reporting, matching the scheduler."""
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=100,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["layer0"],
|
||||
FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=4, head_size=16, dtype=torch.float16
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["layer1"],
|
||||
FullAttentionSpec(
|
||||
block_size=32, num_kv_heads=4, head_size=16, dtype=torch.float16
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
worker = _make_kv_consumer_worker(kv_cache_config=kv_cache_config)
|
||||
assert worker._is_hma_required
|
||||
|
||||
|
||||
def test_worker_failed_recv_reports_request_level_failure_with_hma():
|
||||
"""With HMA, load failures report the request, not ambiguous block IDs."""
|
||||
worker = _make_kv_consumer_worker(swa_enabled=True)
|
||||
|
||||
@@ -2222,15 +2222,40 @@ def test_worker_init_excludes_nonprefix_cache_groups(monkeypatch):
|
||||
assert [db.block_size for db in store_worker.token_dbs] == [800, 800]
|
||||
|
||||
|
||||
def _make_two_full_attention_groups_kv_cache_config():
|
||||
"""Two full-attention groups with different block sizes: no hybrid
|
||||
layers, but block-level failure reporting is still unsupported."""
|
||||
return KVCacheConfig(
|
||||
num_blocks=10,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full0"],
|
||||
FullAttentionSpec(
|
||||
block_size=800, num_kv_heads=8, head_size=64, dtype=None
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["full1"],
|
||||
FullAttentionSpec(
|
||||
block_size=1600, num_kv_heads=8, head_size=64, dtype=None
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("disable_hybrid_kv_cache_manager", "expected_is_hma_required"),
|
||||
("make_config", "expected_is_hma_required"),
|
||||
[
|
||||
(True, False),
|
||||
(False, True),
|
||||
(lambda: _make_qsa_hybrid_kv_cache_config(), True),
|
||||
(lambda: _make_two_full_attention_groups_kv_cache_config(), True),
|
||||
(lambda: _make_kv_cache_config(block_size=800), False),
|
||||
],
|
||||
ids=["hybrid", "two-full-attention-groups", "single-group"],
|
||||
)
|
||||
def test_worker_is_hma_required_from_kv_cache_groups(
|
||||
monkeypatch, disable_hybrid_kv_cache_manager, expected_is_hma_required
|
||||
monkeypatch, make_config, expected_is_hma_required
|
||||
):
|
||||
store = MagicMock()
|
||||
store.setup.return_value = 0
|
||||
@@ -2241,16 +2266,12 @@ def test_worker_is_hma_required_from_kv_cache_groups(
|
||||
"load_from_config",
|
||||
staticmethod(lambda: _make_config()),
|
||||
)
|
||||
vllm_config = _make_vllm_config(
|
||||
disable_hybrid_kv_cache_manager=disable_hybrid_kv_cache_manager
|
||||
)
|
||||
vllm_config = _make_vllm_config()
|
||||
vllm_config.cache_config.block_size = 800
|
||||
vllm_config.cache_config.enable_prefix_caching = True
|
||||
vllm_config.cache_config.prefix_match_unit = None
|
||||
|
||||
store_worker = worker.MooncakeStoreWorker(
|
||||
vllm_config, _make_qsa_hybrid_kv_cache_config()
|
||||
)
|
||||
store_worker = worker.MooncakeStoreWorker(vllm_config, make_config())
|
||||
|
||||
assert store_worker._is_hma_required is expected_is_hma_required
|
||||
|
||||
|
||||
@@ -1051,15 +1051,9 @@ class MooncakeConnectorWorker:
|
||||
self.finished_recving_reqs: set[ReqId] = set()
|
||||
# Written from the receiver loop, drained from the worker thread.
|
||||
self._invalid_block_ids: queue.Queue[set[int]] = queue.Queue()
|
||||
self._is_hma_required = (
|
||||
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
|
||||
and any(
|
||||
not isinstance(g.kv_cache_spec, FullAttentionSpec)
|
||||
for g in kv_cache_config.transfer_groups
|
||||
)
|
||||
)
|
||||
# Block IDs are only unique within a group; with HMA the scheduler
|
||||
# tracks a single merged group, so failures are reported per request.
|
||||
self._is_hma_required = len(kv_cache_config.kv_cache_groups) > 1
|
||||
# Block IDs are only unique within a group, so with multiple groups
|
||||
# load failures are reported per request instead.
|
||||
self._failed_recv_reqs: queue.Queue[ReqId] = queue.Queue()
|
||||
|
||||
self.xfer_stats = MooncakeKVConnectorStats()
|
||||
|
||||
@@ -1461,13 +1461,7 @@ class MooncakeStoreWorker:
|
||||
and not self.can_put
|
||||
)
|
||||
self.cache_config = vllm_config.cache_config
|
||||
self._is_hma_required = (
|
||||
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
|
||||
and any(
|
||||
not isinstance(g.kv_cache_spec, FullAttentionSpec)
|
||||
for g in kv_cache_config.transfer_groups
|
||||
)
|
||||
)
|
||||
self._is_hma_required = len(kv_cache_config.kv_cache_groups) > 1
|
||||
self.block_size, self.hash_block_size = resolve_kv_cache_block_sizes(
|
||||
kv_cache_config, vllm_config
|
||||
)
|
||||
|
||||
@@ -3207,8 +3207,8 @@ class Scheduler(SchedulerInterface):
|
||||
# invalid block IDs cannot be mapped back to requests.
|
||||
raise RuntimeError(
|
||||
"A KV connector reported block-level load failures "
|
||||
"(invalid_block_ids) on a layout with multiple KV cache "
|
||||
"groups, where block IDs are only unique within a group. "
|
||||
"(invalid_block_ids) which is not supported for models "
|
||||
"with multiple KV cache groups. "
|
||||
"Connectors must report failed requests via "
|
||||
"KVConnectorTransferResults.failed_recving instead."
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user