[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com> Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Lucas Wilkinson
Lucas Wilkinson
Nicolò Lucchesi
OpenAI Codex
Claude Opus 5
Lucas Wilkinson
parent
120ec4ebd2
commit
d43bb2f37f
@@ -19,7 +19,12 @@ from vllm.config import (
|
||||
SpeculativeConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import (
|
||||
HiSparseConnector,
|
||||
HiSparseConnectorScheduler,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.multi_connector import MultiConnector
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalKwargsItem,
|
||||
@@ -34,6 +39,7 @@ from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
from vllm.v1.core.single_type_kv_cache_manager import register_all_kvcache_specs
|
||||
from vllm.v1.engine import FinishReason
|
||||
from vllm.v1.hisparse.coordinator import get_hisparse_coordinator
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
@@ -148,6 +154,30 @@ def test_get_num_unfinished_requests():
|
||||
assert scheduler.get_num_unfinished_requests() == len(requests) - i - 1
|
||||
|
||||
|
||||
def _bind_hisparse_connector(scheduler):
|
||||
connector = object.__new__(HiSparseConnector)
|
||||
connector.connector_scheduler = HiSparseConnectorScheduler(async_speculative=False)
|
||||
from tests.v1.core.test_prefix_caching import make_hisparse_kv_cache_manager
|
||||
|
||||
coordinator = get_hisparse_coordinator(make_hisparse_kv_cache_manager(16, 16))
|
||||
connector.connector_scheduler.bind_coordinator(coordinator)
|
||||
composite = object.__new__(MultiConnector)
|
||||
composite._connectors = (connector,)
|
||||
scheduler.connector = composite
|
||||
return coordinator
|
||||
|
||||
|
||||
def test_pending_hisparse_spill_keeps_scheduler_alive():
|
||||
"""A final host spill must complete after the last request finishes."""
|
||||
scheduler = create_scheduler()
|
||||
coordinator = _bind_hisparse_connector(scheduler)
|
||||
pending = coordinator.pending_spills
|
||||
pending[0] = Mock()
|
||||
assert scheduler.has_requests()
|
||||
pending.clear()
|
||||
assert not scheduler.has_requests()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"enable_prefix_caching, prompt_logprobs",
|
||||
[
|
||||
@@ -1181,6 +1211,45 @@ def test_preempt_during_execution():
|
||||
assert requests[1].output_token_ids[0] == 42
|
||||
|
||||
|
||||
def test_pending_hisparse_reclamation_defers_preemption(monkeypatch):
|
||||
"""Pending reclamation must stall rather than preempt its owning request."""
|
||||
scheduler = create_scheduler(enable_prefix_caching=False)
|
||||
request = create_requests(num_requests=1, num_tokens=16, block_size=16)[0]
|
||||
scheduler.add_request(request)
|
||||
scheduler_output = scheduler.schedule()
|
||||
scheduler.update_from_output(
|
||||
scheduler_output,
|
||||
ModelRunnerOutput(
|
||||
req_ids=[request.request_id],
|
||||
req_id_to_index={request.request_id: 0},
|
||||
sampled_token_ids=[[0]],
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
scheduler.kv_cache_manager, "allocate_slots", Mock(return_value=None)
|
||||
)
|
||||
coordinator = _bind_hisparse_connector(scheduler)
|
||||
monkeypatch.setattr(
|
||||
scheduler.connector, "build_connector_meta", Mock(return_value=None)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
coordinator,
|
||||
"has_pending_reclamation",
|
||||
Mock(return_value=True),
|
||||
)
|
||||
|
||||
deferred_output = scheduler.schedule()
|
||||
|
||||
assert not deferred_output.num_scheduled_tokens
|
||||
assert scheduler.running == [request]
|
||||
assert request.status == RequestStatus.RUNNING
|
||||
assert request.num_preemptions == 0
|
||||
|
||||
|
||||
def test_prefix_cache_query_not_inflated_by_connector_defer():
|
||||
"""The GPU prefix-cache query is recorded at admission, so a request the
|
||||
connector defers several times is counted once, not once per retry."""
|
||||
|
||||
Reference in New Issue
Block a user