[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com>
Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Matthew Bonanni
2026-09-12 00:36:47 -04:00
committed by GitHub
co-authored by OpenAI Codex Lucas Wilkinson Lucas Wilkinson Nicolò Lucchesi OpenAI Codex Claude Opus 5 Lucas Wilkinson
parent 120ec4ebd2
commit d43bb2f37f
124 changed files with 14262 additions and 906 deletions
@@ -41,7 +41,9 @@ def test_connector_without_divergent_hit_support_uses_common_lookup():
manager = MagicMock()
manager.get_computed_blocks.return_value = (common_blocks, 0, 0)
scheduler = SimpleNamespace(
connector=SimpleNamespace(supports_divergent_local_hybrid_hits=False),
connector=SimpleNamespace(
supports_divergent_local_hybrid_hits=False,
),
kv_cache_manager=manager,
)
@@ -61,7 +63,9 @@ def test_capable_connector_uses_divergent_partial_hit_lookup():
True,
)
scheduler = SimpleNamespace(
connector=SimpleNamespace(supports_divergent_local_hybrid_hits=True),
connector=SimpleNamespace(
supports_divergent_local_hybrid_hits=True,
),
kv_cache_manager=manager,
)
@@ -922,71 +926,6 @@ def test_connector_finish_registers_partial_tail_before_cleanup():
scheduler.kv_cache_manager.remove_skipped_blocks.assert_called_once()
def test_block_pool_touch_pins_released_cow_target():
"""The connector can rescue an offered CoW target after its step-scoped
retention is released by using the bound BlockPool's touch method."""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
num_blocks=24,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=hash_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
)
manager = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
req0.num_computed_tokens = 6
req0.append_output_token_ids([3])
assert manager.allocate_slots(req0, 1) is not None
# Retention released before the drain (defer_block_free=False ordering).
_copies, retained = manager.take_kv_cache_block_copies()
manager.block_pool.free_blocks(retained)
offloads = drain_boundary_state_offloads(manager)
((_group_id, block_id, boundary_tokens),) = offloads["0"]
assert boundary_tokens == 6
cow_block = manager.block_pool.blocks[block_id]
assert cow_block.ref_cnt == 0
assert block_id in _free_block_ids(manager)
manager.block_pool.touch([cow_block])
assert cow_block.ref_cnt == 1
assert block_id not in _free_block_ids(manager)
# The connector-pinned block is out of the free queue: draining every free block
# neither trips the allocator's ref_cnt assert nor hands it out.
new_blocks = manager.block_pool.get_new_blocks(
manager.block_pool.get_num_free_blocks()
)
assert block_id not in {b.block_id for b in new_blocks}
def test_boundary_state_offload_dropped_when_request_freed_before_drain():
"""A hand-off recorded in the same scheduling pass as the request's death
must not be drained: its release hook has already run, so draining would
+27 -1
View File
@@ -13,7 +13,7 @@ from vllm.v1.request import RequestStatus
from vllm.v1.structured_output import StructuredOutputGrammar
from vllm.v1.utils import ConstantList
from .utils import create_requests, create_scheduler
from .utils import create_requests, create_scheduler, mock_kv
pytestmark = pytest.mark.cpu_test
@@ -145,6 +145,32 @@ def test_abort():
assert req.num_output_tokens == abort_order_copy.index(i)
def test_connector_metadata_precedes_async_placeholder_advance(monkeypatch):
"""Mirroring must see earlier unresolved outputs, excluding the current step."""
scheduler = create_scheduler(
async_scheduling=True,
use_kv_connector=mock_kv(matched_tokens=0, is_async=False),
)
(request,) = create_requests(num_requests=1, num_tokens=4, max_tokens=4)
scheduler.add_request(request)
assert scheduler.connector is not None
build_connector_meta = scheduler.connector.build_connector_meta
observed_placeholders = []
def record_placeholders(scheduler_output):
observed_placeholders.append(request.num_output_placeholders)
return build_connector_meta(scheduler_output)
monkeypatch.setattr(
scheduler.connector, "build_connector_meta", record_placeholders
)
scheduler.schedule()
scheduler.schedule()
assert observed_placeholders == [0, 1]
def test_preempt():
scheduler = create_scheduler(async_scheduling=True)
requests = create_requests(num_requests=10, max_tokens=20)
@@ -75,6 +75,7 @@ def _mock_vllm_config(layout: str | None):
config.cache_config = CacheConfig()
config.cache_config.num_gpu_blocks_override = None
config.cache_config.kv_cache_layout = layout
config.attention_config.hisparse_config = None
return config
+144
View File
@@ -22,6 +22,7 @@ from vllm.config import (
SchedulerConfig,
VllmConfig,
)
from vllm.config.attention import HiSparseConfig
from vllm.config.kv_events import KVEventsConfig
from vllm.lora.request import LoRARequest
from vllm.multimodal.inputs import (
@@ -52,10 +53,16 @@ from vllm.v1.core.kv_cache_utils import (
make_block_hash_with_group_id,
tensor_data,
)
from vllm.v1.hisparse.layout import (
create_hisparse_layout,
get_hisparse_gpu_memory_usage,
)
from vllm.v1.kv_cache_interface import (
ChunkedLocalAttentionSpec,
FullAttentionSpec,
HiddenStateCacheSpec,
HiSparseHotSpec,
HiSparseResidentSpec,
KpoolTailSpec,
KVCacheConfig,
KVCacheGroupSpec,
@@ -68,18 +75,139 @@ from vllm.v1.kv_cache_interface import (
SinkFullAttentionSpec,
SlidingWindowMLASpec,
SlidingWindowSpec,
SparseCacheRole,
UniformTypeKVCacheSpecs,
get_kv_cache_spec_kind,
get_kv_cache_spec_sliding_window,
is_full_attention_spec,
iter_layer_specs,
)
from vllm.v1.kv_cache_layout import KVCacheLayout
from vllm.v1.metrics.stats import CachingMetrics, PrefixCacheStats
from vllm.v1.request import Request
pytestmark = pytest.mark.cpu_test
@pytest.mark.parametrize("gpu_block_size", [32, 64])
def test_hisparse_hma_uses_resolved_gpu_block_size(monkeypatch, gpu_block_size):
specs = {
"model.layers.0.self_attn": MLAAttentionSpec(
block_size=gpu_block_size,
num_kv_heads=1,
head_size=576,
dtype=torch.bfloat16,
is_index_group_leader=True,
),
"model.layers.0.self_attn.indexer": MLAAttentionSpec(
block_size=gpu_block_size,
num_kv_heads=1,
head_size=128,
dtype=torch.bfloat16,
cache_role=SparseCacheRole.INDEXER,
),
}
group_spec = UniformTypeKVCacheSpecs.from_specs(specs)
assert group_spec is not None
group = KVCacheGroupSpec(list(specs), group_spec)
config = SimpleNamespace(
attention_config=SimpleNamespace(hisparse_config=HiSparseConfig()),
model_config=SimpleNamespace(
hf_config=SimpleNamespace(index_topk=128),
max_model_len=gpu_block_size,
),
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
cache_config=SimpleNamespace(
num_gpu_blocks_override=7,
prefix_cache_retention_interval=None,
get_resolved_kv_cache_layout=lambda: KVCacheLayout.BLHNC,
),
)
indexer_spec = specs["model.layers.0.self_attn.indexer"]
assert get_hisparse_gpu_memory_usage(config, [group]) == (
indexer_spec.max_memory_usage_bytes(config)
)
monkeypatch.setattr(kv_cache_utils, "get_hisparse_host_pool_bytes", lambda _: 2**30)
cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
config, [group], available_memory=2**30
)
assert cache_config.num_blocks == 7
assert cache_config.hisparse_host_num_blocks is not None
assert cache_config.hisparse_host_num_blocks > 7
host_group, indexer_group, *auxiliary_groups = cache_config.kv_cache_groups
assert host_group.host_resident
assert not any(group.host_resident for group in [indexer_group, *auxiliary_groups])
host_layers = set(host_group.layer_names)
for tensor in cache_config.kv_cache_tensors:
assert all(
(name in host_layers) == tensor.host_resident for name in tensor.layers
)
host_tensor = next(t for t in cache_config.kv_cache_tensors if t.host_resident)
host_spec = host_group.kv_cache_spec.kv_cache_specs[host_tensor.layers[0]]
assert host_tensor.block_stride == host_spec.page_size_bytes
assert host_tensor.layer_stride == (
host_spec.page_size_bytes * cache_config.hisparse_host_num_blocks
)
assert host_group.kv_cache_spec.block_size == gpu_block_size
assert indexer_group.kv_cache_spec.block_size == gpu_block_size
host_specs = host_group.kv_cache_spec.kv_cache_specs
gpu_indexer_specs = indexer_group.kv_cache_spec.kv_cache_specs
assert set(host_specs) == {"model.layers.0.self_attn"}
assert set(gpu_indexer_specs) == {"model.layers.0.self_attn.indexer"}
assert indexer_group.kv_cache_spec.prefix_cacheable
assert host_group.enable_kv_transfer
auxiliary_specs = [group.kv_cache_spec for group in auxiliary_groups]
assert any(isinstance(spec, HiSparseResidentSpec) for spec in auxiliary_specs)
assert any(isinstance(spec, HiSparseHotSpec) for spec in auxiliary_specs)
assert all(
spec.block_size == gpu_block_size
for spec in auxiliary_specs
if isinstance(spec, (HiSparseResidentSpec, HiSparseHotSpec))
)
scheduler_block_size, hash_block_size = kv_cache_utils.resolve_kv_cache_block_sizes(
cache_config,
SimpleNamespace(
cache_config=SimpleNamespace(
block_size=16,
enable_prefix_caching=True,
prefix_match_unit=None,
),
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
kv_transfer_config=object(),
),
)
assert scheduler_block_size == hash_block_size == gpu_block_size
def test_hisparse_rejects_deepseek_v4():
full_specs = {
"model.layers.0.attn": MLAAttentionSpec(
block_size=256,
num_kv_heads=1,
head_size=512,
dtype=torch.uint8,
model_version="deepseek_v4",
)
}
full_uniform = UniformTypeKVCacheSpecs.from_specs(full_specs)
assert full_uniform is not None
group = KVCacheGroupSpec(list(full_specs), full_uniform)
config = SimpleNamespace(
attention_config=SimpleNamespace(hisparse_config=HiSparseConfig()),
model_config=SimpleNamespace(hf_config=SimpleNamespace(index_topk=512)),
cache_config=SimpleNamespace(num_gpu_blocks_override=7),
)
with pytest.raises(ValueError, match="does not support DeepSeek V4"):
create_hisparse_layout(
config,
[group],
host_budget=2**30,
)
@pytest.fixture(autouse=True)
def _auto_init_hash_fn(request):
hash_fn: Callable
@@ -3848,6 +3976,22 @@ def test_iter_layer_specs_returns_group_members():
assert list(iter_layer_specs(wrapped)) == [full, mla]
def test_wrapped_mamba_group_requires_block_zeroing():
mamba = MambaSpec(
block_size=4,
shapes=((4, 1),),
dtypes=(torch.float32,),
)
wrapped = UniformTypeKVCacheSpecs(block_size=4, kv_cache_specs={"mamba": mamba})
config = KVCacheConfig(
num_blocks=4,
kv_cache_tensors=[],
kv_cache_groups=[KVCacheGroupSpec(["mamba"], wrapped)],
)
assert config.needs_kv_cache_zeroing
def _spec_decode_grouping_config(method="dspark", model_type=None):
"""Grouping config with an EAGLE-family speculative method enabled."""
return SimpleNamespace(
File diff suppressed because it is too large Load Diff
+69
View File
@@ -19,7 +19,12 @@ from vllm.config import (
SpeculativeConfig,
VllmConfig,
)
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import (
HiSparseConnector,
HiSparseConnectorScheduler,
)
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats
from vllm.distributed.kv_transfer.kv_connector.v1.multi_connector import MultiConnector
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
MultiModalKwargsItem,
@@ -34,6 +39,7 @@ from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput
from vllm.v1.core.sched.scheduler import Scheduler
from vllm.v1.core.single_type_kv_cache_manager import register_all_kvcache_specs
from vllm.v1.engine import FinishReason
from vllm.v1.hisparse.coordinator import get_hisparse_coordinator
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
@@ -148,6 +154,30 @@ def test_get_num_unfinished_requests():
assert scheduler.get_num_unfinished_requests() == len(requests) - i - 1
def _bind_hisparse_connector(scheduler):
connector = object.__new__(HiSparseConnector)
connector.connector_scheduler = HiSparseConnectorScheduler(async_speculative=False)
from tests.v1.core.test_prefix_caching import make_hisparse_kv_cache_manager
coordinator = get_hisparse_coordinator(make_hisparse_kv_cache_manager(16, 16))
connector.connector_scheduler.bind_coordinator(coordinator)
composite = object.__new__(MultiConnector)
composite._connectors = (connector,)
scheduler.connector = composite
return coordinator
def test_pending_hisparse_spill_keeps_scheduler_alive():
"""A final host spill must complete after the last request finishes."""
scheduler = create_scheduler()
coordinator = _bind_hisparse_connector(scheduler)
pending = coordinator.pending_spills
pending[0] = Mock()
assert scheduler.has_requests()
pending.clear()
assert not scheduler.has_requests()
@pytest.mark.parametrize(
"enable_prefix_caching, prompt_logprobs",
[
@@ -1181,6 +1211,45 @@ def test_preempt_during_execution():
assert requests[1].output_token_ids[0] == 42
def test_pending_hisparse_reclamation_defers_preemption(monkeypatch):
"""Pending reclamation must stall rather than preempt its owning request."""
scheduler = create_scheduler(enable_prefix_caching=False)
request = create_requests(num_requests=1, num_tokens=16, block_size=16)[0]
scheduler.add_request(request)
scheduler_output = scheduler.schedule()
scheduler.update_from_output(
scheduler_output,
ModelRunnerOutput(
req_ids=[request.request_id],
req_id_to_index={request.request_id: 0},
sampled_token_ids=[[0]],
logprobs=None,
prompt_logprobs_dict={},
pooler_output=[],
),
)
monkeypatch.setattr(
scheduler.kv_cache_manager, "allocate_slots", Mock(return_value=None)
)
coordinator = _bind_hisparse_connector(scheduler)
monkeypatch.setattr(
scheduler.connector, "build_connector_meta", Mock(return_value=None)
)
monkeypatch.setattr(
coordinator,
"has_pending_reclamation",
Mock(return_value=True),
)
deferred_output = scheduler.schedule()
assert not deferred_output.num_scheduled_tokens
assert scheduler.running == [request]
assert request.status == RequestStatus.RUNNING
assert request.num_preemptions == 0
def test_prefix_cache_query_not_inflated_by_connector_defer():
"""The GPU prefix-cache query is recorded at admission, so a request the
connector defers several times is counted once, not once per retry."""