[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com> Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Lucas Wilkinson
Lucas Wilkinson
Nicolò Lucchesi
OpenAI Codex
Claude Opus 5
Lucas Wilkinson
parent
120ec4ebd2
commit
d43bb2f37f
@@ -41,7 +41,9 @@ def test_connector_without_divergent_hit_support_uses_common_lookup():
|
||||
manager = MagicMock()
|
||||
manager.get_computed_blocks.return_value = (common_blocks, 0, 0)
|
||||
scheduler = SimpleNamespace(
|
||||
connector=SimpleNamespace(supports_divergent_local_hybrid_hits=False),
|
||||
connector=SimpleNamespace(
|
||||
supports_divergent_local_hybrid_hits=False,
|
||||
),
|
||||
kv_cache_manager=manager,
|
||||
)
|
||||
|
||||
@@ -61,7 +63,9 @@ def test_capable_connector_uses_divergent_partial_hit_lookup():
|
||||
True,
|
||||
)
|
||||
scheduler = SimpleNamespace(
|
||||
connector=SimpleNamespace(supports_divergent_local_hybrid_hits=True),
|
||||
connector=SimpleNamespace(
|
||||
supports_divergent_local_hybrid_hits=True,
|
||||
),
|
||||
kv_cache_manager=manager,
|
||||
)
|
||||
|
||||
@@ -922,71 +926,6 @@ def test_connector_finish_registers_partial_tail_before_cleanup():
|
||||
scheduler.kv_cache_manager.remove_skipped_blocks.assert_called_once()
|
||||
|
||||
|
||||
def test_block_pool_touch_pins_released_cow_target():
|
||||
"""The connector can rescue an offered CoW target after its step-scoped
|
||||
retention is released by using the bound BlockPool's touch method."""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=24,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=hash_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
req0.num_computed_tokens = 6
|
||||
req0.append_output_token_ids([3])
|
||||
assert manager.allocate_slots(req0, 1) is not None
|
||||
|
||||
# Retention released before the drain (defer_block_free=False ordering).
|
||||
_copies, retained = manager.take_kv_cache_block_copies()
|
||||
manager.block_pool.free_blocks(retained)
|
||||
|
||||
offloads = drain_boundary_state_offloads(manager)
|
||||
((_group_id, block_id, boundary_tokens),) = offloads["0"]
|
||||
assert boundary_tokens == 6
|
||||
cow_block = manager.block_pool.blocks[block_id]
|
||||
assert cow_block.ref_cnt == 0
|
||||
assert block_id in _free_block_ids(manager)
|
||||
|
||||
manager.block_pool.touch([cow_block])
|
||||
assert cow_block.ref_cnt == 1
|
||||
assert block_id not in _free_block_ids(manager)
|
||||
|
||||
# The connector-pinned block is out of the free queue: draining every free block
|
||||
# neither trips the allocator's ref_cnt assert nor hands it out.
|
||||
new_blocks = manager.block_pool.get_new_blocks(
|
||||
manager.block_pool.get_num_free_blocks()
|
||||
)
|
||||
assert block_id not in {b.block_id for b in new_blocks}
|
||||
|
||||
|
||||
def test_boundary_state_offload_dropped_when_request_freed_before_drain():
|
||||
"""A hand-off recorded in the same scheduling pass as the request's death
|
||||
must not be drained: its release hook has already run, so draining would
|
||||
|
||||
@@ -13,7 +13,7 @@ from vllm.v1.request import RequestStatus
|
||||
from vllm.v1.structured_output import StructuredOutputGrammar
|
||||
from vllm.v1.utils import ConstantList
|
||||
|
||||
from .utils import create_requests, create_scheduler
|
||||
from .utils import create_requests, create_scheduler, mock_kv
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
@@ -145,6 +145,32 @@ def test_abort():
|
||||
assert req.num_output_tokens == abort_order_copy.index(i)
|
||||
|
||||
|
||||
def test_connector_metadata_precedes_async_placeholder_advance(monkeypatch):
|
||||
"""Mirroring must see earlier unresolved outputs, excluding the current step."""
|
||||
scheduler = create_scheduler(
|
||||
async_scheduling=True,
|
||||
use_kv_connector=mock_kv(matched_tokens=0, is_async=False),
|
||||
)
|
||||
(request,) = create_requests(num_requests=1, num_tokens=4, max_tokens=4)
|
||||
scheduler.add_request(request)
|
||||
|
||||
assert scheduler.connector is not None
|
||||
build_connector_meta = scheduler.connector.build_connector_meta
|
||||
observed_placeholders = []
|
||||
|
||||
def record_placeholders(scheduler_output):
|
||||
observed_placeholders.append(request.num_output_placeholders)
|
||||
return build_connector_meta(scheduler_output)
|
||||
|
||||
monkeypatch.setattr(
|
||||
scheduler.connector, "build_connector_meta", record_placeholders
|
||||
)
|
||||
scheduler.schedule()
|
||||
scheduler.schedule()
|
||||
|
||||
assert observed_placeholders == [0, 1]
|
||||
|
||||
|
||||
def test_preempt():
|
||||
scheduler = create_scheduler(async_scheduling=True)
|
||||
requests = create_requests(num_requests=10, max_tokens=20)
|
||||
|
||||
@@ -75,6 +75,7 @@ def _mock_vllm_config(layout: str | None):
|
||||
config.cache_config = CacheConfig()
|
||||
config.cache_config.num_gpu_blocks_override = None
|
||||
config.cache_config.kv_cache_layout = layout
|
||||
config.attention_config.hisparse_config = None
|
||||
return config
|
||||
|
||||
|
||||
|
||||
@@ -22,6 +22,7 @@ from vllm.config import (
|
||||
SchedulerConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.config.attention import HiSparseConfig
|
||||
from vllm.config.kv_events import KVEventsConfig
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.multimodal.inputs import (
|
||||
@@ -52,10 +53,16 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
make_block_hash_with_group_id,
|
||||
tensor_data,
|
||||
)
|
||||
from vllm.v1.hisparse.layout import (
|
||||
create_hisparse_layout,
|
||||
get_hisparse_gpu_memory_usage,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
ChunkedLocalAttentionSpec,
|
||||
FullAttentionSpec,
|
||||
HiddenStateCacheSpec,
|
||||
HiSparseHotSpec,
|
||||
HiSparseResidentSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
@@ -68,18 +75,139 @@ from vllm.v1.kv_cache_interface import (
|
||||
SinkFullAttentionSpec,
|
||||
SlidingWindowMLASpec,
|
||||
SlidingWindowSpec,
|
||||
SparseCacheRole,
|
||||
UniformTypeKVCacheSpecs,
|
||||
get_kv_cache_spec_kind,
|
||||
get_kv_cache_spec_sliding_window,
|
||||
is_full_attention_spec,
|
||||
iter_layer_specs,
|
||||
)
|
||||
from vllm.v1.kv_cache_layout import KVCacheLayout
|
||||
from vllm.v1.metrics.stats import CachingMetrics, PrefixCacheStats
|
||||
from vllm.v1.request import Request
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
@pytest.mark.parametrize("gpu_block_size", [32, 64])
|
||||
def test_hisparse_hma_uses_resolved_gpu_block_size(monkeypatch, gpu_block_size):
|
||||
specs = {
|
||||
"model.layers.0.self_attn": MLAAttentionSpec(
|
||||
block_size=gpu_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
is_index_group_leader=True,
|
||||
),
|
||||
"model.layers.0.self_attn.indexer": MLAAttentionSpec(
|
||||
block_size=gpu_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
cache_role=SparseCacheRole.INDEXER,
|
||||
),
|
||||
}
|
||||
group_spec = UniformTypeKVCacheSpecs.from_specs(specs)
|
||||
assert group_spec is not None
|
||||
group = KVCacheGroupSpec(list(specs), group_spec)
|
||||
config = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(hisparse_config=HiSparseConfig()),
|
||||
model_config=SimpleNamespace(
|
||||
hf_config=SimpleNamespace(index_topk=128),
|
||||
max_model_len=gpu_block_size,
|
||||
),
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
|
||||
cache_config=SimpleNamespace(
|
||||
num_gpu_blocks_override=7,
|
||||
prefix_cache_retention_interval=None,
|
||||
get_resolved_kv_cache_layout=lambda: KVCacheLayout.BLHNC,
|
||||
),
|
||||
)
|
||||
indexer_spec = specs["model.layers.0.self_attn.indexer"]
|
||||
assert get_hisparse_gpu_memory_usage(config, [group]) == (
|
||||
indexer_spec.max_memory_usage_bytes(config)
|
||||
)
|
||||
|
||||
monkeypatch.setattr(kv_cache_utils, "get_hisparse_host_pool_bytes", lambda _: 2**30)
|
||||
cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
config, [group], available_memory=2**30
|
||||
)
|
||||
assert cache_config.num_blocks == 7
|
||||
assert cache_config.hisparse_host_num_blocks is not None
|
||||
assert cache_config.hisparse_host_num_blocks > 7
|
||||
|
||||
host_group, indexer_group, *auxiliary_groups = cache_config.kv_cache_groups
|
||||
assert host_group.host_resident
|
||||
assert not any(group.host_resident for group in [indexer_group, *auxiliary_groups])
|
||||
host_layers = set(host_group.layer_names)
|
||||
for tensor in cache_config.kv_cache_tensors:
|
||||
assert all(
|
||||
(name in host_layers) == tensor.host_resident for name in tensor.layers
|
||||
)
|
||||
host_tensor = next(t for t in cache_config.kv_cache_tensors if t.host_resident)
|
||||
host_spec = host_group.kv_cache_spec.kv_cache_specs[host_tensor.layers[0]]
|
||||
assert host_tensor.block_stride == host_spec.page_size_bytes
|
||||
assert host_tensor.layer_stride == (
|
||||
host_spec.page_size_bytes * cache_config.hisparse_host_num_blocks
|
||||
)
|
||||
assert host_group.kv_cache_spec.block_size == gpu_block_size
|
||||
assert indexer_group.kv_cache_spec.block_size == gpu_block_size
|
||||
host_specs = host_group.kv_cache_spec.kv_cache_specs
|
||||
gpu_indexer_specs = indexer_group.kv_cache_spec.kv_cache_specs
|
||||
assert set(host_specs) == {"model.layers.0.self_attn"}
|
||||
assert set(gpu_indexer_specs) == {"model.layers.0.self_attn.indexer"}
|
||||
assert indexer_group.kv_cache_spec.prefix_cacheable
|
||||
assert host_group.enable_kv_transfer
|
||||
auxiliary_specs = [group.kv_cache_spec for group in auxiliary_groups]
|
||||
assert any(isinstance(spec, HiSparseResidentSpec) for spec in auxiliary_specs)
|
||||
assert any(isinstance(spec, HiSparseHotSpec) for spec in auxiliary_specs)
|
||||
assert all(
|
||||
spec.block_size == gpu_block_size
|
||||
for spec in auxiliary_specs
|
||||
if isinstance(spec, (HiSparseResidentSpec, HiSparseHotSpec))
|
||||
)
|
||||
scheduler_block_size, hash_block_size = kv_cache_utils.resolve_kv_cache_block_sizes(
|
||||
cache_config,
|
||||
SimpleNamespace(
|
||||
cache_config=SimpleNamespace(
|
||||
block_size=16,
|
||||
enable_prefix_caching=True,
|
||||
prefix_match_unit=None,
|
||||
),
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
|
||||
kv_transfer_config=object(),
|
||||
),
|
||||
)
|
||||
assert scheduler_block_size == hash_block_size == gpu_block_size
|
||||
|
||||
|
||||
def test_hisparse_rejects_deepseek_v4():
|
||||
full_specs = {
|
||||
"model.layers.0.attn": MLAAttentionSpec(
|
||||
block_size=256,
|
||||
num_kv_heads=1,
|
||||
head_size=512,
|
||||
dtype=torch.uint8,
|
||||
model_version="deepseek_v4",
|
||||
)
|
||||
}
|
||||
full_uniform = UniformTypeKVCacheSpecs.from_specs(full_specs)
|
||||
assert full_uniform is not None
|
||||
group = KVCacheGroupSpec(list(full_specs), full_uniform)
|
||||
config = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(hisparse_config=HiSparseConfig()),
|
||||
model_config=SimpleNamespace(hf_config=SimpleNamespace(index_topk=512)),
|
||||
cache_config=SimpleNamespace(num_gpu_blocks_override=7),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="does not support DeepSeek V4"):
|
||||
create_hisparse_layout(
|
||||
config,
|
||||
[group],
|
||||
host_budget=2**30,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _auto_init_hash_fn(request):
|
||||
hash_fn: Callable
|
||||
@@ -3848,6 +3976,22 @@ def test_iter_layer_specs_returns_group_members():
|
||||
assert list(iter_layer_specs(wrapped)) == [full, mla]
|
||||
|
||||
|
||||
def test_wrapped_mamba_group_requires_block_zeroing():
|
||||
mamba = MambaSpec(
|
||||
block_size=4,
|
||||
shapes=((4, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
)
|
||||
wrapped = UniformTypeKVCacheSpecs(block_size=4, kv_cache_specs={"mamba": mamba})
|
||||
config = KVCacheConfig(
|
||||
num_blocks=4,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[KVCacheGroupSpec(["mamba"], wrapped)],
|
||||
)
|
||||
|
||||
assert config.needs_kv_cache_zeroing
|
||||
|
||||
|
||||
def _spec_decode_grouping_config(method="dspark", model_type=None):
|
||||
"""Grouping config with an EAGLE-family speculative method enabled."""
|
||||
return SimpleNamespace(
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -19,7 +19,12 @@ from vllm.config import (
|
||||
SpeculativeConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import (
|
||||
HiSparseConnector,
|
||||
HiSparseConnectorScheduler,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.multi_connector import MultiConnector
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalKwargsItem,
|
||||
@@ -34,6 +39,7 @@ from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
from vllm.v1.core.single_type_kv_cache_manager import register_all_kvcache_specs
|
||||
from vllm.v1.engine import FinishReason
|
||||
from vllm.v1.hisparse.coordinator import get_hisparse_coordinator
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
@@ -148,6 +154,30 @@ def test_get_num_unfinished_requests():
|
||||
assert scheduler.get_num_unfinished_requests() == len(requests) - i - 1
|
||||
|
||||
|
||||
def _bind_hisparse_connector(scheduler):
|
||||
connector = object.__new__(HiSparseConnector)
|
||||
connector.connector_scheduler = HiSparseConnectorScheduler(async_speculative=False)
|
||||
from tests.v1.core.test_prefix_caching import make_hisparse_kv_cache_manager
|
||||
|
||||
coordinator = get_hisparse_coordinator(make_hisparse_kv_cache_manager(16, 16))
|
||||
connector.connector_scheduler.bind_coordinator(coordinator)
|
||||
composite = object.__new__(MultiConnector)
|
||||
composite._connectors = (connector,)
|
||||
scheduler.connector = composite
|
||||
return coordinator
|
||||
|
||||
|
||||
def test_pending_hisparse_spill_keeps_scheduler_alive():
|
||||
"""A final host spill must complete after the last request finishes."""
|
||||
scheduler = create_scheduler()
|
||||
coordinator = _bind_hisparse_connector(scheduler)
|
||||
pending = coordinator.pending_spills
|
||||
pending[0] = Mock()
|
||||
assert scheduler.has_requests()
|
||||
pending.clear()
|
||||
assert not scheduler.has_requests()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"enable_prefix_caching, prompt_logprobs",
|
||||
[
|
||||
@@ -1181,6 +1211,45 @@ def test_preempt_during_execution():
|
||||
assert requests[1].output_token_ids[0] == 42
|
||||
|
||||
|
||||
def test_pending_hisparse_reclamation_defers_preemption(monkeypatch):
|
||||
"""Pending reclamation must stall rather than preempt its owning request."""
|
||||
scheduler = create_scheduler(enable_prefix_caching=False)
|
||||
request = create_requests(num_requests=1, num_tokens=16, block_size=16)[0]
|
||||
scheduler.add_request(request)
|
||||
scheduler_output = scheduler.schedule()
|
||||
scheduler.update_from_output(
|
||||
scheduler_output,
|
||||
ModelRunnerOutput(
|
||||
req_ids=[request.request_id],
|
||||
req_id_to_index={request.request_id: 0},
|
||||
sampled_token_ids=[[0]],
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
scheduler.kv_cache_manager, "allocate_slots", Mock(return_value=None)
|
||||
)
|
||||
coordinator = _bind_hisparse_connector(scheduler)
|
||||
monkeypatch.setattr(
|
||||
scheduler.connector, "build_connector_meta", Mock(return_value=None)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
coordinator,
|
||||
"has_pending_reclamation",
|
||||
Mock(return_value=True),
|
||||
)
|
||||
|
||||
deferred_output = scheduler.schedule()
|
||||
|
||||
assert not deferred_output.num_scheduled_tokens
|
||||
assert scheduler.running == [request]
|
||||
assert request.status == RequestStatus.RUNNING
|
||||
assert request.num_preemptions == 0
|
||||
|
||||
|
||||
def test_prefix_cache_query_not_inflated_by_connector_defer():
|
||||
"""The GPU prefix-cache query is recorded at admission, so a request the
|
||||
connector defers several times is counted once, not once per retry."""
|
||||
|
||||
Reference in New Issue
Block a user