[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com>
Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Matthew Bonanni
2026-09-12 00:36:47 -04:00
committed by GitHub
co-authored by OpenAI Codex Lucas Wilkinson Lucas Wilkinson Nicolò Lucchesi OpenAI Codex Claude Opus 5 Lucas Wilkinson
parent 120ec4ebd2
commit d43bb2f37f
124 changed files with 14262 additions and 906 deletions
+36 -11
View File
@@ -18,6 +18,10 @@ from vllm.v1.attention.backend import (
CommonAttentionMetadata,
)
from vllm.v1.attention.backends.utils import create_fast_prefill_custom_backend
from vllm.v1.hisparse.binding import (
init_hisparse_kv_cache,
resolve_hisparse_block_size,
)
from vllm.v1.kv_cache_interface import (
AttentionSpec,
KVCacheConfig,
@@ -30,11 +34,12 @@ from vllm.v1.worker.utils import (
AttentionGroup,
add_kv_sharing_layers_to_kv_cache_groups,
allocate_kv_cache,
bind_kv_cache,
bind_kv_cache_to_layers,
prepare_kernel_block_sizes,
)
if TYPE_CHECKING:
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.cudagraph_utils import (
BatchExecutionDescriptor,
CudaGraphManager,
@@ -132,6 +137,7 @@ def get_kv_cache_spec(vllm_config: VllmConfig) -> dict[str, KVCacheSpec]:
if isinstance(spec, AttentionSpec):
spec = attn_module.get_attn_backend().customize_spec(spec)
kv_cache_spec[layer_name] = spec
resolve_hisparse_block_size(vllm_config, kv_cache_spec, attn_layers)
return kv_cache_spec
@@ -198,6 +204,9 @@ def init_attn_backend(
kv_cache_config.kv_cache_groups
):
layer_names = kv_cache_group_spec.layer_names
if not kv_cache_group_spec.kv_cache_spec.has_layer_views:
attn_groups.append([])
continue
if active_layer_names is not None:
layer_names = list(active_layer_names.intersection(layer_names))
@@ -316,22 +325,34 @@ def get_query_lens_mismatch_unsupported_backend(
def init_kv_cache(
runner_kv_caches: list[torch.Tensor | list[torch.Tensor]],
forward_context: dict[str, Any],
kv_cache_config: KVCacheConfig,
device: torch.device,
kernel_block_sizes: list[int],
vllm_config: VllmConfig,
kv_cache_allocation_context: AbstractContextManager | None = None,
*,
block_tables: "BlockTables | None" = None,
) -> dict[str, Any]:
allocation_context = kv_cache_allocation_context or nullcontext()
with allocation_context:
kv_caches = allocate_kv_cache(
kv_cache_config,
device,
vllm_config.cache_config.get_resolved_kv_cache_layout(),
kernel_block_sizes,
)
if vllm_config.attention_config.hisparse_config is not None:
assert block_tables is not None
kv_caches = init_hisparse_kv_cache(
kv_cache_config,
device,
kernel_block_sizes,
vllm_config,
forward_context,
block_tables,
)
else:
kv_caches = allocate_kv_cache(
kv_cache_config,
device,
vllm_config.cache_config.get_resolved_kv_cache_layout(),
kernel_block_sizes,
)
for layer_name, target in get_shared_kv_cache_layers(vllm_config).items():
kv_caches[layer_name] = kv_caches[target]
# Dual-attention models (e.g. LongCat-Flash) put two Attention modules per
@@ -342,10 +363,12 @@ def init_kv_cache(
in ("longcat_flash", "longcat_flash_ngram")
else 1
)
bind_kv_cache(
kv_caches,
bindable_caches = {
name: cache for name, cache in kv_caches.items() if name in forward_context
}
bind_kv_cache_to_layers(
bindable_caches,
forward_context,
runner_kv_caches,
num_attn_module,
kv_cache_groups=kv_cache_config.kv_cache_groups,
)
@@ -396,6 +419,8 @@ def build_attn_metadata(
attn_metadata: dict[str, Any] = {}
num_kv_cache_groups = len(kv_cache_config.kv_cache_groups)
for i in range(num_kv_cache_groups):
if not attn_groups[i]:
continue
block_table = block_tables[i]
slot_mapping = slot_mappings[i]
# Per-group causal for hybrid drafters (mixed SWA/full attention).
+2
View File
@@ -34,6 +34,7 @@ from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.utils.math_utils import round_up
from vllm.utils.torch_utils import current_stream
from vllm.v1.hisparse.binding import release_hisparse_profiling_cache
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
@@ -931,6 +932,7 @@ def _teardown_profiling_state(runner: "GPUModelRunner") -> None:
if (model := getattr(runner, "model", None)) is not None:
layers = itertools.chain(layers, model.modules())
clear_layer_kv_caches(layers)
release_hisparse_profiling_cache(runner.compilation_config.static_forward_context)
runner.cache_config.num_gpu_blocks = None
runner.maybe_remove_all_loras(runner.lora_config)
gc.collect()
+31 -15
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
import torch
@@ -29,7 +29,10 @@ if TYPE_CHECKING:
class KVConnector:
"""KVConnector interface used by GPUModelRunner."""
def pre_forward(self, scheduler_output: "SchedulerOutput") -> None:
def pre_forward(self, scheduler_output: "SchedulerOutput", **kwargs: Any) -> None:
pass
def finish_forward(self) -> None:
pass
def post_forward(
@@ -43,6 +46,9 @@ class KVConnector:
def set_disabled(self, disabled: bool) -> None:
pass
def reset_capture_state(self) -> None:
pass
class ActiveKVConnector(KVConnector):
def __init__(
@@ -54,10 +60,10 @@ class ActiveKVConnector(KVConnector):
self.kv_connector.register_kv_caches(kv_caches_dict)
self.kv_connector.set_host_xfer_buffer_ops(copy_kv_blocks)
self._pending_load_start = False
self._pending_load_kwargs: dict[str, Any] | None = None
self._disabled = False
def pre_forward(self, scheduler_output: "SchedulerOutput") -> None:
def pre_forward(self, scheduler_output: "SchedulerOutput", **kwargs: Any) -> None:
if self._disabled:
return
@@ -65,23 +71,31 @@ class ActiveKVConnector(KVConnector):
assert kv_connector_metadata is not None
self.kv_connector.handle_preemptions(kv_connector_metadata)
self.kv_connector.bind_connector_metadata(kv_connector_metadata)
self._pending_load_kwargs = kwargs
if scheduler_output.has_sync_kv_loads:
# Sync loads need to run before this step's forward.
self._start_load_kv()
else:
# Start any async loads in post-forward instead, keeping
# their host-side submission cost off the critical path.
self._pending_load_start = True
# Otherwise start the async load after forward to keep submission
# off the critical path.
def _start_load_kv(self) -> None:
self._pending_load_start = False
load_kwargs = self._pending_load_kwargs
assert load_kwargs is not None
self._pending_load_kwargs = None
# TODO: sort out KV Connectors' use of forward_context
if is_forward_context_available():
self.kv_connector.start_load_kv(get_forward_context())
self.kv_connector.start_load_kv(get_forward_context(), **load_kwargs)
else:
with set_forward_context(None, self.vllm_config):
self.kv_connector.start_load_kv(get_forward_context())
self.kv_connector.start_load_kv(get_forward_context(), **load_kwargs)
def finish_forward(self) -> None:
if not self._disabled:
self.kv_connector.finish_forward()
def reset_capture_state(self) -> None:
self.kv_connector.reset_capture_state()
def post_forward(
self, finished_req_ids: set[str], wait_for_save: bool = True
@@ -89,15 +103,16 @@ class ActiveKVConnector(KVConnector):
if self._disabled:
return None
if self._pending_load_start:
if self._pending_load_kwargs is not None:
self._start_load_kv()
output = KVConnectorOutput()
if wait_for_save:
self.kv_connector.wait_for_save()
output.finished_sending, output.finished_recving = (
self.kv_connector.get_finished(finished_req_ids)
)
transfer_results = self.kv_connector.get_transfer_results(finished_req_ids)
output.finished_sending = transfer_results.finished_sending or None
output.finished_recving = transfer_results.finished_recving or None
output.failed_recving = transfer_results.failed_recving
output.invalid_block_ids = self.kv_connector.get_block_ids_with_load_errors()
output.kv_connector_stats = self.kv_connector.get_kv_connector_stats()
output.kv_cache_events = self.kv_connector.get_kv_connector_kv_cache_events()
@@ -112,6 +127,7 @@ class ActiveKVConnector(KVConnector):
return EMPTY_MODEL_RUNNER_OUTPUT
self.pre_forward(scheduler_output)
self.finish_forward()
finished_req_ids = scheduler_output.finished_req_ids
kv_connector_output = self.post_forward(finished_req_ids, wait_for_save=False)
return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
+29 -13
View File
@@ -67,7 +67,6 @@ from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
KVCacheConfig,
MambaSpec,
UniformTypeKVCacheSpecs,
@@ -590,7 +589,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
layer_spec = (
spec.first_spec if isinstance(spec, UniformTypeKVCacheSpecs) else spec
)
slot_mapping_enabled.append(not isinstance(layer_spec, CircularBufferSpec))
slot_mapping_enabled.append(layer_spec.uses_slot_mapping)
# Let each cache type account for CP. Attention KV is DCP-sharded,
# while Mamba/GDN recurrent state is replicated across DCP ranks.
max_num_blocks = spec.max_num_blocks_per_req(
@@ -598,12 +597,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
)
# Preserve each cache type's alignment requirements after applying
# its topology-aware block-table width.
if isinstance(layer_spec, (MambaSpec, CircularBufferSpec)):
max_num_blocks = get_block_table_width(
max_num_blocks, spec.block_size, token_alignment=None
)
else:
max_num_blocks = get_block_table_width(max_num_blocks, spec.block_size)
max_num_blocks = get_block_table_width(
max_num_blocks,
spec.block_size,
token_alignment=spec.block_table_token_alignment,
)
max_num_blocks_per_group.append(max_num_blocks)
target_attn_layer_names = None
@@ -722,18 +720,20 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# to its own attention support.
self.speculator.init_cudagraph_manager(cudagraph_mode)
self.kv_caches: list[torch.Tensor] = []
# Capture warmup providers that depend on allocated KV-cache strides.
with self.jit_warmup_registry.activate():
kv_caches_dict = init_kv_cache(
self.kv_caches,
self.compilation_config.static_forward_context,
self.kv_cache_config,
self.device,
self.kernel_block_sizes,
self.vllm_config,
kv_cache_allocation_context=kv_cache_allocation_context,
block_tables=self.block_tables,
)
self.kv_caches = [
cache for cache in kv_caches_dict.values() if cache.device == self.device
]
if is_profiling:
self.kv_connector = NO_OP_KV_CONNECTOR
else:
@@ -1014,6 +1014,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
):
self._dummy_run(**batch)
self.adaptive_verification.set_initial_cost_curves(timings)
self.kv_connector.reset_capture_state()
end_free_gpu_memory = torch.accelerator.get_memory_info()[0]
@@ -1141,13 +1142,17 @@ class GPUModelRunner(LoRAModelRunnerMixin):
def update_requests(self, scheduler_output: SchedulerOutput) -> None:
# Add new blocks and update num_computed_tokens for the existing requests.
reqs = scheduler_output.scheduled_cached_reqs
table_updates = scheduler_output.block_table_updates or {}
for req_id, block_ids in table_updates.items():
if (idx := self.req_states.req_id_to_index.get(req_id)) is not None:
self.block_tables.append_block_ids(idx, block_ids, overwrite=True)
num_computed_tokens_np = self.req_states.num_computed_tokens_np
for req_id, num_computed_tokens, req_new_block_ids in zip(
reqs.req_ids, reqs.num_computed_tokens, reqs.new_block_ids
):
req_index = self.req_states.req_id_to_index[req_id]
num_computed_tokens_np[req_index] = num_computed_tokens
if req_new_block_ids is not None:
if req_new_block_ids is not None and req_id not in table_updates:
self.block_tables.append_block_ids(
req_index, req_new_block_ids, overwrite=False
)
@@ -1841,13 +1846,22 @@ class GPUModelRunner(LoRAModelRunnerMixin):
)
self.step_timing.forward_start()
connector_kwargs = dict(
scheduler_output=scheduler_output,
request_state_indices=input_batch.idx_mapping,
request_ids=input_batch.req_ids,
num_tokens=input_batch.num_tokens,
)
# Run model.
if batch_desc.cg_mode == CUDAGraphMode.FULL:
# Use explicit cudagraph replay for FULL mode.
# NOTE(woosuk): Here, we don't need to pass the input tensors,
# because they are already copied to the CUDA graph input buffers.
assert self.cudagraph_manager is not None
self.kv_connector.pre_forward(scheduler_output)
self.kv_connector.pre_forward(
**connector_kwargs, attn_metadata=attn_metadata
)
model_output = self.cudagraph_manager.run_fullgraph(batch_desc)
else:
# For piecewise and eager mode, just call model().
@@ -1871,7 +1885,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
skip_compiled=skip_compiled,
is_padding=input_batch.is_padding,
):
self.kv_connector.pre_forward(scheduler_output)
self.kv_connector.pre_forward(**connector_kwargs)
if ubatch_state is not None:
assert self.ubatch_runner is not None
model_output = self.ubatch_runner.run(
@@ -1889,6 +1903,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# Eager (NONE): call the raw model directly.
model_output = self.model(**model_inputs)
self.kv_connector.finish_forward()
if self.is_last_pp_rank:
if self.use_aux_hidden_state_outputs:
assert isinstance(model_output, tuple)
+10
View File
@@ -224,10 +224,20 @@ def warmup_kernels(
# fixed draft counts here, then restore the manager for capture and serving.
adaptive_verification = model_runner.adaptive_verification
model_runner.adaptive_verification = None
rejection_sampler = model_runner.rejection_sampler
adaptive_sampling = (
rejection_sampler is not None and rejection_sampler.enable_adaptive_verification
)
if adaptive_sampling:
assert rejection_sampler is not None
rejection_sampler.enable_adaptive_verification = False
try:
_warmup_kernels(model_runner, worker_execute_model, worker_sample_tokens)
finally:
model_runner.adaptive_verification = adaptive_verification
if adaptive_sampling:
assert rejection_sampler is not None
rejection_sampler.enable_adaptive_verification = True
def _warmup_kernels(