[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com>
Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Matthew Bonanni
2026-09-12 00:36:47 -04:00
committed by GitHub
co-authored by OpenAI Codex Lucas Wilkinson Lucas Wilkinson Nicolò Lucchesi OpenAI Codex Claude Opus 5 Lucas Wilkinson
parent 120ec4ebd2
commit d43bb2f37f
124 changed files with 14262 additions and 906 deletions
+29 -13
View File
@@ -67,7 +67,6 @@ from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
KVCacheConfig,
MambaSpec,
UniformTypeKVCacheSpecs,
@@ -590,7 +589,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
layer_spec = (
spec.first_spec if isinstance(spec, UniformTypeKVCacheSpecs) else spec
)
slot_mapping_enabled.append(not isinstance(layer_spec, CircularBufferSpec))
slot_mapping_enabled.append(layer_spec.uses_slot_mapping)
# Let each cache type account for CP. Attention KV is DCP-sharded,
# while Mamba/GDN recurrent state is replicated across DCP ranks.
max_num_blocks = spec.max_num_blocks_per_req(
@@ -598,12 +597,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
)
# Preserve each cache type's alignment requirements after applying
# its topology-aware block-table width.
if isinstance(layer_spec, (MambaSpec, CircularBufferSpec)):
max_num_blocks = get_block_table_width(
max_num_blocks, spec.block_size, token_alignment=None
)
else:
max_num_blocks = get_block_table_width(max_num_blocks, spec.block_size)
max_num_blocks = get_block_table_width(
max_num_blocks,
spec.block_size,
token_alignment=spec.block_table_token_alignment,
)
max_num_blocks_per_group.append(max_num_blocks)
target_attn_layer_names = None
@@ -722,18 +720,20 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# to its own attention support.
self.speculator.init_cudagraph_manager(cudagraph_mode)
self.kv_caches: list[torch.Tensor] = []
# Capture warmup providers that depend on allocated KV-cache strides.
with self.jit_warmup_registry.activate():
kv_caches_dict = init_kv_cache(
self.kv_caches,
self.compilation_config.static_forward_context,
self.kv_cache_config,
self.device,
self.kernel_block_sizes,
self.vllm_config,
kv_cache_allocation_context=kv_cache_allocation_context,
block_tables=self.block_tables,
)
self.kv_caches = [
cache for cache in kv_caches_dict.values() if cache.device == self.device
]
if is_profiling:
self.kv_connector = NO_OP_KV_CONNECTOR
else:
@@ -1014,6 +1014,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
):
self._dummy_run(**batch)
self.adaptive_verification.set_initial_cost_curves(timings)
self.kv_connector.reset_capture_state()
end_free_gpu_memory = torch.accelerator.get_memory_info()[0]
@@ -1141,13 +1142,17 @@ class GPUModelRunner(LoRAModelRunnerMixin):
def update_requests(self, scheduler_output: SchedulerOutput) -> None:
# Add new blocks and update num_computed_tokens for the existing requests.
reqs = scheduler_output.scheduled_cached_reqs
table_updates = scheduler_output.block_table_updates or {}
for req_id, block_ids in table_updates.items():
if (idx := self.req_states.req_id_to_index.get(req_id)) is not None:
self.block_tables.append_block_ids(idx, block_ids, overwrite=True)
num_computed_tokens_np = self.req_states.num_computed_tokens_np
for req_id, num_computed_tokens, req_new_block_ids in zip(
reqs.req_ids, reqs.num_computed_tokens, reqs.new_block_ids
):
req_index = self.req_states.req_id_to_index[req_id]
num_computed_tokens_np[req_index] = num_computed_tokens
if req_new_block_ids is not None:
if req_new_block_ids is not None and req_id not in table_updates:
self.block_tables.append_block_ids(
req_index, req_new_block_ids, overwrite=False
)
@@ -1841,13 +1846,22 @@ class GPUModelRunner(LoRAModelRunnerMixin):
)
self.step_timing.forward_start()
connector_kwargs = dict(
scheduler_output=scheduler_output,
request_state_indices=input_batch.idx_mapping,
request_ids=input_batch.req_ids,
num_tokens=input_batch.num_tokens,
)
# Run model.
if batch_desc.cg_mode == CUDAGraphMode.FULL:
# Use explicit cudagraph replay for FULL mode.
# NOTE(woosuk): Here, we don't need to pass the input tensors,
# because they are already copied to the CUDA graph input buffers.
assert self.cudagraph_manager is not None
self.kv_connector.pre_forward(scheduler_output)
self.kv_connector.pre_forward(
**connector_kwargs, attn_metadata=attn_metadata
)
model_output = self.cudagraph_manager.run_fullgraph(batch_desc)
else:
# For piecewise and eager mode, just call model().
@@ -1871,7 +1885,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
skip_compiled=skip_compiled,
is_padding=input_batch.is_padding,
):
self.kv_connector.pre_forward(scheduler_output)
self.kv_connector.pre_forward(**connector_kwargs)
if ubatch_state is not None:
assert self.ubatch_runner is not None
model_output = self.ubatch_runner.run(
@@ -1889,6 +1903,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# Eager (NONE): call the raw model directly.
model_output = self.model(**model_inputs)
self.kv_connector.finish_forward()
if self.is_last_pp_rank:
if self.use_aux_hidden_state_outputs:
assert isinstance(model_output, tuple)