[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com> Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Lucas Wilkinson
Lucas Wilkinson
Nicolò Lucchesi
OpenAI Codex
Claude Opus 5
Lucas Wilkinson
parent
120ec4ebd2
commit
d43bb2f37f
@@ -67,7 +67,6 @@ from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
|
||||
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
|
||||
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
KVCacheConfig,
|
||||
MambaSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
@@ -590,7 +589,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
layer_spec = (
|
||||
spec.first_spec if isinstance(spec, UniformTypeKVCacheSpecs) else spec
|
||||
)
|
||||
slot_mapping_enabled.append(not isinstance(layer_spec, CircularBufferSpec))
|
||||
slot_mapping_enabled.append(layer_spec.uses_slot_mapping)
|
||||
# Let each cache type account for CP. Attention KV is DCP-sharded,
|
||||
# while Mamba/GDN recurrent state is replicated across DCP ranks.
|
||||
max_num_blocks = spec.max_num_blocks_per_req(
|
||||
@@ -598,12 +597,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
)
|
||||
# Preserve each cache type's alignment requirements after applying
|
||||
# its topology-aware block-table width.
|
||||
if isinstance(layer_spec, (MambaSpec, CircularBufferSpec)):
|
||||
max_num_blocks = get_block_table_width(
|
||||
max_num_blocks, spec.block_size, token_alignment=None
|
||||
)
|
||||
else:
|
||||
max_num_blocks = get_block_table_width(max_num_blocks, spec.block_size)
|
||||
max_num_blocks = get_block_table_width(
|
||||
max_num_blocks,
|
||||
spec.block_size,
|
||||
token_alignment=spec.block_table_token_alignment,
|
||||
)
|
||||
max_num_blocks_per_group.append(max_num_blocks)
|
||||
|
||||
target_attn_layer_names = None
|
||||
@@ -722,18 +720,20 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# to its own attention support.
|
||||
self.speculator.init_cudagraph_manager(cudagraph_mode)
|
||||
|
||||
self.kv_caches: list[torch.Tensor] = []
|
||||
# Capture warmup providers that depend on allocated KV-cache strides.
|
||||
with self.jit_warmup_registry.activate():
|
||||
kv_caches_dict = init_kv_cache(
|
||||
self.kv_caches,
|
||||
self.compilation_config.static_forward_context,
|
||||
self.kv_cache_config,
|
||||
self.device,
|
||||
self.kernel_block_sizes,
|
||||
self.vllm_config,
|
||||
kv_cache_allocation_context=kv_cache_allocation_context,
|
||||
block_tables=self.block_tables,
|
||||
)
|
||||
self.kv_caches = [
|
||||
cache for cache in kv_caches_dict.values() if cache.device == self.device
|
||||
]
|
||||
if is_profiling:
|
||||
self.kv_connector = NO_OP_KV_CONNECTOR
|
||||
else:
|
||||
@@ -1014,6 +1014,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
):
|
||||
self._dummy_run(**batch)
|
||||
self.adaptive_verification.set_initial_cost_curves(timings)
|
||||
self.kv_connector.reset_capture_state()
|
||||
|
||||
end_free_gpu_memory = torch.accelerator.get_memory_info()[0]
|
||||
|
||||
@@ -1141,13 +1142,17 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
def update_requests(self, scheduler_output: SchedulerOutput) -> None:
|
||||
# Add new blocks and update num_computed_tokens for the existing requests.
|
||||
reqs = scheduler_output.scheduled_cached_reqs
|
||||
table_updates = scheduler_output.block_table_updates or {}
|
||||
for req_id, block_ids in table_updates.items():
|
||||
if (idx := self.req_states.req_id_to_index.get(req_id)) is not None:
|
||||
self.block_tables.append_block_ids(idx, block_ids, overwrite=True)
|
||||
num_computed_tokens_np = self.req_states.num_computed_tokens_np
|
||||
for req_id, num_computed_tokens, req_new_block_ids in zip(
|
||||
reqs.req_ids, reqs.num_computed_tokens, reqs.new_block_ids
|
||||
):
|
||||
req_index = self.req_states.req_id_to_index[req_id]
|
||||
num_computed_tokens_np[req_index] = num_computed_tokens
|
||||
if req_new_block_ids is not None:
|
||||
if req_new_block_ids is not None and req_id not in table_updates:
|
||||
self.block_tables.append_block_ids(
|
||||
req_index, req_new_block_ids, overwrite=False
|
||||
)
|
||||
@@ -1841,13 +1846,22 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
)
|
||||
self.step_timing.forward_start()
|
||||
|
||||
connector_kwargs = dict(
|
||||
scheduler_output=scheduler_output,
|
||||
request_state_indices=input_batch.idx_mapping,
|
||||
request_ids=input_batch.req_ids,
|
||||
num_tokens=input_batch.num_tokens,
|
||||
)
|
||||
|
||||
# Run model.
|
||||
if batch_desc.cg_mode == CUDAGraphMode.FULL:
|
||||
# Use explicit cudagraph replay for FULL mode.
|
||||
# NOTE(woosuk): Here, we don't need to pass the input tensors,
|
||||
# because they are already copied to the CUDA graph input buffers.
|
||||
assert self.cudagraph_manager is not None
|
||||
self.kv_connector.pre_forward(scheduler_output)
|
||||
self.kv_connector.pre_forward(
|
||||
**connector_kwargs, attn_metadata=attn_metadata
|
||||
)
|
||||
model_output = self.cudagraph_manager.run_fullgraph(batch_desc)
|
||||
else:
|
||||
# For piecewise and eager mode, just call model().
|
||||
@@ -1871,7 +1885,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
skip_compiled=skip_compiled,
|
||||
is_padding=input_batch.is_padding,
|
||||
):
|
||||
self.kv_connector.pre_forward(scheduler_output)
|
||||
self.kv_connector.pre_forward(**connector_kwargs)
|
||||
if ubatch_state is not None:
|
||||
assert self.ubatch_runner is not None
|
||||
model_output = self.ubatch_runner.run(
|
||||
@@ -1889,6 +1903,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# Eager (NONE): call the raw model directly.
|
||||
model_output = self.model(**model_inputs)
|
||||
|
||||
self.kv_connector.finish_forward()
|
||||
|
||||
if self.is_last_pp_rank:
|
||||
if self.use_aux_hidden_state_outputs:
|
||||
assert isinstance(model_output, tuple)
|
||||
|
||||
Reference in New Issue
Block a user