[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com> Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Lucas Wilkinson
Lucas Wilkinson
Nicolò Lucchesi
OpenAI Codex
Claude Opus 5
Lucas Wilkinson
parent
120ec4ebd2
commit
d43bb2f37f
@@ -18,6 +18,10 @@ from vllm.v1.attention.backend import (
|
||||
CommonAttentionMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import create_fast_prefill_custom_backend
|
||||
from vllm.v1.hisparse.binding import (
|
||||
init_hisparse_kv_cache,
|
||||
resolve_hisparse_block_size,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
KVCacheConfig,
|
||||
@@ -30,11 +34,12 @@ from vllm.v1.worker.utils import (
|
||||
AttentionGroup,
|
||||
add_kv_sharing_layers_to_kv_cache_groups,
|
||||
allocate_kv_cache,
|
||||
bind_kv_cache,
|
||||
bind_kv_cache_to_layers,
|
||||
prepare_kernel_block_sizes,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
BatchExecutionDescriptor,
|
||||
CudaGraphManager,
|
||||
@@ -132,6 +137,7 @@ def get_kv_cache_spec(vllm_config: VllmConfig) -> dict[str, KVCacheSpec]:
|
||||
if isinstance(spec, AttentionSpec):
|
||||
spec = attn_module.get_attn_backend().customize_spec(spec)
|
||||
kv_cache_spec[layer_name] = spec
|
||||
resolve_hisparse_block_size(vllm_config, kv_cache_spec, attn_layers)
|
||||
return kv_cache_spec
|
||||
|
||||
|
||||
@@ -198,6 +204,9 @@ def init_attn_backend(
|
||||
kv_cache_config.kv_cache_groups
|
||||
):
|
||||
layer_names = kv_cache_group_spec.layer_names
|
||||
if not kv_cache_group_spec.kv_cache_spec.has_layer_views:
|
||||
attn_groups.append([])
|
||||
continue
|
||||
if active_layer_names is not None:
|
||||
layer_names = list(active_layer_names.intersection(layer_names))
|
||||
|
||||
@@ -316,22 +325,34 @@ def get_query_lens_mismatch_unsupported_backend(
|
||||
|
||||
|
||||
def init_kv_cache(
|
||||
runner_kv_caches: list[torch.Tensor | list[torch.Tensor]],
|
||||
forward_context: dict[str, Any],
|
||||
kv_cache_config: KVCacheConfig,
|
||||
device: torch.device,
|
||||
kernel_block_sizes: list[int],
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_allocation_context: AbstractContextManager | None = None,
|
||||
*,
|
||||
block_tables: "BlockTables | None" = None,
|
||||
) -> dict[str, Any]:
|
||||
allocation_context = kv_cache_allocation_context or nullcontext()
|
||||
with allocation_context:
|
||||
kv_caches = allocate_kv_cache(
|
||||
kv_cache_config,
|
||||
device,
|
||||
vllm_config.cache_config.get_resolved_kv_cache_layout(),
|
||||
kernel_block_sizes,
|
||||
)
|
||||
if vllm_config.attention_config.hisparse_config is not None:
|
||||
assert block_tables is not None
|
||||
kv_caches = init_hisparse_kv_cache(
|
||||
kv_cache_config,
|
||||
device,
|
||||
kernel_block_sizes,
|
||||
vllm_config,
|
||||
forward_context,
|
||||
block_tables,
|
||||
)
|
||||
else:
|
||||
kv_caches = allocate_kv_cache(
|
||||
kv_cache_config,
|
||||
device,
|
||||
vllm_config.cache_config.get_resolved_kv_cache_layout(),
|
||||
kernel_block_sizes,
|
||||
)
|
||||
for layer_name, target in get_shared_kv_cache_layers(vllm_config).items():
|
||||
kv_caches[layer_name] = kv_caches[target]
|
||||
# Dual-attention models (e.g. LongCat-Flash) put two Attention modules per
|
||||
@@ -342,10 +363,12 @@ def init_kv_cache(
|
||||
in ("longcat_flash", "longcat_flash_ngram")
|
||||
else 1
|
||||
)
|
||||
bind_kv_cache(
|
||||
kv_caches,
|
||||
bindable_caches = {
|
||||
name: cache for name, cache in kv_caches.items() if name in forward_context
|
||||
}
|
||||
bind_kv_cache_to_layers(
|
||||
bindable_caches,
|
||||
forward_context,
|
||||
runner_kv_caches,
|
||||
num_attn_module,
|
||||
kv_cache_groups=kv_cache_config.kv_cache_groups,
|
||||
)
|
||||
@@ -396,6 +419,8 @@ def build_attn_metadata(
|
||||
attn_metadata: dict[str, Any] = {}
|
||||
num_kv_cache_groups = len(kv_cache_config.kv_cache_groups)
|
||||
for i in range(num_kv_cache_groups):
|
||||
if not attn_groups[i]:
|
||||
continue
|
||||
block_table = block_tables[i]
|
||||
slot_mapping = slot_mappings[i]
|
||||
# Per-group causal for hybrid drafters (mixed SWA/full attention).
|
||||
|
||||
@@ -34,6 +34,7 @@ from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.utils.torch_utils import current_stream
|
||||
from vllm.v1.hisparse.binding import release_hisparse_profiling_cache
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup
|
||||
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
|
||||
@@ -931,6 +932,7 @@ def _teardown_profiling_state(runner: "GPUModelRunner") -> None:
|
||||
if (model := getattr(runner, "model", None)) is not None:
|
||||
layers = itertools.chain(layers, model.modules())
|
||||
clear_layer_kv_caches(layers)
|
||||
release_hisparse_profiling_cache(runner.compilation_config.static_forward_context)
|
||||
runner.cache_config.num_gpu_blocks = None
|
||||
runner.maybe_remove_all_loras(runner.lora_config)
|
||||
gc.collect()
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
@@ -29,7 +29,10 @@ if TYPE_CHECKING:
|
||||
class KVConnector:
|
||||
"""KVConnector interface used by GPUModelRunner."""
|
||||
|
||||
def pre_forward(self, scheduler_output: "SchedulerOutput") -> None:
|
||||
def pre_forward(self, scheduler_output: "SchedulerOutput", **kwargs: Any) -> None:
|
||||
pass
|
||||
|
||||
def finish_forward(self) -> None:
|
||||
pass
|
||||
|
||||
def post_forward(
|
||||
@@ -43,6 +46,9 @@ class KVConnector:
|
||||
def set_disabled(self, disabled: bool) -> None:
|
||||
pass
|
||||
|
||||
def reset_capture_state(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
class ActiveKVConnector(KVConnector):
|
||||
def __init__(
|
||||
@@ -54,10 +60,10 @@ class ActiveKVConnector(KVConnector):
|
||||
self.kv_connector.register_kv_caches(kv_caches_dict)
|
||||
self.kv_connector.set_host_xfer_buffer_ops(copy_kv_blocks)
|
||||
|
||||
self._pending_load_start = False
|
||||
self._pending_load_kwargs: dict[str, Any] | None = None
|
||||
self._disabled = False
|
||||
|
||||
def pre_forward(self, scheduler_output: "SchedulerOutput") -> None:
|
||||
def pre_forward(self, scheduler_output: "SchedulerOutput", **kwargs: Any) -> None:
|
||||
if self._disabled:
|
||||
return
|
||||
|
||||
@@ -65,23 +71,31 @@ class ActiveKVConnector(KVConnector):
|
||||
assert kv_connector_metadata is not None
|
||||
self.kv_connector.handle_preemptions(kv_connector_metadata)
|
||||
self.kv_connector.bind_connector_metadata(kv_connector_metadata)
|
||||
self._pending_load_kwargs = kwargs
|
||||
|
||||
if scheduler_output.has_sync_kv_loads:
|
||||
# Sync loads need to run before this step's forward.
|
||||
self._start_load_kv()
|
||||
else:
|
||||
# Start any async loads in post-forward instead, keeping
|
||||
# their host-side submission cost off the critical path.
|
||||
self._pending_load_start = True
|
||||
# Otherwise start the async load after forward to keep submission
|
||||
# off the critical path.
|
||||
|
||||
def _start_load_kv(self) -> None:
|
||||
self._pending_load_start = False
|
||||
load_kwargs = self._pending_load_kwargs
|
||||
assert load_kwargs is not None
|
||||
self._pending_load_kwargs = None
|
||||
# TODO: sort out KV Connectors' use of forward_context
|
||||
if is_forward_context_available():
|
||||
self.kv_connector.start_load_kv(get_forward_context())
|
||||
self.kv_connector.start_load_kv(get_forward_context(), **load_kwargs)
|
||||
else:
|
||||
with set_forward_context(None, self.vllm_config):
|
||||
self.kv_connector.start_load_kv(get_forward_context())
|
||||
self.kv_connector.start_load_kv(get_forward_context(), **load_kwargs)
|
||||
|
||||
def finish_forward(self) -> None:
|
||||
if not self._disabled:
|
||||
self.kv_connector.finish_forward()
|
||||
|
||||
def reset_capture_state(self) -> None:
|
||||
self.kv_connector.reset_capture_state()
|
||||
|
||||
def post_forward(
|
||||
self, finished_req_ids: set[str], wait_for_save: bool = True
|
||||
@@ -89,15 +103,16 @@ class ActiveKVConnector(KVConnector):
|
||||
if self._disabled:
|
||||
return None
|
||||
|
||||
if self._pending_load_start:
|
||||
if self._pending_load_kwargs is not None:
|
||||
self._start_load_kv()
|
||||
|
||||
output = KVConnectorOutput()
|
||||
if wait_for_save:
|
||||
self.kv_connector.wait_for_save()
|
||||
output.finished_sending, output.finished_recving = (
|
||||
self.kv_connector.get_finished(finished_req_ids)
|
||||
)
|
||||
transfer_results = self.kv_connector.get_transfer_results(finished_req_ids)
|
||||
output.finished_sending = transfer_results.finished_sending or None
|
||||
output.finished_recving = transfer_results.finished_recving or None
|
||||
output.failed_recving = transfer_results.failed_recving
|
||||
output.invalid_block_ids = self.kv_connector.get_block_ids_with_load_errors()
|
||||
output.kv_connector_stats = self.kv_connector.get_kv_connector_stats()
|
||||
output.kv_cache_events = self.kv_connector.get_kv_connector_kv_cache_events()
|
||||
@@ -112,6 +127,7 @@ class ActiveKVConnector(KVConnector):
|
||||
return EMPTY_MODEL_RUNNER_OUTPUT
|
||||
|
||||
self.pre_forward(scheduler_output)
|
||||
self.finish_forward()
|
||||
finished_req_ids = scheduler_output.finished_req_ids
|
||||
kv_connector_output = self.post_forward(finished_req_ids, wait_for_save=False)
|
||||
return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
|
||||
|
||||
@@ -67,7 +67,6 @@ from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
|
||||
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
|
||||
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
KVCacheConfig,
|
||||
MambaSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
@@ -590,7 +589,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
layer_spec = (
|
||||
spec.first_spec if isinstance(spec, UniformTypeKVCacheSpecs) else spec
|
||||
)
|
||||
slot_mapping_enabled.append(not isinstance(layer_spec, CircularBufferSpec))
|
||||
slot_mapping_enabled.append(layer_spec.uses_slot_mapping)
|
||||
# Let each cache type account for CP. Attention KV is DCP-sharded,
|
||||
# while Mamba/GDN recurrent state is replicated across DCP ranks.
|
||||
max_num_blocks = spec.max_num_blocks_per_req(
|
||||
@@ -598,12 +597,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
)
|
||||
# Preserve each cache type's alignment requirements after applying
|
||||
# its topology-aware block-table width.
|
||||
if isinstance(layer_spec, (MambaSpec, CircularBufferSpec)):
|
||||
max_num_blocks = get_block_table_width(
|
||||
max_num_blocks, spec.block_size, token_alignment=None
|
||||
)
|
||||
else:
|
||||
max_num_blocks = get_block_table_width(max_num_blocks, spec.block_size)
|
||||
max_num_blocks = get_block_table_width(
|
||||
max_num_blocks,
|
||||
spec.block_size,
|
||||
token_alignment=spec.block_table_token_alignment,
|
||||
)
|
||||
max_num_blocks_per_group.append(max_num_blocks)
|
||||
|
||||
target_attn_layer_names = None
|
||||
@@ -722,18 +720,20 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# to its own attention support.
|
||||
self.speculator.init_cudagraph_manager(cudagraph_mode)
|
||||
|
||||
self.kv_caches: list[torch.Tensor] = []
|
||||
# Capture warmup providers that depend on allocated KV-cache strides.
|
||||
with self.jit_warmup_registry.activate():
|
||||
kv_caches_dict = init_kv_cache(
|
||||
self.kv_caches,
|
||||
self.compilation_config.static_forward_context,
|
||||
self.kv_cache_config,
|
||||
self.device,
|
||||
self.kernel_block_sizes,
|
||||
self.vllm_config,
|
||||
kv_cache_allocation_context=kv_cache_allocation_context,
|
||||
block_tables=self.block_tables,
|
||||
)
|
||||
self.kv_caches = [
|
||||
cache for cache in kv_caches_dict.values() if cache.device == self.device
|
||||
]
|
||||
if is_profiling:
|
||||
self.kv_connector = NO_OP_KV_CONNECTOR
|
||||
else:
|
||||
@@ -1014,6 +1014,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
):
|
||||
self._dummy_run(**batch)
|
||||
self.adaptive_verification.set_initial_cost_curves(timings)
|
||||
self.kv_connector.reset_capture_state()
|
||||
|
||||
end_free_gpu_memory = torch.accelerator.get_memory_info()[0]
|
||||
|
||||
@@ -1141,13 +1142,17 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
def update_requests(self, scheduler_output: SchedulerOutput) -> None:
|
||||
# Add new blocks and update num_computed_tokens for the existing requests.
|
||||
reqs = scheduler_output.scheduled_cached_reqs
|
||||
table_updates = scheduler_output.block_table_updates or {}
|
||||
for req_id, block_ids in table_updates.items():
|
||||
if (idx := self.req_states.req_id_to_index.get(req_id)) is not None:
|
||||
self.block_tables.append_block_ids(idx, block_ids, overwrite=True)
|
||||
num_computed_tokens_np = self.req_states.num_computed_tokens_np
|
||||
for req_id, num_computed_tokens, req_new_block_ids in zip(
|
||||
reqs.req_ids, reqs.num_computed_tokens, reqs.new_block_ids
|
||||
):
|
||||
req_index = self.req_states.req_id_to_index[req_id]
|
||||
num_computed_tokens_np[req_index] = num_computed_tokens
|
||||
if req_new_block_ids is not None:
|
||||
if req_new_block_ids is not None and req_id not in table_updates:
|
||||
self.block_tables.append_block_ids(
|
||||
req_index, req_new_block_ids, overwrite=False
|
||||
)
|
||||
@@ -1841,13 +1846,22 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
)
|
||||
self.step_timing.forward_start()
|
||||
|
||||
connector_kwargs = dict(
|
||||
scheduler_output=scheduler_output,
|
||||
request_state_indices=input_batch.idx_mapping,
|
||||
request_ids=input_batch.req_ids,
|
||||
num_tokens=input_batch.num_tokens,
|
||||
)
|
||||
|
||||
# Run model.
|
||||
if batch_desc.cg_mode == CUDAGraphMode.FULL:
|
||||
# Use explicit cudagraph replay for FULL mode.
|
||||
# NOTE(woosuk): Here, we don't need to pass the input tensors,
|
||||
# because they are already copied to the CUDA graph input buffers.
|
||||
assert self.cudagraph_manager is not None
|
||||
self.kv_connector.pre_forward(scheduler_output)
|
||||
self.kv_connector.pre_forward(
|
||||
**connector_kwargs, attn_metadata=attn_metadata
|
||||
)
|
||||
model_output = self.cudagraph_manager.run_fullgraph(batch_desc)
|
||||
else:
|
||||
# For piecewise and eager mode, just call model().
|
||||
@@ -1871,7 +1885,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
skip_compiled=skip_compiled,
|
||||
is_padding=input_batch.is_padding,
|
||||
):
|
||||
self.kv_connector.pre_forward(scheduler_output)
|
||||
self.kv_connector.pre_forward(**connector_kwargs)
|
||||
if ubatch_state is not None:
|
||||
assert self.ubatch_runner is not None
|
||||
model_output = self.ubatch_runner.run(
|
||||
@@ -1889,6 +1903,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# Eager (NONE): call the raw model directly.
|
||||
model_output = self.model(**model_inputs)
|
||||
|
||||
self.kv_connector.finish_forward()
|
||||
|
||||
if self.is_last_pp_rank:
|
||||
if self.use_aux_hidden_state_outputs:
|
||||
assert isinstance(model_output, tuple)
|
||||
|
||||
@@ -224,10 +224,20 @@ def warmup_kernels(
|
||||
# fixed draft counts here, then restore the manager for capture and serving.
|
||||
adaptive_verification = model_runner.adaptive_verification
|
||||
model_runner.adaptive_verification = None
|
||||
rejection_sampler = model_runner.rejection_sampler
|
||||
adaptive_sampling = (
|
||||
rejection_sampler is not None and rejection_sampler.enable_adaptive_verification
|
||||
)
|
||||
if adaptive_sampling:
|
||||
assert rejection_sampler is not None
|
||||
rejection_sampler.enable_adaptive_verification = False
|
||||
try:
|
||||
_warmup_kernels(model_runner, worker_execute_model, worker_sample_tokens)
|
||||
finally:
|
||||
model_runner.adaptive_verification = adaptive_verification
|
||||
if adaptive_sampling:
|
||||
assert rejection_sampler is not None
|
||||
rejection_sampler.enable_adaptive_verification = True
|
||||
|
||||
|
||||
def _warmup_kernels(
|
||||
|
||||
Reference in New Issue
Block a user