Merge branch 'main' into patches

This commit is contained in:
2026-10-18 16:04:36 +07:00
299 changed files with 51483 additions and 2195 deletions
+3 -1
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable, Mapping, Sequence
from contextlib import AbstractContextManager, nullcontext
from dataclasses import dataclass
from dataclasses import dataclass, replace
from typing import TYPE_CHECKING, Any, cast
import numpy as np
@@ -148,6 +148,8 @@ def get_kv_cache_spec(vllm_config: VllmConfig) -> dict[str, KVCacheSpec]:
if spec := attn_module.get_kv_cache_spec(vllm_config):
if isinstance(spec, AttentionSpec):
spec = customize_attention_spec(attn_module.get_attn_backend(), spec)
if attn_module.is_draft_layer:
spec = replace(spec, is_draft=True)
kv_cache_spec[layer_name] = spec
if vllm_config.attention_config.hisparse_config is not None:
kv_cache_spec = resolve_hisparse_specs(vllm_config, kv_cache_spec, attn_layers)
+13 -6
View File
@@ -74,6 +74,7 @@ class BlockTables:
self.input_block_tables: list[torch.Tensor] = [
torch.zeros_like(b.gpu) for b in self.block_tables
]
self._num_dirty_rows = 0
self.slot_mappings = torch.zeros(
self.num_kv_cache_groups,
@@ -151,15 +152,18 @@ class BlockTables:
) -> tuple[torch.Tensor, ...]:
if self.num_kv_cache_groups == 0:
return ()
num_reqs = idx_mapping.shape[0]
num_rows = num_reqs_padded
if out is None:
out = tuple(self.input_block_tables)
out_ptrs = self.input_block_table_ptrs
num_rows = max(num_rows, self._num_dirty_rows)
self._num_dirty_rows = num_reqs
else:
assert out_ptrs is not None
assert len(out) == self.num_kv_cache_groups
num_reqs = idx_mapping.shape[0]
# Launch kernel with num_reqs_padded to fuse zeroing of padded rows.
_gather_block_tables_kernel[(self.num_kv_cache_groups, num_reqs_padded)](
# Also clear rows left by a larger batch in the shared buffer.
_gather_block_tables_kernel[(self.num_kv_cache_groups, num_rows)](
idx_mapping,
self.block_table_ptrs,
out_ptrs,
@@ -180,9 +184,12 @@ class BlockTables:
# Zero the rows so dummy runs write mamba state to the reserved null
# block rather than through the previous real step's (stale) block
# ids, which may point at blocks since freed and reallocated.
return tuple(
block_table[:num_reqs].zero_() for block_table in self.input_block_tables
)
num_rows = max(num_reqs, self._num_dirty_rows)
for block_table in self.input_block_tables:
block_table[:num_rows].zero_()
# Profiling may populate the returned rows with valid block IDs.
self._num_dirty_rows = num_reqs
return tuple(block_table[:num_reqs] for block_table in self.input_block_tables)
def compute_slot_mappings(
self,
+19 -4
View File
@@ -41,9 +41,14 @@ class KVConnector:
def finish_forward(self) -> None:
pass
def post_forward(self, finished_req_ids: set[str]) -> KVConnectorOutput | None:
def post_forward(
self, finished_req_ids: set[str], defer_finalize: bool = False
) -> KVConnectorOutput | None:
return None
def finalize_step(self) -> None:
pass
def no_forward(self, scheduler_output: "SchedulerOutput") -> ModelRunnerOutput:
return EMPTY_MODEL_RUNNER_OUTPUT
@@ -128,7 +133,9 @@ class ActiveKVConnector(KVConnector):
def reset_capture_state(self) -> None:
self.kv_connector.reset_capture_state()
def post_forward(self, finished_req_ids: set[str]) -> KVConnectorOutput | None:
def post_forward(
self, finished_req_ids: set[str], defer_finalize: bool = False
) -> KVConnectorOutput | None:
if self._disabled:
return None
@@ -136,7 +143,8 @@ class ActiveKVConnector(KVConnector):
self._start_load_kv()
output = KVConnectorOutput()
self.kv_connector.wait_for_save()
if not defer_finalize:
self.kv_connector.wait_for_save()
transfer_results = self.kv_connector.get_transfer_results(finished_req_ids)
output.finished_sending = transfer_results.finished_sending or None
output.finished_recving = transfer_results.finished_recving or None
@@ -147,9 +155,16 @@ class ActiveKVConnector(KVConnector):
output.kv_connector_worker_meta = (
self.kv_connector.build_connector_worker_meta()
)
self.kv_connector.clear_connector_metadata()
if not defer_finalize:
self.kv_connector.clear_connector_metadata()
return output
def finalize_step(self) -> None:
if self._disabled:
return
self.kv_connector.wait_for_save()
self.kv_connector.clear_connector_metadata()
def no_forward(self, scheduler_output: "SchedulerOutput") -> ModelRunnerOutput:
if self._disabled:
return EMPTY_MODEL_RUNNER_OUTPUT
+8 -3
View File
@@ -2207,6 +2207,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
pending_aux_output=pending_aux_output,
)
# Collect KV transfer results now; finalize saves after the drafter runs.
defer_kv_finalize = self.speculator is not None
model_runner_output.kv_connector_output = self.kv_connector.post_forward(
finished_req_ids, defer_finalize=defer_kv_finalize
)
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None
# The encoder runner exists only on the first PP rank, so later ranks
# have no cached embeddings to gather.
@@ -2288,9 +2294,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.req_states.draft_tokens, input_batch
)
# Post-step KV connector related operations.
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
model_runner_output.kv_connector_output = kv_connector_output
if defer_kv_finalize:
self.kv_connector.finalize_step()
model_runner_output.ec_connector_output = ec_connector_output
return async_output
+2 -1
View File
@@ -238,7 +238,8 @@ def warmup_kernels(
rejection_sampler.enable_adaptive_verification = False
try:
_warmup_kernels(model_runner, worker_execute_model, worker_sample_tokens)
if rejection_sampler is not None:
# All PP ranks must run the same forwards to match sends and receives.
if model_runner.speculative_config is not None:
# Greedy batches skip logits processing, so rejection kernels
# also see model-dtype logits instead of FP32.
_warmup_kernels(