Merge branch 'main' into patches
This commit is contained in:
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from contextlib import AbstractContextManager, nullcontext
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import numpy as np
|
||||
@@ -148,6 +148,8 @@ def get_kv_cache_spec(vllm_config: VllmConfig) -> dict[str, KVCacheSpec]:
|
||||
if spec := attn_module.get_kv_cache_spec(vllm_config):
|
||||
if isinstance(spec, AttentionSpec):
|
||||
spec = customize_attention_spec(attn_module.get_attn_backend(), spec)
|
||||
if attn_module.is_draft_layer:
|
||||
spec = replace(spec, is_draft=True)
|
||||
kv_cache_spec[layer_name] = spec
|
||||
if vllm_config.attention_config.hisparse_config is not None:
|
||||
kv_cache_spec = resolve_hisparse_specs(vllm_config, kv_cache_spec, attn_layers)
|
||||
|
||||
@@ -74,6 +74,7 @@ class BlockTables:
|
||||
self.input_block_tables: list[torch.Tensor] = [
|
||||
torch.zeros_like(b.gpu) for b in self.block_tables
|
||||
]
|
||||
self._num_dirty_rows = 0
|
||||
|
||||
self.slot_mappings = torch.zeros(
|
||||
self.num_kv_cache_groups,
|
||||
@@ -151,15 +152,18 @@ class BlockTables:
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
if self.num_kv_cache_groups == 0:
|
||||
return ()
|
||||
num_reqs = idx_mapping.shape[0]
|
||||
num_rows = num_reqs_padded
|
||||
if out is None:
|
||||
out = tuple(self.input_block_tables)
|
||||
out_ptrs = self.input_block_table_ptrs
|
||||
num_rows = max(num_rows, self._num_dirty_rows)
|
||||
self._num_dirty_rows = num_reqs
|
||||
else:
|
||||
assert out_ptrs is not None
|
||||
assert len(out) == self.num_kv_cache_groups
|
||||
num_reqs = idx_mapping.shape[0]
|
||||
# Launch kernel with num_reqs_padded to fuse zeroing of padded rows.
|
||||
_gather_block_tables_kernel[(self.num_kv_cache_groups, num_reqs_padded)](
|
||||
# Also clear rows left by a larger batch in the shared buffer.
|
||||
_gather_block_tables_kernel[(self.num_kv_cache_groups, num_rows)](
|
||||
idx_mapping,
|
||||
self.block_table_ptrs,
|
||||
out_ptrs,
|
||||
@@ -180,9 +184,12 @@ class BlockTables:
|
||||
# Zero the rows so dummy runs write mamba state to the reserved null
|
||||
# block rather than through the previous real step's (stale) block
|
||||
# ids, which may point at blocks since freed and reallocated.
|
||||
return tuple(
|
||||
block_table[:num_reqs].zero_() for block_table in self.input_block_tables
|
||||
)
|
||||
num_rows = max(num_reqs, self._num_dirty_rows)
|
||||
for block_table in self.input_block_tables:
|
||||
block_table[:num_rows].zero_()
|
||||
# Profiling may populate the returned rows with valid block IDs.
|
||||
self._num_dirty_rows = num_reqs
|
||||
return tuple(block_table[:num_reqs] for block_table in self.input_block_tables)
|
||||
|
||||
def compute_slot_mappings(
|
||||
self,
|
||||
|
||||
@@ -41,9 +41,14 @@ class KVConnector:
|
||||
def finish_forward(self) -> None:
|
||||
pass
|
||||
|
||||
def post_forward(self, finished_req_ids: set[str]) -> KVConnectorOutput | None:
|
||||
def post_forward(
|
||||
self, finished_req_ids: set[str], defer_finalize: bool = False
|
||||
) -> KVConnectorOutput | None:
|
||||
return None
|
||||
|
||||
def finalize_step(self) -> None:
|
||||
pass
|
||||
|
||||
def no_forward(self, scheduler_output: "SchedulerOutput") -> ModelRunnerOutput:
|
||||
return EMPTY_MODEL_RUNNER_OUTPUT
|
||||
|
||||
@@ -128,7 +133,9 @@ class ActiveKVConnector(KVConnector):
|
||||
def reset_capture_state(self) -> None:
|
||||
self.kv_connector.reset_capture_state()
|
||||
|
||||
def post_forward(self, finished_req_ids: set[str]) -> KVConnectorOutput | None:
|
||||
def post_forward(
|
||||
self, finished_req_ids: set[str], defer_finalize: bool = False
|
||||
) -> KVConnectorOutput | None:
|
||||
if self._disabled:
|
||||
return None
|
||||
|
||||
@@ -136,7 +143,8 @@ class ActiveKVConnector(KVConnector):
|
||||
self._start_load_kv()
|
||||
|
||||
output = KVConnectorOutput()
|
||||
self.kv_connector.wait_for_save()
|
||||
if not defer_finalize:
|
||||
self.kv_connector.wait_for_save()
|
||||
transfer_results = self.kv_connector.get_transfer_results(finished_req_ids)
|
||||
output.finished_sending = transfer_results.finished_sending or None
|
||||
output.finished_recving = transfer_results.finished_recving or None
|
||||
@@ -147,9 +155,16 @@ class ActiveKVConnector(KVConnector):
|
||||
output.kv_connector_worker_meta = (
|
||||
self.kv_connector.build_connector_worker_meta()
|
||||
)
|
||||
self.kv_connector.clear_connector_metadata()
|
||||
if not defer_finalize:
|
||||
self.kv_connector.clear_connector_metadata()
|
||||
return output
|
||||
|
||||
def finalize_step(self) -> None:
|
||||
if self._disabled:
|
||||
return
|
||||
self.kv_connector.wait_for_save()
|
||||
self.kv_connector.clear_connector_metadata()
|
||||
|
||||
def no_forward(self, scheduler_output: "SchedulerOutput") -> ModelRunnerOutput:
|
||||
if self._disabled:
|
||||
return EMPTY_MODEL_RUNNER_OUTPUT
|
||||
|
||||
@@ -2207,6 +2207,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
pending_aux_output=pending_aux_output,
|
||||
)
|
||||
|
||||
# Collect KV transfer results now; finalize saves after the drafter runs.
|
||||
defer_kv_finalize = self.speculator is not None
|
||||
model_runner_output.kv_connector_output = self.kv_connector.post_forward(
|
||||
finished_req_ids, defer_finalize=defer_kv_finalize
|
||||
)
|
||||
|
||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None
|
||||
# The encoder runner exists only on the first PP rank, so later ranks
|
||||
# have no cached embeddings to gather.
|
||||
@@ -2288,9 +2294,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.req_states.draft_tokens, input_batch
|
||||
)
|
||||
|
||||
# Post-step KV connector related operations.
|
||||
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
|
||||
model_runner_output.kv_connector_output = kv_connector_output
|
||||
if defer_kv_finalize:
|
||||
self.kv_connector.finalize_step()
|
||||
model_runner_output.ec_connector_output = ec_connector_output
|
||||
|
||||
return async_output
|
||||
|
||||
@@ -238,7 +238,8 @@ def warmup_kernels(
|
||||
rejection_sampler.enable_adaptive_verification = False
|
||||
try:
|
||||
_warmup_kernels(model_runner, worker_execute_model, worker_sample_tokens)
|
||||
if rejection_sampler is not None:
|
||||
# All PP ranks must run the same forwards to match sends and receives.
|
||||
if model_runner.speculative_config is not None:
|
||||
# Greedy batches skip logits processing, so rejection kernels
|
||||
# also see model-dtype logits instead of FP32.
|
||||
_warmup_kernels(
|
||||
|
||||
Reference in New Issue
Block a user