[Feature][Frontend] Expose effective attention block size for DCP (#56538)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Signed-off-by: Julien Darve <jdarve@NVIDIA.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Bugen Zhao
OpenAI Codex
parent
0d8173d153
commit
f37c550bf6
@@ -211,6 +211,8 @@ class CacheConfig:
|
||||
"""The number of blocks to allocate for CPU memory."""
|
||||
|
||||
# Set after KV cache initialization.
|
||||
effective_attention_block_size: int | None = field(default=None, init=False)
|
||||
"""Full-attention block size in tokens, including DCP, or None if unavailable."""
|
||||
kv_cache_size_tokens: int | None = field(default=None, init=False)
|
||||
"""Per-DP-engine KV cache capacity in tokens (group-aware). Uses
|
||||
group-aware capacity since num_gpu_blocks * block_size can be wrong
|
||||
@@ -277,6 +279,7 @@ class CacheConfig:
|
||||
# Post-init/derived counters
|
||||
"num_gpu_blocks",
|
||||
"num_cpu_blocks",
|
||||
"effective_attention_block_size",
|
||||
"kv_cache_size_tokens",
|
||||
"kv_cache_max_concurrency",
|
||||
# WIP feature toggle not impacting compiled graph shape
|
||||
|
||||
@@ -102,6 +102,8 @@ class EngineCoreReadyResponse:
|
||||
weight_transfer_backend: str | None = None
|
||||
enable_sleep_mode: bool = False
|
||||
supports_draft_weight_updates: bool = False
|
||||
# Full-attention block size in tokens after initialization, or unavailable.
|
||||
effective_attention_block_size: int | None = None
|
||||
|
||||
|
||||
class EngineCoreRequest(
|
||||
|
||||
+20
-1
@@ -85,7 +85,7 @@ from vllm.v1.fault_tolerance.engine_core_sentinel import (
|
||||
EngineCoreSentinel,
|
||||
fault_tolerant_wrapper,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, is_full_attention_spec
|
||||
from vllm.v1.metrics.stats import SchedulerIterationDetails, SchedulerStats
|
||||
from vllm.v1.outputs import ModelRunnerOutput
|
||||
from vllm.v1.request import Request, RequestStatus
|
||||
@@ -168,6 +168,7 @@ class EngineCore:
|
||||
block_size=scheduler_block_size,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
self._initialize_effective_attention_block_size()
|
||||
self.use_spec_decode = vllm_config.speculative_config is not None
|
||||
self.check_for_draft_tokens = (
|
||||
self.use_spec_decode or vllm_config.model_config.is_diffusion
|
||||
@@ -382,6 +383,21 @@ class EngineCore:
|
||||
)
|
||||
return scheduler_kv_cache_config
|
||||
|
||||
def _initialize_effective_attention_block_size(self) -> None:
|
||||
cache_config = self.vllm_config.cache_config
|
||||
cache_config.effective_attention_block_size = None
|
||||
cache_manager = getattr(self.scheduler, "kv_cache_manager", None)
|
||||
if cache_manager is None:
|
||||
return
|
||||
block_sizes = {
|
||||
manager.block_size
|
||||
for manager in cache_manager.coordinator.single_type_managers
|
||||
if is_full_attention_spec(manager.kv_cache_spec)
|
||||
}
|
||||
cache_config.effective_attention_block_size = (
|
||||
block_sizes.pop() if len(block_sizes) == 1 else None
|
||||
)
|
||||
|
||||
def get_supported_tasks(self) -> tuple[SupportedTask, ...]:
|
||||
supported_tasks = self.model_executor.supported_tasks
|
||||
self._log_pooler_config(supported_tasks)
|
||||
@@ -1649,6 +1665,9 @@ class EngineCoreProc(EngineCore):
|
||||
num_gpu_blocks=self.vllm_config.cache_config.num_gpu_blocks or 0,
|
||||
block_size=self.vllm_config.cache_config.block_size,
|
||||
mamba_block_size=self.vllm_config.cache_config.mamba_block_size,
|
||||
effective_attention_block_size=(
|
||||
self.vllm_config.cache_config.effective_attention_block_size
|
||||
),
|
||||
dp_stats_address=self.frontend_stats_publish_address,
|
||||
dtype=str(self.vllm_config.model_config.dtype).removeprefix("torch."),
|
||||
vllm_version=VLLM_VERSION,
|
||||
|
||||
@@ -566,6 +566,7 @@ class MPClient(EngineCoreClient):
|
||||
):
|
||||
self.vllm_config = vllm_config
|
||||
self._renderer: BaseRenderer | None = renderer
|
||||
self._effective_attention_block_sizes: set[int | None] = set()
|
||||
|
||||
# ZMQ setup.
|
||||
sync_ctx = zmq.Context(io_threads=2)
|
||||
@@ -803,10 +804,21 @@ class MPClient(EngineCoreClient):
|
||||
def _apply_ready_response(self, payload: bytes) -> None:
|
||||
"""Decode an EngineCoreReadyResponse and sync any post-initialization
|
||||
config changes (e.g. auto-fitted max_model_len) back to the frontend."""
|
||||
if not payload:
|
||||
return
|
||||
vllm_config = self.vllm_config
|
||||
response = msgspec.msgpack.decode(payload, type=EngineCoreReadyResponse)
|
||||
response = (
|
||||
msgspec.msgpack.decode(payload, type=EngineCoreReadyResponse)
|
||||
if payload
|
||||
else None
|
||||
)
|
||||
self._effective_attention_block_sizes.add(
|
||||
response.effective_attention_block_size if response is not None else None
|
||||
)
|
||||
sizes = self._effective_attention_block_sizes
|
||||
vllm_config.cache_config.effective_attention_block_size = (
|
||||
next(iter(sizes)) if len(sizes) == 1 else None
|
||||
)
|
||||
if response is None:
|
||||
return
|
||||
vllm_config.model_config.max_model_len = min(
|
||||
vllm_config.model_config.max_model_len, response.max_model_len
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user