[Feature][Frontend] Expose effective attention block size for DCP (#56538)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Signed-off-by: Julien Darve <jdarve@NVIDIA.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
JulienDarve
2026-09-16 08:29:43 +00:00
committed by GitHub
co-authored by Bugen Zhao OpenAI Codex
parent 0d8173d153
commit f37c550bf6
18 changed files with 159 additions and 11 deletions
+3
View File
@@ -211,6 +211,8 @@ class CacheConfig:
"""The number of blocks to allocate for CPU memory."""
# Set after KV cache initialization.
effective_attention_block_size: int | None = field(default=None, init=False)
"""Full-attention block size in tokens, including DCP, or None if unavailable."""
kv_cache_size_tokens: int | None = field(default=None, init=False)
"""Per-DP-engine KV cache capacity in tokens (group-aware). Uses
group-aware capacity since num_gpu_blocks * block_size can be wrong
@@ -277,6 +279,7 @@ class CacheConfig:
# Post-init/derived counters
"num_gpu_blocks",
"num_cpu_blocks",
"effective_attention_block_size",
"kv_cache_size_tokens",
"kv_cache_max_concurrency",
# WIP feature toggle not impacting compiled graph shape
+2
View File
@@ -102,6 +102,8 @@ class EngineCoreReadyResponse:
weight_transfer_backend: str | None = None
enable_sleep_mode: bool = False
supports_draft_weight_updates: bool = False
# Full-attention block size in tokens after initialization, or unavailable.
effective_attention_block_size: int | None = None
class EngineCoreRequest(
+20 -1
View File
@@ -85,7 +85,7 @@ from vllm.v1.fault_tolerance.engine_core_sentinel import (
EngineCoreSentinel,
fault_tolerant_wrapper,
)
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.kv_cache_interface import KVCacheConfig, is_full_attention_spec
from vllm.v1.metrics.stats import SchedulerIterationDetails, SchedulerStats
from vllm.v1.outputs import ModelRunnerOutput
from vllm.v1.request import Request, RequestStatus
@@ -168,6 +168,7 @@ class EngineCore:
block_size=scheduler_block_size,
hash_block_size=hash_block_size,
)
self._initialize_effective_attention_block_size()
self.use_spec_decode = vllm_config.speculative_config is not None
self.check_for_draft_tokens = (
self.use_spec_decode or vllm_config.model_config.is_diffusion
@@ -382,6 +383,21 @@ class EngineCore:
)
return scheduler_kv_cache_config
def _initialize_effective_attention_block_size(self) -> None:
cache_config = self.vllm_config.cache_config
cache_config.effective_attention_block_size = None
cache_manager = getattr(self.scheduler, "kv_cache_manager", None)
if cache_manager is None:
return
block_sizes = {
manager.block_size
for manager in cache_manager.coordinator.single_type_managers
if is_full_attention_spec(manager.kv_cache_spec)
}
cache_config.effective_attention_block_size = (
block_sizes.pop() if len(block_sizes) == 1 else None
)
def get_supported_tasks(self) -> tuple[SupportedTask, ...]:
supported_tasks = self.model_executor.supported_tasks
self._log_pooler_config(supported_tasks)
@@ -1649,6 +1665,9 @@ class EngineCoreProc(EngineCore):
num_gpu_blocks=self.vllm_config.cache_config.num_gpu_blocks or 0,
block_size=self.vllm_config.cache_config.block_size,
mamba_block_size=self.vllm_config.cache_config.mamba_block_size,
effective_attention_block_size=(
self.vllm_config.cache_config.effective_attention_block_size
),
dp_stats_address=self.frontend_stats_publish_address,
dtype=str(self.vllm_config.model_config.dtype).removeprefix("torch."),
vllm_version=VLLM_VERSION,
+15 -3
View File
@@ -566,6 +566,7 @@ class MPClient(EngineCoreClient):
):
self.vllm_config = vllm_config
self._renderer: BaseRenderer | None = renderer
self._effective_attention_block_sizes: set[int | None] = set()
# ZMQ setup.
sync_ctx = zmq.Context(io_threads=2)
@@ -803,10 +804,21 @@ class MPClient(EngineCoreClient):
def _apply_ready_response(self, payload: bytes) -> None:
"""Decode an EngineCoreReadyResponse and sync any post-initialization
config changes (e.g. auto-fitted max_model_len) back to the frontend."""
if not payload:
return
vllm_config = self.vllm_config
response = msgspec.msgpack.decode(payload, type=EngineCoreReadyResponse)
response = (
msgspec.msgpack.decode(payload, type=EngineCoreReadyResponse)
if payload
else None
)
self._effective_attention_block_sizes.add(
response.effective_attention_block_size if response is not None else None
)
sizes = self._effective_attention_block_sizes
vllm_config.cache_config.effective_attention_block_size = (
next(iter(sizes)) if len(sizes) == 1 else None
)
if response is None:
return
vllm_config.model_config.max_model_len = min(
vllm_config.model_config.max_model_len, response.max_model_len
)