[Feature][Frontend] Expose effective attention block size for DCP (#56538)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Signed-off-by: Julien Darve <jdarve@NVIDIA.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Bugen Zhao
OpenAI Codex
parent
0d8173d153
commit
f37c550bf6
@@ -566,6 +566,7 @@ class MPClient(EngineCoreClient):
|
||||
):
|
||||
self.vllm_config = vllm_config
|
||||
self._renderer: BaseRenderer | None = renderer
|
||||
self._effective_attention_block_sizes: set[int | None] = set()
|
||||
|
||||
# ZMQ setup.
|
||||
sync_ctx = zmq.Context(io_threads=2)
|
||||
@@ -803,10 +804,21 @@ class MPClient(EngineCoreClient):
|
||||
def _apply_ready_response(self, payload: bytes) -> None:
|
||||
"""Decode an EngineCoreReadyResponse and sync any post-initialization
|
||||
config changes (e.g. auto-fitted max_model_len) back to the frontend."""
|
||||
if not payload:
|
||||
return
|
||||
vllm_config = self.vllm_config
|
||||
response = msgspec.msgpack.decode(payload, type=EngineCoreReadyResponse)
|
||||
response = (
|
||||
msgspec.msgpack.decode(payload, type=EngineCoreReadyResponse)
|
||||
if payload
|
||||
else None
|
||||
)
|
||||
self._effective_attention_block_sizes.add(
|
||||
response.effective_attention_block_size if response is not None else None
|
||||
)
|
||||
sizes = self._effective_attention_block_sizes
|
||||
vllm_config.cache_config.effective_attention_block_size = (
|
||||
next(iter(sizes)) if len(sizes) == 1 else None
|
||||
)
|
||||
if response is None:
|
||||
return
|
||||
vllm_config.model_config.max_model_len = min(
|
||||
vllm_config.model_config.max_model_len, response.max_model_len
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user