[Mooncake] Fix mixed MLA+Eagle block-size validation (#39596)

Signed-off-by: Zhewen Li <zhewenli@inferact.ai>
Co-authored-by: Zhewen Li <zhewenli@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Zhewen Li
2026-04-15 11:36:47 -07:00
committed by GitHub
co-authored by Zhewen Li OpenAI Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 21e5a9f48e
commit 55e1a8e103
2 changed files with 65 additions and 3 deletions
@@ -23,6 +23,7 @@ from vllm.distributed.kv_transfer.kv_connector.utils import (
EngineId,
TpKVTopology,
get_current_attn_backend,
get_current_attn_backends,
)
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
@@ -47,6 +48,7 @@ from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.attention.backends.utils import get_kv_cache_layout
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.request import RequestStatus
from vllm.v1.worker.utils import select_common_block_size
logger = init_logger(__name__)
@@ -751,6 +753,7 @@ class MooncakeConnectorWorker:
self.model_config = vllm_config.model_config
self.cache_config = vllm_config.cache_config
self.use_mla = self.model_config.use_mla
self._sync_block_size_with_kernel()
# Get the attention backend from the first layer
# NOTE (NickLucche) models with multiple backends are not supported yet
@@ -777,6 +780,23 @@ class MooncakeConnectorWorker:
self._xfer_meta_decoder = msgspec.msgpack.Decoder(MooncakeXferMetadata)
self._xfer_resp_decoder = msgspec.msgpack.Decoder(MooncakeXferResponse)
def _sync_block_size_with_kernel(self) -> None:
# When speculative decoding (e.g. Eagle) is enabled, the main model
# and draft model may use different attention backends with different
# physical block sizes. Pick the common (smallest) block size so that
# KV-cache registration and transfer work correctly for both models.
backends = get_current_attn_backends(self.vllm_config)
kernel_block_size = select_common_block_size(self.block_size, backends)
if self.block_size != kernel_block_size:
logger.info_once(
"User-specified logical block size (%s) does not match"
" physical kernel block size (%s). Using the latter.",
self.block_size,
kernel_block_size,
)
assert self.block_size > kernel_block_size
self.block_size = kernel_block_size
def __del__(self):
self.shutdown()
@@ -1268,9 +1288,6 @@ class MooncakeConnectorWorker:
self.block_len_per_layer.append(
curr_tensor_size_bytes // self.num_blocks
)
kernel_block_size = cache.shape[-2 if self.use_mla else -3]
assert self.block_size == kernel_block_size
kv_data_ptrs.append(base_addr)
kv_data_lens.append(curr_tensor_size_bytes)