[Mooncake] Fix mixed MLA+Eagle block-size validation (#39596)
Signed-off-by: Zhewen Li <zhewenli@inferact.ai> Co-authored-by: Zhewen Li <zhewenli@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Zhewen Li
OpenAI Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
21e5a9f48e
commit
55e1a8e103
@@ -23,6 +23,7 @@ from vllm.distributed.kv_transfer.kv_connector.utils import (
|
||||
EngineId,
|
||||
TpKVTopology,
|
||||
get_current_attn_backend,
|
||||
get_current_attn_backends,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
@@ -47,6 +48,7 @@ from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.attention.backends.utils import get_kv_cache_layout
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.request import RequestStatus
|
||||
from vllm.v1.worker.utils import select_common_block_size
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -751,6 +753,7 @@ class MooncakeConnectorWorker:
|
||||
self.model_config = vllm_config.model_config
|
||||
self.cache_config = vllm_config.cache_config
|
||||
self.use_mla = self.model_config.use_mla
|
||||
self._sync_block_size_with_kernel()
|
||||
|
||||
# Get the attention backend from the first layer
|
||||
# NOTE (NickLucche) models with multiple backends are not supported yet
|
||||
@@ -777,6 +780,23 @@ class MooncakeConnectorWorker:
|
||||
self._xfer_meta_decoder = msgspec.msgpack.Decoder(MooncakeXferMetadata)
|
||||
self._xfer_resp_decoder = msgspec.msgpack.Decoder(MooncakeXferResponse)
|
||||
|
||||
def _sync_block_size_with_kernel(self) -> None:
|
||||
# When speculative decoding (e.g. Eagle) is enabled, the main model
|
||||
# and draft model may use different attention backends with different
|
||||
# physical block sizes. Pick the common (smallest) block size so that
|
||||
# KV-cache registration and transfer work correctly for both models.
|
||||
backends = get_current_attn_backends(self.vllm_config)
|
||||
kernel_block_size = select_common_block_size(self.block_size, backends)
|
||||
if self.block_size != kernel_block_size:
|
||||
logger.info_once(
|
||||
"User-specified logical block size (%s) does not match"
|
||||
" physical kernel block size (%s). Using the latter.",
|
||||
self.block_size,
|
||||
kernel_block_size,
|
||||
)
|
||||
assert self.block_size > kernel_block_size
|
||||
self.block_size = kernel_block_size
|
||||
|
||||
def __del__(self):
|
||||
self.shutdown()
|
||||
|
||||
@@ -1268,9 +1288,6 @@ class MooncakeConnectorWorker:
|
||||
self.block_len_per_layer.append(
|
||||
curr_tensor_size_bytes // self.num_blocks
|
||||
)
|
||||
|
||||
kernel_block_size = cache.shape[-2 if self.use_mla else -3]
|
||||
assert self.block_size == kernel_block_size
|
||||
kv_data_ptrs.append(base_addr)
|
||||
kv_data_lens.append(curr_tensor_size_bytes)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user