[Bugfix][ROCm][Disagg] Allow MoRIIO HMA per-layer block_len (#58585)
Signed-off-by: avininjamay8 <Avinash.Paul@amd.com> Signed-off-by: avininjamay8 <avpaul@amd.com> Co-authored-by: avininjamay8 <avpaul@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
co-authored by
avininjamay8
Cursor
Shanshan Shen
parent
de2b869ee9
commit
429a3da665
@@ -2453,10 +2453,15 @@ class MoRIIOConnectorWorker:
|
||||
caches_data = []
|
||||
self.layer_base_addr_index = {}
|
||||
base_addr_idx = 0
|
||||
|
||||
# HMA (#53721 Gemma4 SWA, GLM-5.2 DSA, MiniMax): transfers already
|
||||
# size RDMA from per-layer geometry, so allow divergent block_len
|
||||
# (bytes) always, and divergent block_size (tokens/page) only when
|
||||
# the hybrid KV cache manager is enabled. Without HMA, block_size
|
||||
# must stay uniform for the peer page handshake.
|
||||
hma_enabled = (
|
||||
not self.vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
|
||||
)
|
||||
|
||||
for layer_name in kv_caches:
|
||||
self.layer_base_addr_index[layer_name] = base_addr_idx
|
||||
if self._is_mamba_layer(layer_name):
|
||||
@@ -2471,16 +2476,31 @@ class MoRIIOConnectorWorker:
|
||||
base_addr_idx += 1
|
||||
continue
|
||||
geometry = self._get_layer_transfer_geometry(layer_name)
|
||||
if hma_enabled:
|
||||
if geometry.block_len != self.block_len:
|
||||
if geometry.block_size != self.block_size:
|
||||
if not hma_enabled:
|
||||
raise ValueError(
|
||||
"MoRIIO KV cache block length mismatch for layer "
|
||||
f"{layer_name}: {geometry.block_len} != {self.block_len}"
|
||||
"MoRIIO KV cache block size mismatch for layer "
|
||||
f"{layer_name}: {geometry.block_size} != "
|
||||
f"{self.block_size}"
|
||||
)
|
||||
elif geometry.block_size != self.block_size:
|
||||
raise ValueError(
|
||||
"MoRIIO KV cache block size mismatch for layer "
|
||||
f"{layer_name}: {geometry.block_size} != {self.block_size}"
|
||||
logger.debug(
|
||||
"MoRIIO HMA per-layer block_size for %s: %d "
|
||||
"(page/default %d); continuing with per-layer sizes.",
|
||||
layer_name,
|
||||
geometry.block_size,
|
||||
self.block_size,
|
||||
)
|
||||
if geometry.block_len != self.block_len:
|
||||
# Unlike block_size above, divergent block_len is allowed
|
||||
# unconditionally (not gated on hma_enabled) -- it covers
|
||||
# non-hybrid models too, e.g. GLM-5.2 DSA's per-layer
|
||||
# geometry. Don't attribute it to HMA in the log.
|
||||
logger.debug(
|
||||
"MoRIIO per-layer block_len for %s: %d "
|
||||
"(page/default %d); continuing with per-layer sizes.",
|
||||
layer_name,
|
||||
geometry.block_len,
|
||||
self.block_len,
|
||||
)
|
||||
# num_blocks is advertised as a single scalar to the peer, so must it
|
||||
# be uniform
|
||||
@@ -2510,6 +2530,7 @@ class MoRIIOConnectorWorker:
|
||||
# and a run that has three should not look identical in the log.
|
||||
distinct_block_lens = sorted(set(self.block_lens.values()))
|
||||
if len(distinct_block_lens) > 1:
|
||||
self.block_len = max(distinct_block_lens)
|
||||
logger.info(
|
||||
"MoRIIO registered %d attention layers with %d distinct block "
|
||||
"lengths (bytes): %s; transfer sizes and strides are per layer, "
|
||||
|
||||
Reference in New Issue
Block a user