[PD][Bugfix] Fix validation of cache shape for attn backends enforcing different kernel_block_size (#48125)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
@@ -1169,7 +1169,12 @@ class NixlBaseConnectorWorker:
|
|||||||
"All non-MLA kv cache tensors must have the same size"
|
"All non-MLA kv cache tensors must have the same size"
|
||||||
)
|
)
|
||||||
|
|
||||||
if cache.shape[0] != num_blocks:
|
# When there's a mismatch between kbs<>bs, we rely on HMA to ensure
|
||||||
|
# caches are either [NB, PS] or [NB*r, PS/r] where r is bs/kbs.
|
||||||
|
if (
|
||||||
|
self._physical_blocks_per_logical_kv_block == 1
|
||||||
|
and cache.shape[0] != num_blocks
|
||||||
|
):
|
||||||
raise AssertionError(
|
raise AssertionError(
|
||||||
"All kv cache tensors must have the same number of "
|
"All kv cache tensors must have the same number of "
|
||||||
f"blocks; layer={layer_name}, "
|
f"blocks; layer={layer_name}, "
|
||||||
|
|||||||
Reference in New Issue
Block a user