[Bugfix] Guard _may_reorder_batch for encoder-only models on CPU (#24319) (#24348)

Signed-off-by: Remy <eunhwan.shin@dtonic.io>
Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
Remy
2025-09-10 14:24:42 +08:00
committed by GitHub
co-authored by Li, Jiang <jiang1.li@intel.com>
parent 91130ae376
commit feaf202e93
3 changed files with 26 additions and 7 deletions
+16 -4
View File
@@ -55,11 +55,23 @@ class CPUModelRunner(GPUModelRunner):
raise ValueError("Multiple KVCacheGroups is not"
"currently supported with CPU model runner.")
assert type(self.attn_groups[0]
[0].metadata_builder) is TorchSDPAMetadataBuilderV1
# Guard against encoder-only / pooling models where `attn_groups`
# may be empty or lack the expected metadata_builder.
# Without this check, accessing `attn_groups[0][0]` would trigger
# an AssertionError on CPU backend.
if not hasattr(self, "attn_groups") or not self.attn_groups:
return
if not self.attn_groups[0]:
return
self.attn_groups[0][0].metadata_builder.reorder_batch(
self.input_batch, scheduler_output)
mb = getattr(self.attn_groups[0][0], "metadata_builder", None)
if not isinstance(mb, TorchSDPAMetadataBuilderV1):
# Encoder-only / rerank models do not benefit from reordering,
# so we safely skip here.
return
# Safe path for decoder/attention-heavy models
mb.reorder_batch(self.input_batch, scheduler_output)
def _postprocess_tensors(self) -> None:
# Note: replace device tensors with cpu tensors