[Model Runner V2] Use actual batch max_seq_len for attn metadata (#43991)
Signed-off-by: zhuhaoran <zhuhaoran.zhr@alibaba-inc.com>
This commit is contained in:
@@ -178,7 +178,7 @@ class DefaultModelState(ModelState):
|
||||
# Capture with worst-case max_seq_len so the graph is valid at any replay.
|
||||
max_seq_len = self.max_model_len
|
||||
else:
|
||||
max_seq_len = int(seq_lens_cpu_upper_bound[:num_reqs].max().item())
|
||||
max_seq_len = seq_lens_cpu_upper_bound[:num_reqs].max().item()
|
||||
attn_metadata = build_attn_metadata(
|
||||
attn_groups=attn_groups,
|
||||
num_reqs=num_reqs,
|
||||
|
||||
Reference in New Issue
Block a user