Support encoder-only models without KV-Cache (#21270)
Signed-off-by: Max de Bayser <maxdebayser@gmail.com> Signed-off-by: Max de Bayser <mbayser@br.ibm.com> Co-authored-by: Russell Bryant <rbryant@redhat.com>
This commit is contained in:
co-authored by
Russell Bryant
parent
f27fdfc3ed
commit
1cd6eaba54
@@ -330,6 +330,7 @@ class EagleProposer:
|
||||
max_query_len=new_query_len_per_req.max().item(),
|
||||
block_table_tensor=common_attn_metadata.block_table_tensor,
|
||||
slot_mapping=common_attn_metadata.slot_mapping[token_indices],
|
||||
causal=True,
|
||||
)
|
||||
|
||||
return spec_common_attn_metadata, token_indices
|
||||
|
||||
Reference in New Issue
Block a user