[Bugfix][Granite4Vision] Fix deepstack buffer causing decode slowdown in compiled mode (#40917)
Signed-off-by: artemspector <artems@il.ibm.com> Co-authored-by: artemspector <artems@il.ibm.com>
This commit is contained in:
co-authored by
artemspector
parent
ea74f701db
commit
bde0efdbb7
@@ -887,9 +887,10 @@ class Granite4VisionForConditionalGeneration(
|
|||||||
and get_pp_group().is_first_rank
|
and get_pp_group().is_first_rank
|
||||||
and self._ds_layer_indices
|
and self._ds_layer_indices
|
||||||
):
|
):
|
||||||
|
n = inputs_embeds.size(0)
|
||||||
ds: IntermediateTensors | None = IntermediateTensors(
|
ds: IntermediateTensors | None = IntermediateTensors(
|
||||||
{
|
{
|
||||||
f"ds_{llm_layer}": self._ds_buffers[lvl]
|
f"ds_{llm_layer}": self._ds_buffers[lvl][:n]
|
||||||
for lvl, llm_layer in enumerate(self._ds_layer_indices)
|
for lvl, llm_layer in enumerate(self._ds_layer_indices)
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user