[Model][Jamba] Mamba cache single buffer (#6739)

Co-authored-by: Mor Zusman <morz@ai21.com>
This commit is contained in:
Mor Zusman
2024-08-09 10:07:06 -04:00
committed by GitHub
co-authored by Mor Zusman
parent b4e9528f95
commit 07ab160741
2 changed files with 148 additions and 124 deletions
-3
View File
@@ -1711,9 +1711,6 @@ class CUDAGraphRunner:
non_blocking=True)
# Run the graph.
self.graph.replay()
if "seqlen_agnostic_capture_inputs" in self.input_buffers:
self.model.copy_outputs_after_cuda_graphs(self.input_buffers,
**kwargs)
# Return the output tensor.
if get_pp_group().is_last_rank:
return self.output_buffers["hidden_states"]