[Mamba] Add FlashInfer ReplaySSM backend (#52506)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Signed-off-by: Andrii Skliar <andreyws96@gmail.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com>
This commit is contained in:
co-authored by
Andrii Skliar
parent
25efcfa788
commit
adebc41b7e
@@ -747,7 +747,8 @@ class NemotronHForCausalLM(
|
||||
)
|
||||
if cache_config.use_replayssm:
|
||||
return MambaStateDtypeCalculator.append_replayssm_ring(
|
||||
base_dtype, vllm_config.model_config.dtype
|
||||
base_dtype,
|
||||
vllm_config.model_config.dtype,
|
||||
)
|
||||
return base_dtype
|
||||
|
||||
@@ -784,10 +785,11 @@ class NemotronHForCausalLM(
|
||||
)
|
||||
if cache_config.use_replayssm:
|
||||
return MambaStateShapeCalculator.append_replayssm_ring(
|
||||
base_shape,
|
||||
hf_config.n_groups,
|
||||
parallel_config.tensor_parallel_size,
|
||||
cache_config.replayssm_buffer_len,
|
||||
base_shapes=base_shape,
|
||||
n_groups=hf_config.n_groups,
|
||||
tp_world_size=parallel_config.tensor_parallel_size,
|
||||
logical_window=cache_config.replayssm_buffer_len,
|
||||
backend=vllm_config.mamba_config.backend,
|
||||
)
|
||||
return base_shape
|
||||
|
||||
|
||||
Reference in New Issue
Block a user