[Bugfix][Hybrid][NemotronH] Fix mamba_cache_mode=all + speculative decoding crash (#41233)
Signed-off-by: Roi Koren <roik@nvidia.com>
This commit is contained in:
@@ -582,7 +582,9 @@ class MambaSpec(KVCacheSpec):
|
||||
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
|
||||
if vllm_config.cache_config.mamba_cache_mode == "all":
|
||||
max_model_len = vllm_config.model_config.max_model_len
|
||||
return cdiv(max_model_len, self.block_size) * self.page_size_bytes
|
||||
return (
|
||||
cdiv(max_model_len, self.block_size) + self.num_speculative_blocks
|
||||
) * self.page_size_bytes
|
||||
elif vllm_config.cache_config.mamba_cache_mode == "align":
|
||||
return self.page_size_bytes * (2 + self.num_speculative_blocks)
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user