[Bugfix][Hybrid][NemotronH] Fix mamba_cache_mode=all + speculative decoding crash (#41233)
Signed-off-by: Roi Koren <roik@nvidia.com>
This commit is contained in:
@@ -364,26 +364,34 @@ def get_fake_process_mamba_fn(
|
||||
def fake_post_process_mamba_fn(
|
||||
scheduler_output: SchedulerOutput,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
cache_config: CacheConfig,
|
||||
input_batch: GPUInputBatch,
|
||||
requests: dict[str, CachedRequestState],
|
||||
mamba_state_idx: dict[str, int],
|
||||
forward_context: dict[str, Any],
|
||||
mamba_state_copy_funcs: tuple[MambaStateCopyFunc, ...],
|
||||
copy_bufs: mamba_utils.MambaCopyBuffers,
|
||||
num_spec_tokens: int,
|
||||
num_reqs: int,
|
||||
*,
|
||||
forward_context: dict[str, Any] | None = None,
|
||||
mamba_state_copy_funcs: tuple[MambaStateCopyFunc, ...] | None = None,
|
||||
copy_bufs: mamba_utils.MambaCopyBuffers | None = None,
|
||||
):
|
||||
nonlocal copy_info
|
||||
copy_info = None
|
||||
ret = original_post_process_mamba_fn(
|
||||
scheduler_output,
|
||||
kv_cache_config,
|
||||
cache_config,
|
||||
input_batch,
|
||||
requests,
|
||||
mamba_state_idx,
|
||||
forward_context,
|
||||
mamba_state_copy_funcs,
|
||||
copy_bufs,
|
||||
num_spec_tokens,
|
||||
num_reqs,
|
||||
forward_context=forward_context,
|
||||
mamba_state_copy_funcs=mamba_state_copy_funcs,
|
||||
copy_bufs=copy_bufs,
|
||||
)
|
||||
if cur_step_action is not None:
|
||||
assert forward_context is not None
|
||||
check_copy_info(
|
||||
cur_step_action.postprocess_copy_idx,
|
||||
kv_cache_config,
|
||||
|
||||
Reference in New Issue
Block a user