[Bugfix][Hybrid][NemotronH] Fix mamba_cache_mode=all + speculative decoding crash (#41233)

Signed-off-by: Roi Koren <roik@nvidia.com>
This commit is contained in:
roikoren755
2026-05-18 14:54:00 +03:00
committed by GitHub
parent e414e1f1c0
commit 737bfa3a43
10 changed files with 568 additions and 117 deletions
+41 -11
View File
@@ -888,6 +888,11 @@ class GPUModelRunner(
self.kv_connector_output: KVConnectorOutput | None = None
self.mamba_state_idx: dict[str, int] = {}
self._mamba_copy_bufs: mamba_utils.MambaCopyBuffers | None = None
self.mamba_prev_last_scheduled_idx: CpuGpuBuffer | None = None
if self.cache_config.mamba_cache_mode == "all" and self.num_spec_tokens > 0:
self.mamba_prev_last_scheduled_idx = self._make_buffer(
self.max_num_reqs, dtype=torch.int32
)
self.layerwise_nvtx_hooks_registered = False
def update_max_model_len(self, max_model_len: int) -> None:
@@ -1493,21 +1498,12 @@ class GPUModelRunner(
num_reqs = output_token_ids.size(0)
self.num_accepted_tokens.gpu[:num_reqs] = (output_token_ids != -1).sum(dim=1)
if self.cache_config.mamba_cache_mode == "align":
is_align = self.cache_config.mamba_cache_mode == "align"
if is_align:
for i, num_tokens in enumerate(
self.num_accepted_tokens.gpu[:num_reqs].cpu().numpy()
):
self.input_batch.num_accepted_tokens_cpu[i] = num_tokens
mamba_utils.postprocess_mamba(
scheduler_output,
self.kv_cache_config,
self.input_batch,
self.requests,
self.mamba_state_idx,
self.compilation_config.static_forward_context,
self.model.get_mamba_state_copy_func(),
self._get_mamba_copy_bufs(),
)
else:
self.input_batch.num_accepted_tokens_cpu_tensor[:num_reqs].copy_(
self.num_accepted_tokens.gpu[:num_reqs], non_blocking=True
@@ -1515,6 +1511,24 @@ class GPUModelRunner(
assert self.num_accepted_tokens_event is not None
self.num_accepted_tokens_event.record()
mamba_utils.postprocess_mamba(
scheduler_output,
self.kv_cache_config,
self.cache_config,
self.input_batch,
self.requests,
self.mamba_state_idx,
self.num_spec_tokens,
num_reqs,
forward_context=(
self.compilation_config.static_forward_context if is_align else None
),
mamba_state_copy_funcs=(
self.model.get_mamba_state_copy_func() if is_align else None
),
copy_bufs=self._get_mamba_copy_bufs() if is_align else None,
)
def _update_streaming_request(
self, req_id: str, new_req_data: NewRequestData
) -> CachedRequestState:
@@ -2012,6 +2026,15 @@ class GPUModelRunner(
self.num_accepted_tokens.np.fill(1)
self.num_accepted_tokens.gpu.fill_(1)
if self.mamba_prev_last_scheduled_idx is not None:
mamba_utils.preprocess_mamba_all_specdec(
scheduler_output,
self.input_batch,
self.mamba_state_idx,
num_reqs,
self.mamba_prev_last_scheduled_idx,
)
# Update num_computed_tokens on GPU. In async spec decode,
# CPU values are optimistic (all drafts accepted). The kernel
# corrects on GPU using the previous step's
@@ -2319,6 +2342,13 @@ class GPUModelRunner(
:num_reqs_padded
],
)
if (
isinstance(builder, Mamba2AttentionMetadataBuilder)
and self.mamba_prev_last_scheduled_idx is not None
):
extra_attn_metadata_args["prev_last_scheduled_idx"] = (
self.mamba_prev_last_scheduled_idx.gpu[:num_reqs_padded]
)
if for_cudagraph_capture:
attn_metadata_i = builder.build_for_cudagraph_capture(