[Bugfix][Hybrid][NemotronH] Fix mamba_cache_mode=all + speculative decoding crash (#41233)
Signed-off-by: Roi Koren <roik@nvidia.com>
This commit is contained in:
@@ -888,6 +888,11 @@ class GPUModelRunner(
|
||||
self.kv_connector_output: KVConnectorOutput | None = None
|
||||
self.mamba_state_idx: dict[str, int] = {}
|
||||
self._mamba_copy_bufs: mamba_utils.MambaCopyBuffers | None = None
|
||||
self.mamba_prev_last_scheduled_idx: CpuGpuBuffer | None = None
|
||||
if self.cache_config.mamba_cache_mode == "all" and self.num_spec_tokens > 0:
|
||||
self.mamba_prev_last_scheduled_idx = self._make_buffer(
|
||||
self.max_num_reqs, dtype=torch.int32
|
||||
)
|
||||
self.layerwise_nvtx_hooks_registered = False
|
||||
|
||||
def update_max_model_len(self, max_model_len: int) -> None:
|
||||
@@ -1493,21 +1498,12 @@ class GPUModelRunner(
|
||||
num_reqs = output_token_ids.size(0)
|
||||
self.num_accepted_tokens.gpu[:num_reqs] = (output_token_ids != -1).sum(dim=1)
|
||||
|
||||
if self.cache_config.mamba_cache_mode == "align":
|
||||
is_align = self.cache_config.mamba_cache_mode == "align"
|
||||
if is_align:
|
||||
for i, num_tokens in enumerate(
|
||||
self.num_accepted_tokens.gpu[:num_reqs].cpu().numpy()
|
||||
):
|
||||
self.input_batch.num_accepted_tokens_cpu[i] = num_tokens
|
||||
mamba_utils.postprocess_mamba(
|
||||
scheduler_output,
|
||||
self.kv_cache_config,
|
||||
self.input_batch,
|
||||
self.requests,
|
||||
self.mamba_state_idx,
|
||||
self.compilation_config.static_forward_context,
|
||||
self.model.get_mamba_state_copy_func(),
|
||||
self._get_mamba_copy_bufs(),
|
||||
)
|
||||
else:
|
||||
self.input_batch.num_accepted_tokens_cpu_tensor[:num_reqs].copy_(
|
||||
self.num_accepted_tokens.gpu[:num_reqs], non_blocking=True
|
||||
@@ -1515,6 +1511,24 @@ class GPUModelRunner(
|
||||
assert self.num_accepted_tokens_event is not None
|
||||
self.num_accepted_tokens_event.record()
|
||||
|
||||
mamba_utils.postprocess_mamba(
|
||||
scheduler_output,
|
||||
self.kv_cache_config,
|
||||
self.cache_config,
|
||||
self.input_batch,
|
||||
self.requests,
|
||||
self.mamba_state_idx,
|
||||
self.num_spec_tokens,
|
||||
num_reqs,
|
||||
forward_context=(
|
||||
self.compilation_config.static_forward_context if is_align else None
|
||||
),
|
||||
mamba_state_copy_funcs=(
|
||||
self.model.get_mamba_state_copy_func() if is_align else None
|
||||
),
|
||||
copy_bufs=self._get_mamba_copy_bufs() if is_align else None,
|
||||
)
|
||||
|
||||
def _update_streaming_request(
|
||||
self, req_id: str, new_req_data: NewRequestData
|
||||
) -> CachedRequestState:
|
||||
@@ -2012,6 +2026,15 @@ class GPUModelRunner(
|
||||
self.num_accepted_tokens.np.fill(1)
|
||||
self.num_accepted_tokens.gpu.fill_(1)
|
||||
|
||||
if self.mamba_prev_last_scheduled_idx is not None:
|
||||
mamba_utils.preprocess_mamba_all_specdec(
|
||||
scheduler_output,
|
||||
self.input_batch,
|
||||
self.mamba_state_idx,
|
||||
num_reqs,
|
||||
self.mamba_prev_last_scheduled_idx,
|
||||
)
|
||||
|
||||
# Update num_computed_tokens on GPU. In async spec decode,
|
||||
# CPU values are optimistic (all drafts accepted). The kernel
|
||||
# corrects on GPU using the previous step's
|
||||
@@ -2319,6 +2342,13 @@ class GPUModelRunner(
|
||||
:num_reqs_padded
|
||||
],
|
||||
)
|
||||
if (
|
||||
isinstance(builder, Mamba2AttentionMetadataBuilder)
|
||||
and self.mamba_prev_last_scheduled_idx is not None
|
||||
):
|
||||
extra_attn_metadata_args["prev_last_scheduled_idx"] = (
|
||||
self.mamba_prev_last_scheduled_idx.gpu[:num_reqs_padded]
|
||||
)
|
||||
|
||||
if for_cudagraph_capture:
|
||||
attn_metadata_i = builder.build_for_cudagraph_capture(
|
||||
|
||||
Reference in New Issue
Block a user