[MRV2] Extensible CG dispatch rework (#35959)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
Lucas Wilkinson
2026-03-09 13:58:45 -07:00
committed by GitHub
parent 4e571ce643
commit 483463f735
9 changed files with 561 additions and 652 deletions
+4 -1
View File
@@ -37,6 +37,7 @@ class InputBatch:
# batch_idx -> req_id
req_ids: list[str]
num_reqs: int
num_reqs_after_padding: int
# batch_idx -> req_state_idx
idx_mapping: torch.Tensor
@@ -123,6 +124,7 @@ class InputBatch:
return cls(
req_ids=req_ids,
num_reqs=num_reqs,
num_reqs_after_padding=num_reqs,
idx_mapping=idx_mapping,
idx_mapping_np=idx_mapping_np,
expanded_idx_mapping=expanded_idx_mapping,
@@ -330,7 +332,8 @@ def combine_sampled_and_draft_tokens(
cu_num_logits: torch.Tensor,
num_logits: int,
) -> torch.Tensor:
num_reqs = seq_lens.shape[0]
# use idx_mapping.shape[0] for actual request count
num_reqs = idx_mapping.shape[0]
num_speculative_steps = draft_tokens.shape[-1]
logits_indices = torch.empty(