[ROCm][CI] Add MTP and other spec-decode acceptance coverage (#53399)
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
8052102c21
commit
e52407ef4c
@@ -118,6 +118,7 @@ class Gemma4Proposer(SpecDecodeBaseProposer):
|
||||
"""Capture CUDA graphs for centroids get_top_tokens at key sizes."""
|
||||
masked_emb = self.model.masked_embedding
|
||||
lm_head_weight = self.model._get_full_lm_head_weight()
|
||||
capture_stream = current_stream()
|
||||
|
||||
for size in [1, 2, 4, 8, 16, 32, 64]:
|
||||
static_input = torch.zeros(
|
||||
@@ -131,7 +132,7 @@ class Gemma4Proposer(SpecDecodeBaseProposer):
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
g = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(g, stream=current_stream()):
|
||||
with torch.cuda.graph(g, stream=capture_stream):
|
||||
static_output = masked_emb.get_top_tokens(
|
||||
static_input,
|
||||
lm_head_weight,
|
||||
|
||||
Reference in New Issue
Block a user