Add Medusa speculative decoding e2e test (#41396)
Signed-off-by: Anshika Ojha <anshikao@nvidia.com> Signed-off-by: Rishi Puri <riship@nvidia.com> Signed-off-by: Rishi Puri <puririshi98@berkeley.edu> Signed-off-by: Stefano Castagnetta <scastagnetta@nvidia.com> Co-authored-by: Anshika Ojha <215760622+ojhaanshika@users.noreply.github.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Stefano Castagnetta <scastagnetta@nvidia.com>
This commit is contained in:
co-authored by
Anshika Ojha
Claude
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Stefano Castagnetta
parent
25671cb520
commit
727971f1c1
@@ -136,6 +136,26 @@ class Medusa(nn.Module):
|
||||
|
||||
return logits_lst
|
||||
|
||||
@staticmethod
|
||||
def _remap_old_checkpoint_key(name: str) -> str:
|
||||
"""Map old FasterDecoding checkpoint keys to vLLM parameter names.
|
||||
|
||||
Old format uses bare numeric prefixes:
|
||||
'{head}.{layer}.linear.weight' -> 'blocks.{head}.layers.{layer}.weight'
|
||||
'{head}.{layer}.linear.bias' -> 'blocks.{head}.layers.{layer}.bias'
|
||||
'{head}.{N}.weight' -> 'lm_heads.{head}.weight'
|
||||
"""
|
||||
parts = name.split(".")
|
||||
if len(parts) >= 3 and parts[0].isdigit() and parts[1].isdigit():
|
||||
head, layer = parts[0], parts[1]
|
||||
rest = parts[2:]
|
||||
if "linear" in rest:
|
||||
param = ".".join(rest[rest.index("linear") + 1 :])
|
||||
return f"blocks.{head}.layers.{layer}.{param}"
|
||||
else:
|
||||
return f"lm_heads.{head}.{'.'.join(rest)}"
|
||||
return name
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
@@ -144,6 +164,7 @@ class Medusa(nn.Module):
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
name = name.replace("medusa_heads.", "")
|
||||
name = self._remap_old_checkpoint_key(name)
|
||||
|
||||
if name == "token_map":
|
||||
if self.truncated_vocab_size < self.orig_vocab_size:
|
||||
|
||||
Reference in New Issue
Block a user