[Hy4] support Hy4-preview model (#54160)
Signed-off-by: chengvjiang <chengvjiang@tencent.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: chengvjiang <chengvjiang@tencent.com> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: russellfeng <russellfeng@tencent.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
co-authored by
chengvjiang
Jee Jee Li
Bugen Zhao
russellfeng
Jee Jee Li
parent
6c18a54648
commit
b2f685834a
@@ -874,12 +874,20 @@ def _try_load_fp8_indexer_wk(
|
||||
# We have both weight and scale: dequantize FP8 to BF16.
|
||||
weight_fp8, scale_inv = entry["weight"], entry["scale"]
|
||||
del buf[layer_prefix]
|
||||
if scale_inv.dtype in (torch.uint8, torch.float8_e8m0fnu):
|
||||
# MXFP8 checkpoints store power-of-two E8M0 scales, decode them to float.
|
||||
scale_inv = scale_inv.view(torch.float8_e8m0fnu).to(torch.float32)
|
||||
if scale_inv.ndim == 1:
|
||||
# Per-channel scale: one scale per row of [out, in]
|
||||
group_shape = GroupShape(1, weight_fp8.shape[1])
|
||||
else:
|
||||
block_size = weight_fp8.shape[1] // scale_inv.shape[1]
|
||||
group_shape = GroupShape(block_size, block_size)
|
||||
# Derive the block size independently per dim so that per-channel
|
||||
# ([out, 1]), MX ([out, in / 32]) and 2D block ([out / 128, in / 128])
|
||||
# scale layouts are all handled.
|
||||
group_shape = GroupShape(
|
||||
weight_fp8.shape[0] // scale_inv.shape[0],
|
||||
weight_fp8.shape[1] // scale_inv.shape[1],
|
||||
)
|
||||
weight_bf16 = scaled_dequantize(
|
||||
weight_fp8,
|
||||
scale_inv,
|
||||
|
||||
@@ -128,6 +128,7 @@ _TEXT_GENERATION_MODELS = {
|
||||
"GraniteSWAForCausalLM": ("granite", "GraniteForCausalLM"),
|
||||
"HrmTextForCausalLM": ("hrm_text", "HrmTextForCausalLM"),
|
||||
"HYV3ForCausalLM": ("hy_v3", "HYV3ForCausalLM"),
|
||||
"HYV4ForCausalLM": ("vllm.models.hy_v4", "HYV4ForCausalLM"),
|
||||
"HCXVisionV2ForCausalLM": ("hyperclovax_vision_v2", "HCXVisionV2ForCausalLM"),
|
||||
"HyperCLOVAXForCausalLM": ("hyperclovax", "HyperCLOVAXForCausalLM"),
|
||||
"InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
|
||||
@@ -663,6 +664,7 @@ _SPECULATIVE_DECODING_MODELS = {
|
||||
"Qwen3_5MoeMTP": ("qwen3_5_mtp", "Qwen3_5MoeMTP"),
|
||||
"InternS2MobiusMTP": ("interns2_mobius", "InternS2MobiusMTP"),
|
||||
"HYV3MTPModel": ("hy_v3_mtp", "HYV3MTP"),
|
||||
"HYV4MTPModel": ("vllm.models.hy_v4", "HYV4MTP"),
|
||||
"KimiK3MTPModel": ("vllm.models.kimi_k3", "KimiK3MTP"),
|
||||
# Temporarily disabled.
|
||||
# # TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1.
|
||||
|
||||
Reference in New Issue
Block a user