[Model][NVIDIA] Route DSA models to the CUDA non-compiled path (#52861)

This commit is contained in:
Woosuk Kwon
2026-08-19 00:13:52 -07:00
committed by GitHub
parent ee11730751
commit b09bd69b5b
16 changed files with 260 additions and 146 deletions
+7 -5
View File
@@ -341,6 +341,7 @@ class SpeculativeConfig:
@staticmethod
def hf_config_override(hf_config: PretrainedConfig) -> PretrainedConfig:
initial_architecture = hf_config.architectures[0]
use_v32_mtp = hf_config.model_type in ("deepseek_v32", "glm_moe_dsa")
if hf_config.model_type == "dots3_note":
n_predict = getattr(hf_config, "num_nextn_predict_layers", 1)
mtp_layer_types = getattr(hf_config, "mtp_layer_types", None)
@@ -365,7 +366,12 @@ class SpeculativeConfig:
if hf_config.model_type == "deepseek_mtp":
n_predict = getattr(hf_config, "num_nextn_predict_layers", None)
hf_config.update(
{"n_predict": n_predict, "architectures": ["DeepSeekMTPModel"]}
{
"n_predict": n_predict,
"architectures": [
"DeepseekV32MTPModel" if use_v32_mtp else "DeepSeekMTPModel"
],
}
)
if hf_config.model_type == "deepseek_v4":
hf_config.model_type = "deepseek_mtp"
@@ -759,10 +765,6 @@ class SpeculativeConfig:
if self.method == "mtp":
if self.target_model_config is None:
raise ValueError("target_model_config must be present for mtp")
if self.target_model_config.hf_text_config.model_type == "deepseek_v32":
# FIXME(luccafong): cudagraph with v32 MTP is not supported,
# remove this when the issue is fixed.
self.enforce_eager = True
# use the draft model from the same model:
self.model = self.target_model_config.model
# Align the quantization of draft model for cases such as