Neuron up mistral (#18222)

Signed-off-by: Satyajith Chilappagari <satchill@amazon.com>
This commit is contained in:
Satyajith Chilappagari
2025-05-19 09:54:47 -07:00
committed by GitHub
parent 8171221834
commit dc1440cf9f
3 changed files with 36 additions and 2 deletions
@@ -48,6 +48,9 @@ TORCH_DTYPE_TO_NEURON_AMP = {
# Models supported by Neuronx distributed for inference.
_NEURON_SUPPORTED_MODELS: dict[str, tuple[str, str]] = {
"LlamaForCausalLM":
("neuronx_distributed_inference.models.llama.modeling_llama",
"NeuronLlamaForCausalLM"),
"MistralForCausalLM":
("neuronx_distributed_inference.models.llama.modeling_llama",
"NeuronLlamaForCausalLM"),
"DbrxForCausalLM":
+1 -2
View File
@@ -51,8 +51,7 @@ class NeuronPlatform(Platform):
assert (vllm_config.lora_config
is None), "LoRA is not supported for Neuron backend."
cache_config = vllm_config.cache_config
if cache_config:
if vllm_config.cache_config and vllm_config.model_config:
# neuron needs block_size = max_model_len
vllm_config.cache_config.block_size = \
vllm_config.model_config.max_model_len # type: ignore