[bugfix] fix cpu tests (#10585)

Signed-off-by: youkaichao <youkaichao@gmail.com>
This commit is contained in:
youkaichao
2024-11-22 17:34:03 -08:00
committed by GitHub
parent d345f409b7
commit d559979c54
3 changed files with 16 additions and 10 deletions
+3 -1
View File
@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, cast
import torch
from vllm.attention import AttentionMetadata
from vllm.forward_context import set_forward_context
from vllm.model_executor import SamplingMetadata
from vllm.model_executor.layers.sampler import SamplerOutput
from vllm.multimodal import MultiModalKwargs
@@ -303,7 +304,8 @@ class CPUEncoderDecoderModelRunner(
intermediate_tensors,
}
hidden_states = model_executable(**execute_model_kwargs)
with set_forward_context(model_input.attn_metadata, self.vllm_config):
hidden_states = model_executable(**execute_model_kwargs)
# Compute the logits.
logits = self.model.compute_logits(hidden_states,