[Neuron] Support inference with transformers-neuronx (#2569)
This commit is contained in:
@@ -80,9 +80,16 @@ class ModelRunner:
|
||||
self.in_wsl = in_wsl()
|
||||
self.kv_cache_dtype = kv_cache_dtype
|
||||
|
||||
# Set enforce_eager to True for Neuron backend, to avoid capturing graph
|
||||
if self.device_config.is_neuron:
|
||||
self.model_config.enforce_eager = True
|
||||
|
||||
def load_model(self) -> None:
|
||||
self.model = get_model(self.model_config, self.device_config,
|
||||
self.lora_config)
|
||||
self.model = get_model(self.model_config,
|
||||
self.device_config,
|
||||
lora_config=self.lora_config,
|
||||
parallel_config=self.parallel_config,
|
||||
scheduler_config=self.scheduler_config)
|
||||
|
||||
vocab_size = self.model.config.vocab_size
|
||||
|
||||
@@ -393,6 +400,7 @@ class ModelRunner:
|
||||
selected_token_start_idx = 0
|
||||
categorized_sample_indices = {t: [] for t in SamplingType}
|
||||
categorized_sample_indices_start_idx = 0
|
||||
pin_memory = not self.in_wsl and not self.device_config.is_neuron
|
||||
|
||||
max_subquery_len = max(subquery_lens) if subquery_lens else 1
|
||||
for i, seq_group_metadata in enumerate(seq_group_metadata_list):
|
||||
@@ -443,12 +451,12 @@ class ModelRunner:
|
||||
selected_token_indices = _async_h2d(selected_token_indices,
|
||||
dtype=torch.long,
|
||||
target_device=self.device,
|
||||
pin_memory=not self.in_wsl)
|
||||
pin_memory=pin_memory)
|
||||
categorized_sample_indices = {
|
||||
t: _async_h2d(seq_ids,
|
||||
dtype=torch.int,
|
||||
target_device=self.device,
|
||||
pin_memory=not self.in_wsl)
|
||||
pin_memory=pin_memory)
|
||||
for t, seq_ids in categorized_sample_indices.items()
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user