[Bugfix] neuron: enable tensor parallelism (#7562)

Signed-off-by: omrishiv <327609+omrishiv@users.noreply.github.com>
This commit is contained in:
omrishiv
2024-08-26 15:13:13 -07:00
committed by GitHub
parent 05826c887b
commit 760e9f71a8
3 changed files with 44 additions and 11 deletions
+5 -3
View File
@@ -317,9 +317,10 @@ class EngineArgs:
parser.add_argument('--block-size',
type=int,
default=EngineArgs.block_size,
choices=[8, 16, 32, 128, 256, 512, 1024, 2048],
choices=[8, 16, 32],
help='Token block size for contiguous chunks of '
'tokens.')
'tokens. This is ignored on neuron devices and '
'set to max-model-len')
parser.add_argument('--enable-prefix-caching',
action='store_true',
@@ -793,7 +794,8 @@ class EngineArgs:
limit_mm_per_prompt=self.limit_mm_per_prompt,
)
cache_config = CacheConfig(
block_size=self.block_size,
block_size=self.block_size if self.device != "neuron" else
self.max_model_len, # neuron needs block_size = max_model_len
gpu_memory_utilization=self.gpu_memory_utilization,
swap_space=self.swap_space,
cache_dtype=self.kv_cache_dtype,