[ROCm] Enable SimpleCPUOffloadConnector on ROCm (#40549)
Signed-off-by: Hongxia Yang <hongxia.yang@amd.com> Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
This commit is contained in:
@@ -10,8 +10,8 @@ from vllm import LLM, SamplingParams, TokensPrompt
|
||||
from vllm.config import KVTransferConfig
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
pytest.skip("Requires CUDA", allow_module_level=True)
|
||||
if not current_platform.is_cuda_alike():
|
||||
pytest.skip("Requires CUDA or ROCm", allow_module_level=True)
|
||||
|
||||
# Small models for default CI / local runs (accuracy only).
|
||||
SMALL_MODELS = [
|
||||
|
||||
@@ -186,7 +186,13 @@ def make_request(
|
||||
if request_id is None:
|
||||
request_id = f"req-{_req_counter}"
|
||||
|
||||
num_tokens = num_blocks * BLOCK_SIZE
|
||||
# Add one extra token beyond the last full block so that
|
||||
# ``max_cache_hit_length = num_tokens - 1`` (see
|
||||
# KVCacheManager.get_computed_blocks) does not truncate the final
|
||||
# full block: ``find_longest_cache_hit`` uses
|
||||
# ``max_length // block_size`` and would otherwise drop one block
|
||||
# when the prompt is an exact multiple of block_size.
|
||||
num_tokens = num_blocks * BLOCK_SIZE + 1
|
||||
start = _req_counter * 10000
|
||||
prompt_token_ids = list(range(start, start + num_tokens))
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
|
||||
Reference in New Issue
Block a user