Signed-off-by: Karthik Kothuri <karthikkothuri2009@gmail.com> Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
co-authored by
Claude
wang.yuqi
parent
56a357ed33
commit
107a03ba63
@@ -126,3 +126,4 @@ def wikitext_ppl_test(
|
||||
# We are not concerned that the vllm PPL is less than Transformers,
|
||||
# so we only perform one-sided testing.
|
||||
assert differ < atol
|
||||
return vllm_ppl
|
||||
|
||||
@@ -11,4 +11,14 @@ MODELS = [GenerateModelInfo("openai-community/gpt2-large", hf_ppl=19.45705604553
|
||||
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_ppl(hf_runner, vllm_runner, model_info: GenerateModelInfo):
|
||||
wikitext_ppl_test(hf_runner, vllm_runner, model_info)
|
||||
bf16_ppl = wikitext_ppl_test(hf_runner, vllm_runner, model_info)
|
||||
fp32_ppl = wikitext_ppl_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model_info,
|
||||
vllm_extra_kwargs={"hf_overrides": {"head_dtype": "float32"}},
|
||||
)
|
||||
|
||||
differ = ((fp32_ppl - bf16_ppl) / bf16_ppl) * 100
|
||||
print("fp32 head difference (%):", differ)
|
||||
assert differ < 0
|
||||
|
||||
Reference in New Issue
Block a user