From 2381cd5907dde01aa0c190670a405153cded346b Mon Sep 17 00:00:00 2001 From: David Xia Date: Thu, 15 May 2025 19:34:39 +0000 Subject: [PATCH 1/3] [Bugfix] make `test_openai_schema.py` pass by filtering out test cases to `POST /tokenize` endpoint that are known to fail with a reply of HTTP 501 Not Implemented. Enable the test in CI. FIX #18162 Signed-off-by: David Xia --- .buildkite/test-pipeline.yaml | 2 +- .../entrypoints/openai/test_openai_schema.py | 45 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml index 4c2da527de79..0e4a0e2a531b 100644 --- a/.buildkite/test-pipeline.yaml +++ b/.buildkite/test-pipeline.yaml @@ -126,7 +126,7 @@ steps: - pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process - pytest -v -s entrypoints/llm/test_generate_multiple_loras.py # it needs a clean process - VLLM_USE_V1=0 pytest -v -s entrypoints/llm/test_guided_generate.py # it needs a clean process - - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/test_openai_schema.py + - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/correctness/ - pytest -v -s entrypoints/test_chat_utils.py - VLLM_USE_V1=0 pytest -v -s entrypoints/offline_mode # Needs to avoid interference with other tests diff --git a/tests/entrypoints/openai/test_openai_schema.py b/tests/entrypoints/openai/test_openai_schema.py index 5c585d54c429..ed148627bed2 100644 --- a/tests/entrypoints/openai/test_openai_schema.py +++ b/tests/entrypoints/openai/test_openai_schema.py @@ -42,6 +42,51 @@ def get_schema(server): schema = schemathesis.from_pytest_fixture("get_schema") +@schemathesis.hook +def before_generate_case(context: schemathesis.hooks.HookContext, strategy): + op = context.operation + assert op is not None + + def no_file_type(case: schemathesis.models.Case): + """ + This filter skips test cases for the `POST /tokenize` endpoint where the + HTTP request body uses `"type": "file"` in any message's content. + We expect these cases to fail because that type isn't implemented here + https://github.com/vllm-project/vllm/blob/0b34593017953051b3225b1483ce0f4670e3eb0e/vllm/entrypoints/chat_utils.py#L1038-L1095 + + Example test cases that are skipped: + curl -X POST -H 'Content-Type: application/json' \ + -d '{"messages": [{"role": "assistant"}, {"content": [{"file": {}, "type": "file"}], "role": "user"}]}' \ + http://localhost:8000/tokenize + + curl -X POST -H 'Content-Type: application/json' \ + -d '{"messages": [{"content": [{"file": {}, "type": "file"}], "role": "user"}]}' \ + http://localhost:8000/tokenize + """ # noqa: E501 + if op.method.lower() != "post" or op.path != "/tokenize": + return True + + if case.body and isinstance(case.body, dict): + if "messages" not in case.body: + return True + + messages = case.body.get("messages", []) + if not isinstance(messages, list) or len(messages) == 0: + return True + + for message in messages: + if not isinstance(message, dict): + continue + content = message.get("content", []) + if not isinstance(content, list) or len(content) == 0: + continue + if any(item.get("type") == "file" for item in content): + return False + return True + + return strategy.filter(no_file_type) + + @schema.parametrize() @schema.override(headers={"Content-Type": "application/json"}) def test_openapi_stateless(case: schemathesis.Case): From 4001d7350b1776c3b13a9abc762745267c770b9a Mon Sep 17 00:00:00 2001 From: David Xia Date: Tue, 20 May 2025 12:56:52 +0000 Subject: [PATCH 2/3] [Bugfix] increase timeout of POST /v1/chat/completions to 60s Increase both Schemathesis and Hypothesis timeouts (they are configured separately) to 60s to allow the test to pass comfortably in both CI and in local testing. We are interested in testing the OpenAPI schema here not performance. Signed-off-by: David Xia --- tests/entrypoints/openai/test_openai_schema.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/entrypoints/openai/test_openai_schema.py b/tests/entrypoints/openai/test_openai_schema.py index ed148627bed2..655e7e27bf93 100644 --- a/tests/entrypoints/openai/test_openai_schema.py +++ b/tests/entrypoints/openai/test_openai_schema.py @@ -1,6 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 +from typing import Final + import pytest import schemathesis +from hypothesis import settings from schemathesis import GenerationConfig from ...utils import RemoteOpenAIServer @@ -9,6 +12,7 @@ MODEL_NAME = "HuggingFaceTB/SmolVLM-256M-Instruct" MAXIMUM_IMAGES = 2 +DEFAULT_TIMEOUT_SECONDS: Final[int] = 10 @pytest.fixture(scope="module") @@ -89,6 +93,15 @@ def no_file_type(case: schemathesis.models.Case): @schema.parametrize() @schema.override(headers={"Content-Type": "application/json"}) +@settings(deadline=60000) def test_openapi_stateless(case: schemathesis.Case): + key = ( + case.operation.method.upper(), + case.operation.path, + ) + timeout = { + ("POST", "/v1/chat/completions"): 60, + }.get(key, DEFAULT_TIMEOUT_SECONDS) + #No need to verify SSL certificate for localhost - case.call_and_validate(verify=False) + case.call_and_validate(verify=False, timeout=timeout) From 1c7eacf0357599da40829f31e0ecf1bd1e41ca3a Mon Sep 17 00:00:00 2001 From: David Xia Date: Thu, 22 May 2025 08:00:59 -0400 Subject: [PATCH 3/3] Apply suggestions from code review Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: David Xia --- .../entrypoints/openai/test_openai_schema.py | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/tests/entrypoints/openai/test_openai_schema.py b/tests/entrypoints/openai/test_openai_schema.py index 655e7e27bf93..cae2a3b59553 100644 --- a/tests/entrypoints/openai/test_openai_schema.py +++ b/tests/entrypoints/openai/test_openai_schema.py @@ -13,6 +13,7 @@ MODEL_NAME = "HuggingFaceTB/SmolVLM-256M-Instruct" MAXIMUM_IMAGES = 2 DEFAULT_TIMEOUT_SECONDS: Final[int] = 10 +LONG_TIMEOUT_SECONDS: Final[int] = 60 @pytest.fixture(scope="module") @@ -67,18 +68,12 @@ def no_file_type(case: schemathesis.models.Case): -d '{"messages": [{"content": [{"file": {}, "type": "file"}], "role": "user"}]}' \ http://localhost:8000/tokenize """ # noqa: E501 - if op.method.lower() != "post" or op.path != "/tokenize": - return True - - if case.body and isinstance(case.body, dict): - if "messages" not in case.body: - return True - - messages = case.body.get("messages", []) - if not isinstance(messages, list) or len(messages) == 0: - return True - - for message in messages: + if (op.method.lower() == "post" and op.path == "/tokenize" + and hasattr(case, "body") and isinstance(case.body, dict) + and "messages" in case.body + and isinstance(case.body["messages"], list) + and len(case.body["messages"]) > 0): + for message in case.body["messages"]: if not isinstance(message, dict): continue content = message.get("content", []) @@ -93,14 +88,16 @@ def no_file_type(case: schemathesis.models.Case): @schema.parametrize() @schema.override(headers={"Content-Type": "application/json"}) -@settings(deadline=60000) +@settings(deadline=LONG_TIMEOUT_SECONDS * 1000) def test_openapi_stateless(case: schemathesis.Case): key = ( case.operation.method.upper(), case.operation.path, ) timeout = { - ("POST", "/v1/chat/completions"): 60, + # requires a longer timeout + ("POST", "/v1/chat/completions"): + LONG_TIMEOUT_SECONDS, }.get(key, DEFAULT_TIMEOUT_SECONDS) #No need to verify SSL certificate for localhost