diff --git a/tests/e2e/configuration/lightspeed-stack-auth-noop-token.yaml b/tests/e2e/configuration/lightspeed-stack-auth-noop-token.yaml new file mode 100644 index 00000000..1cb8fcf8 --- /dev/null +++ b/tests/e2e/configuration/lightspeed-stack-auth-noop-token.yaml @@ -0,0 +1,25 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Uses a remote llama-stack service + # The instance would have already been started with a llama-stack-run.yaml file + use_as_library_client: false + # Alternative for "as library use" + # use_as_library_client: true + # library_client_config_path: + url: http://llama-stack:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" + +authentication: + module: "noop-with-token" diff --git a/tests/e2e/features/authorized.feature b/tests/e2e/features/authorized.feature deleted file mode 100644 index b92b3a7e..00000000 --- a/tests/e2e/features/authorized.feature +++ /dev/null @@ -1,34 +0,0 @@ -# Feature: Authorized endpoint API tests -# TODO: fix test - -# Background: -# Given The service is started locally -# And REST API service hostname is localhost -# And REST API service port is 8080 -# And REST API service prefix is /v1 - -# Scenario: Check if the OpenAPI endpoint works as expected -# Given The system is in default state -# When I access endpoint "authorized" using HTTP POST method -# Then The status code of the response is 200 -# And The body of the response has proper username - -# Scenario: Check if LLM responds to sent question with error when not authenticated -# Given The system is in default state -# And I remove the auth header -# When I access endpoint "authorized" using HTTP POST method -# Then The status code of the response is 400 -# And The body of the response is the following -# """ -# {"detail": "Unauthorized: No auth header found"} -# """ - -# Scenario: Check if LLM responds to sent question with error when not authorized -# Given The system is in default state -# And I modify the auth header so that the user is it authorized -# When I access endpoint "authorized" using HTTP POST method -# Then The status code of the response is 403 -# And The body of the response is the following -# """ -# {"detail": "Forbidden: User is not authorized to access this resource"} -# """ diff --git a/tests/e2e/features/authorized_noop.feature b/tests/e2e/features/authorized_noop.feature new file mode 100644 index 00000000..9ce63782 --- /dev/null +++ b/tests/e2e/features/authorized_noop.feature @@ -0,0 +1,58 @@ +Feature: Authorized endpoint API tests for the noop authentication module + + Background: + Given The service is started locally + And REST API service hostname is localhost + And REST API service port is 8080 + And REST API service prefix is /v1 + + Scenario: Check if the authorized endpoint works fine when user_id and auth header are not provided + Given The system is in default state + When I access endpoint "authorized" using HTTP POST method + """ + {"placeholder":"abc"} + """ + Then The status code of the response is 200 + And The body of the response is the following + """ + {"user_id": "00000000-0000-0000-0000-000","username": "lightspeed-user"} + """ + + Scenario: Check if the authorized endpoint works when auth token is not provided + Given The system is in default state + When I access endpoint "authorized" using HTTP POST method with user_id "test_user" + Then The status code of the response is 200 + And The body of the response is the following + """ + {"user_id": "test_user","username": "lightspeed-user"} + """ + + Scenario: Check if the authorized endpoint works when user_id is not provided + Given The system is in default state + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + When I access endpoint "authorized" using HTTP POST method without user_id + Then The status code of the response is 200 + And The body of the response is the following + """ + {"user_id": "00000000-0000-0000-0000-000","username": "lightspeed-user"} + """ + + Scenario: Check if the authorized endpoint works when providing empty user_id + Given The system is in default state + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + When I access endpoint "authorized" using HTTP POST method with user_id "" + Then The status code of the response is 200 + And The body of the response is the following + """ + {"user_id": "","username": "lightspeed-user"} + """ + + Scenario: Check if the authorized endpoint works when providing proper user_id + Given The system is in default state + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + When I access endpoint "authorized" using HTTP POST method with user_id "test_user" + Then The status code of the response is 200 + And The body of the response is the following + """ + {"user_id": "test_user","username": "lightspeed-user"} + """ \ No newline at end of file diff --git a/tests/e2e/features/authorized_noop_token.feature b/tests/e2e/features/authorized_noop_token.feature new file mode 100644 index 00000000..a57d2211 --- /dev/null +++ b/tests/e2e/features/authorized_noop_token.feature @@ -0,0 +1,69 @@ +@Authorized +Feature: Authorized endpoint API tests for the noop-with-token + + Background: + Given The service is started locally + And REST API service hostname is localhost + And REST API service port is 8080 + And REST API service prefix is /v1 + + Scenario: Check if the authorized endpoint fails when user_id and auth header are not provided + Given The system is in default state + When I access endpoint "authorized" using HTTP POST method + """ + {"placeholder":"abc"} + """ + Then The status code of the response is 400 + And The body of the response is the following + """ + {"detail": "No Authorization header found"} + """ + + Scenario: Check if the authorized endpoint works when user_id is not provided + Given The system is in default state + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + When I access endpoint "authorized" using HTTP POST method without user_id + Then The status code of the response is 200 + And The body of the response is the following + """ + {"user_id": "00000000-0000-0000-0000-000","username": "lightspeed-user"} + """ + + Scenario: Check if the authorized endpoint works when providing empty user_id + Given The system is in default state + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + When I access endpoint "authorized" using HTTP POST method with user_id "" + Then The status code of the response is 200 + And The body of the response is the following + """ + {"user_id": "","username": "lightspeed-user"} + """ + + Scenario: Check if the authorized endpoint works when providing proper user_id + Given The system is in default state + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + When I access endpoint "authorized" using HTTP POST method with user_id "test_user" + Then The status code of the response is 200 + And The body of the response is the following + """ + {"user_id": "test_user","username": "lightspeed-user"} + """ + + Scenario: Check if the authorized endpoint works with proper user_id but bearer token is not present + Given The system is in default state + When I access endpoint "authorized" using HTTP POST method with user_id "test_user" + Then The status code of the response is 400 + And The body of the response is the following + """ + {"detail": "No Authorization header found"} + """ + + Scenario: Check if the authorized endpoint works when auth token is malformed + Given The system is in default state + And I set the Authorization header to BearereyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + When I access endpoint "authorized" using HTTP POST method with user_id "test_user" + Then The status code of the response is 400 + And The body of the response is the following + """ + {"detail": "No token found in Authorization header"} + """ \ No newline at end of file diff --git a/tests/e2e/features/environment.py b/tests/e2e/features/environment.py index 395cdb31..2c1cbfc4 100644 --- a/tests/e2e/features/environment.py +++ b/tests/e2e/features/environment.py @@ -9,9 +9,11 @@ import subprocess import time -from behave.model import Scenario +from behave.model import Scenario, Feature from behave.runner import Context +from tests.e2e.utils.utils import switch_config_and_restart + try: import os # noqa: F401 except ImportError as e: @@ -82,5 +84,22 @@ def after_scenario(context: Context, scenario: Scenario) -> None: print(f"Warning: Could not restore Llama Stack connection: {e}") -def before_feature(context: Context, feature: Scenario) -> None: +def before_feature(context: Context, feature: Feature) -> None: """Run before each feature file is exercised.""" + if "Authorized" in feature.tags: + context.backup_file = switch_config_and_restart( + "lightspeed-stack.yaml", + "tests/e2e/configuration/lightspeed-stack-auth-noop-token.yaml", + "lightspeed-stack", + ) + + +def after_feature(context: Context, feature: Feature) -> None: + """Run after each feature file is exercised.""" + if "Authorized" in feature.tags: + switch_config_and_restart( + "lightspeed-stack.yaml", + context.backup_file, + "lightspeed-stack", + cleanup=True, + ) diff --git a/tests/e2e/features/steps/auth.py b/tests/e2e/features/steps/auth.py index 2db9aee4..64b5cb00 100644 --- a/tests/e2e/features/steps/auth.py +++ b/tests/e2e/features/steps/auth.py @@ -1,25 +1,60 @@ """Implementation of common test steps.""" -from behave import given, then # pyright: ignore[reportAttributeAccessIssue] +import requests +from behave import given, when # pyright: ignore[reportAttributeAccessIssue] from behave.runner import Context +from tests.e2e.utils.utils import normalize_endpoint -@then("The body of the response has proper username") -def check_body_username(context: Context) -> None: - """Check that the username is correct in response.""" - # TODO: add step implementation - assert context is not None +@given("I set the Authorization header to {header_value}") +def set_authorization_header_custom(context: Context, header_value: str) -> None: + """Set a custom Authorization header value.""" + if not hasattr(context, "auth_headers"): + context.auth_headers = {} + context.auth_headers["Authorization"] = header_value + print(f"🔑 Set Authorization header to: {header_value}") -@given("I remove the auth header") -def remove_auth_header(context: Context) -> None: - """Remove the auth header.""" - # TODO: add step implementation - assert context is not None +@when("I access endpoint {endpoint} using HTTP POST method with user_id {user_id}") +def access_rest_api_endpoint_post( + context: Context, endpoint: str, user_id: str +) -> None: + """Send POST HTTP request with payload in the endpoint as parameter to tested service. + The response is stored in `context.response` attribute. + """ + endpoint = normalize_endpoint(endpoint) + user_id = user_id.replace('"', "") + base = f"http://{context.hostname}:{context.port}" + path = f"{endpoint}?user_id={user_id}".replace("//", "/") + url = base + path -@given("I modify the auth header so that the user is it authorized") -def modify_auth_header(context: Context) -> None: - """Modify the auth header making the user unauthorized.""" - # TODO: add step implementation - assert context is not None + if not hasattr(context, "auth_headers"): + context.auth_headers = {} + + # perform REST API call + context.response = requests.post( + url, json="", headers=context.auth_headers, timeout=10 + ) + + +@when("I access endpoint {endpoint} using HTTP POST method without user_id") +def access_rest_api_endpoint_post_without_param( + context: Context, endpoint: str +) -> None: + """Send POST HTTP request without user_id payload. + + The response is stored in `context.response` attribute. + """ + endpoint = normalize_endpoint(endpoint) + base = f"http://{context.hostname}:{context.port}" + path = f"{endpoint}".replace("//", "/") + url = base + path + + if not hasattr(context, "auth_headers"): + context.auth_headers = {} + + # perform REST API call + context.response = requests.post( + url, json="", headers=context.auth_headers, timeout=10 + ) diff --git a/tests/e2e/features/steps/common_http.py b/tests/e2e/features/steps/common_http.py index 261b0dfd..54909634 100644 --- a/tests/e2e/features/steps/common_http.py +++ b/tests/e2e/features/steps/common_http.py @@ -241,7 +241,7 @@ def access_rest_api_endpoint_get(context: Context, endpoint: str) -> None: context.response = requests.get(url, timeout=DEFAULT_TIMEOUT) -@when("I access endpoint {endpoint:w} using HTTP POST method") +@when("I access endpoint {endpoint} using HTTP POST method") def access_rest_api_endpoint_post(context: Context, endpoint: str) -> None: """Send POST HTTP request with JSON payload to tested service. @@ -249,8 +249,9 @@ def access_rest_api_endpoint_post(context: Context, endpoint: str) -> None: which must not be None. The response is stored in `context.response` attribute. """ + endpoint = normalize_endpoint(endpoint) base = f"http://{context.hostname}:{context.port}" - path = f"{context.api_prefix}/{endpoint}".replace("//", "/") + path = f"{endpoint}".replace("//", "/") url = base + path assert context.text is not None, "Payload needs to be specified" diff --git a/tests/e2e/test_list.txt b/tests/e2e/test_list.txt index 5a6d1ed1..9d7cd0c8 100644 --- a/tests/e2e/test_list.txt +++ b/tests/e2e/test_list.txt @@ -1,5 +1,6 @@ features/smoketests.feature -features/authorized.feature +features/authorized_noop.feature +features/authorized_noop_token.feature features/conversations.feature features/feedback.feature features/health.feature diff --git a/tests/e2e/utils/utils.py b/tests/e2e/utils/utils.py index f1e77b27..2ace9b4c 100644 --- a/tests/e2e/utils/utils.py +++ b/tests/e2e/utils/utils.py @@ -1,5 +1,9 @@ """Unsorted utility functions to be used from other sources and test step definitions.""" +import os +import shutil +import subprocess +import time from typing import Any import jsonschema @@ -26,3 +30,60 @@ def validate_json(message: Any, schema: Any) -> None: except jsonschema.SchemaError as e: assert False, "The provided schema is faulty:" + str(e) + + +def switch_config_and_restart( + original_file: str, + replacement_file: str, + container_name: str, + cleanup: bool = False, +) -> str: + """Switch configuration file and restart container. + + Args: + original_file: Path to the original configuration file + replacement_file: Path to the replacement configuration file + container_name: Name of the container to restart + cleanup: If True, remove the backup file after restoration (default: False) + + Returns: + str: Path to the backup file for restoration + """ + backup_file = f"{original_file}.backup" + + if not cleanup and not os.path.exists(backup_file): + try: + shutil.copy(original_file, backup_file) + except (FileNotFoundError, PermissionError, OSError) as e: + print(f"Failed to create backup: {e}") + raise + + try: + shutil.copy(replacement_file, original_file) + except (FileNotFoundError, PermissionError, OSError) as e: + print(f"Failed to copy replacement file: {e}") + raise + + # Restart container + try: + subprocess.run( + ["docker", "restart", container_name], + capture_output=True, + text=True, + check=True, + ) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: + print(f"Failed to restart container {container_name}: {e.stderr}") + raise + + # Wait for container to be ready + time.sleep(5) + + # Clean up backup file + if cleanup and os.path.exists(backup_file): + try: + os.remove(backup_file) + except OSError as e: + print(f"Warning: Could not remove backup file {backup_file}: {e}") + + return backup_file