Adding more lcore skips and fixing some tests to pass with lcore

Author: JoaoFula

tests/e2e/test_api.py

@@ -135,7 +135,6 @@ def test_one_default_model_provider():
     ), "one model and provider should be selected as default"
 
 
-@pytest.mark.skip_with_lcore
 @pytest.mark.cluster
 def test_improper_token():
     """Test accessing /v1/query endpoint using improper auth. token."""
@@ -145,7 +144,10 @@ def test_improper_token():
         timeout=NON_LLM_REST_API_TIMEOUT,
         headers={"Authorization": "Bearer wrong-token"},
     )
-    assert response.status_code == requests.codes.forbidden
+    if os.getenv("LCORE", "False").lower() not in ("true", "1", "t"):
+        assert response.status_code == requests.codes.forbidden
+    else:
+        assert response.status_code == requests.codes.unauthorized
 
 
 @pytest.mark.skip_with_lcore

tests/e2e/test_attachments.py

@@ -6,6 +6,7 @@
 
 import pytest
 import requests
+import os
 
 from tests.e2e.utils import metrics as metrics_utils
 from tests.e2e.utils import response as response_utils
@@ -202,7 +203,7 @@ def test_valid_question_with_wrong_attachment_format_field_of_different_type() -
         assert details["msg"] == "Input should be a valid string"
         assert details["type"] == "string_type"
 
-
+import ipdb
 @retry(max_attempts=3, wait_between_runs=10)
 def test_valid_question_with_wrong_attachment_format_unknown_attachment_type() -> None:
     """Check the REST API /v1/query with POST HTTP method using attachment with wrong type."""
@@ -231,16 +232,26 @@ def test_valid_question_with_wrong_attachment_format_unknown_attachment_type() -
 
         # the attachment should not be processed correctly
         assert response.status_code == requests.codes.unprocessable_entity
-
         json_response = response.json()
-        expected_response = {
-            "detail": {
-                "response": "Invalid attribute value",
-                "cause": "Invalid attatchment type unknown_type: must be one of frozenset" \
-                "({'alert', 'log', 'event', 'api object', 'error message', 'configuration', 'stack trace'})",
+        if os.getenv("LCORE", "False").lower() not in ("true", "1", "t"):
+            expected_response = {
+                "detail": {
+                    "response": "Invalid attribute value",
+                    "cause": "Invalid attatchment type unknown_type: must be one of frozenset" \
+                    "({'alert', 'log', 'event', 'api object', 'error message', 'configuration', 'stack trace'})",
+                }
             }
-        }
-        assert json_response == expected_response
+            assert json_response == expected_response
+        else:
+            assert "Invalid attribute value" in json_response["detail"]["response"]
+            assert "Invalid attatchment type unknown_type: must be one of frozenset" in json_response["detail"]["cause"]
+            assert "event" in json_response["detail"]["cause"]
+            assert "log" in json_response["detail"]["cause"]
+            assert "stack trace" in json_response["detail"]["cause"]
+            assert "alert" in json_response["detail"]["cause"]
+            assert "configuration" in json_response["detail"]["cause"]
+            assert "api object" in json_response["detail"]["cause"]
+            assert "error message" in json_response["detail"]["cause"]
 
 
 @retry(max_attempts=3, wait_between_runs=10)
@@ -271,13 +282,9 @@ def test_valid_question_with_wrong_attachment_format_unknown_content_type() -> N
 
         # the attachment should not be processed correctly
         assert response.status_code == requests.codes.unprocessable_entity
-
         json_response = response.json()
-        expected_response = {
-            "detail": {
-                "response": "Invalid attribute value",
-                "cause": "Invalid attatchment content type unknown/type: must be one of frozenset" \
-                "({'application/json', 'application/xml', 'application/yaml', 'text/plain'})",
-            }
-        }
-        assert json_response == expected_response
+        assert "Invalid attribute value" in json_response["detail"]["response"]
+        assert "application/json" in json_response["detail"]["cause"]
+        assert "application/xml" in json_response["detail"]["cause"]
+        assert "application/yaml" in json_response["detail"]["cause"]
+        assert "text/plain" in json_response["detail"]["cause"]

tests/e2e/test_query_endpoint.py

@@ -207,29 +207,42 @@ def test_too_long_question() -> None:
         assert json_response["detail"]["response"] == "Prompt is too long"
 
 
-@pytest.mark.skip_with_lcore
 @pytest.mark.smoketest
 @pytest.mark.rag
 def test_valid_question() -> None:
     """Check the REST API /v1/query with POST HTTP method for valid question and no yaml."""
     with metrics_utils.RestAPICallCounterChecker(pytest.metrics_client, QUERY_ENDPOINT):
-        cid = suid.get_suid()
-        response = pytest.client.post(
-            QUERY_ENDPOINT,
-            json={
-                "conversation_id": cid,
-                "query": "what is kubernetes in the context of OpenShift?",
-            },
-            timeout=test_api.LLM_REST_API_TIMEOUT,
-        )
-        assert response.status_code == requests.codes.ok
-
-        response_utils.check_content_type(response, "application/json")
-        print(vars(response))
-        json_response = response.json()
-
-        # checking a few major information from response
-        assert json_response["conversation_id"] == cid
+        if os.getenv("LCORE", "False").lower() not in ("true", "1", "t"):
+            cid = suid.get_suid()
+            response = pytest.client.post(
+                QUERY_ENDPOINT,
+                json={
+                    "conversation_id": cid,
+                    "query": "what is kubernetes in the context of OpenShift?",
+                },
+                timeout=test_api.LLM_REST_API_TIMEOUT,
+            )
+            assert response.status_code == requests.codes.ok
+
+            response_utils.check_content_type(response, "application/json")
+            print(vars(response))
+            json_response = response.json()
+
+            # checking a few major information from response
+            assert json_response["conversation_id"] == cid
+        else:
+            response = pytest.client.post(
+                QUERY_ENDPOINT,
+                json={
+                    "query": "what is kubernetes in the context of OpenShift?",
+                },
+                timeout=test_api.LLM_REST_API_TIMEOUT,
+            )
+            assert response.status_code == requests.codes.ok
+
+            response_utils.check_content_type(response, "application/json")
+            print(vars(response))
+            json_response = response.json()
         assert re.search(
             r"kubernetes|openshift",
             json_response["response"],
@@ -474,6 +487,7 @@ def test_conversation_history() -> None:
         assert "ingress" in response_text, debug_msg
 
 
+@pytest.mark.skip_with_lcore
 def test_query_with_provider_but_not_model() -> None:
     """Check the REST API /v1/query with POST HTTP method for provider specified, but no model."""
     with metrics_utils.RestAPICallCounterChecker(
@@ -503,6 +517,7 @@ def test_query_with_provider_but_not_model() -> None:
         )
 
 
+@pytest.mark.skip_with_lcore
 def test_query_with_model_but_not_provider() -> None:
     """Check the REST API /v1/query with POST HTTP method for model specified, but no provider."""
     with metrics_utils.RestAPICallCounterChecker(
@@ -531,6 +546,7 @@ def test_query_with_model_but_not_provider() -> None:
         )
 
 
+@pytest.mark.skip_with_lcore
 def test_query_with_unknown_provider() -> None:
     """Check the REST API /v1/query with POST HTTP method for unknown provider specified."""
     # retrieve currently selected model
@@ -568,6 +584,7 @@ def test_query_with_unknown_provider() -> None:
         )
 
 
+@pytest.mark.skip_with_lcore
 def test_query_with_unknown_model() -> None:
     """Check the REST API /v1/query with POST HTTP method for unknown model specified."""
     # retrieve currently selected provider

tests/e2e/test_streaming_query_endpoint.py

@@ -6,6 +6,7 @@
 
 import json
 import re
+import os
 
 import pytest
 import requests
@@ -53,6 +54,7 @@ def construct_response_from_streamed_events(events: dict) -> str:
     return response
 
 
+@pytest.mark.skip_with_lcore
 def test_invalid_question():
     """Check the endpoint POST method for invalid question."""
     with metrics_utils.RestAPICallCounterChecker(
@@ -98,7 +100,8 @@ def test_invalid_question_without_conversation_id():
         # new conversation ID should be generated
         assert events[0]["event"] == "start"
         assert events[0]["data"]
-        assert suid.check_suid(events[0]["data"]["conversation_id"])
+        if os.getenv("LCORE", "False").lower() not in ("true", "1", "t"):
+            assert suid.check_suid(events[0]["data"]["conversation_id"])
 
 
 def test_query_call_without_payload():
@@ -139,6 +142,7 @@ def test_query_call_with_improper_payload():
         assert "missing" in response.text
 
 
+@pytest.mark.skip_with_lcore
 def test_valid_question_improper_conversation_id() -> None:
     """Check the endpoint with POST HTTP method for improper conversation ID."""
     with metrics_utils.RestAPICallCounterChecker(
@@ -163,6 +167,7 @@ def test_valid_question_improper_conversation_id() -> None:
         assert json_response == expected_response
 
 
+@pytest.mark.skip_with_lcore
 def test_too_long_question() -> None:
     """Check the endpoint with too long question."""
     # let's make the query really large, larger that context window size
@@ -200,11 +205,24 @@ def test_valid_question() -> None:
     with metrics_utils.RestAPICallCounterChecker(
         pytest.metrics_client, STREAMING_QUERY_ENDPOINT
     ):
-        cid = suid.get_suid()
-        response = post_with_defaults(
-            STREAMING_QUERY_ENDPOINT,
-            json={"conversation_id": cid, "query": "what is kubernetes?"},
-        )
+        if os.getenv("LCORE", "False").lower() not in ("true", "1", "t"):
+            cid = suid.get_suid()
+            response = pytest.client.post(
+                STREAMING_QUERY_ENDPOINT,
+                json={
+                    "conversation_id": cid,
+                    "query": "what is kubernetes in the context of OpenShift?",
+                },
+                timeout=test_api.LLM_REST_API_TIMEOUT,
+            )
+        else:
+            response = pytest.client.post(
+                STREAMING_QUERY_ENDPOINT,
+                json={
+                    "query": "what is kubernetes in the context of OpenShift?",
+                },
+                timeout=test_api.LLM_REST_API_TIMEOUT,
+            )
         assert response.status_code == requests.codes.ok
 
         response_utils.check_content_type(response, constants.MEDIA_TYPE_TEXT)
@@ -244,6 +262,7 @@ def test_ocp_docs_version_same_as_cluster_version() -> None:
         )
 
 
+@pytest.mark.skip_with_lcore
 def test_valid_question_tokens_counter() -> None:
     """Check how the tokens counter are updated accordingly."""
     model, provider = metrics_utils.get_enabled_model_and_provider(
@@ -264,6 +283,7 @@ def test_valid_question_tokens_counter() -> None:
         response_utils.check_content_type(response, constants.MEDIA_TYPE_TEXT)
 
 
+@pytest.mark.skip_with_lcore
 def test_invalid_question_tokens_counter() -> None:
     """Check how the tokens counter are updated accordingly."""
     model, provider = metrics_utils.get_enabled_model_and_provider(
@@ -284,6 +304,7 @@ def test_invalid_question_tokens_counter() -> None:
         response_utils.check_content_type(response, constants.MEDIA_TYPE_TEXT)
 
 
+@pytest.mark.skip_with_lcore
 def test_token_counters_for_query_call_without_payload() -> None:
     """Check how the tokens counter are updated accordingly."""
     model, provider = metrics_utils.get_enabled_model_and_provider(
@@ -311,6 +332,7 @@ def test_token_counters_for_query_call_without_payload() -> None:
         response_utils.check_content_type(response, constants.MEDIA_TYPE_JSON)
 
 
+@pytest.mark.skip_with_lcore
 def test_token_counters_for_query_call_with_improper_payload() -> None:
     """Check how the tokens counter are updated accordingly."""
     model, provider = metrics_utils.get_enabled_model_and_provider(
@@ -372,6 +394,7 @@ def test_rag_question() -> None:
         assert len(set(docs_urls)) == len(docs_urls)
 
 
+@pytest.mark.skip_with_lcore
 @pytest.mark.cluster
 def test_query_filter() -> None:
     """Ensure responses does not include filtered words and redacted words are not logged."""
@@ -463,6 +486,7 @@ def test_conversation_history() -> None:
         assert "ingress" in response_text, scenario_fail_msg
 
 
+@pytest.mark.skip_with_lcore
 def test_query_with_provider_but_not_model() -> None:
     """Check the endpoint with POST HTTP method for provider specified, but no model."""
     with metrics_utils.RestAPICallCounterChecker(
@@ -491,6 +515,7 @@ def test_query_with_provider_but_not_model() -> None:
         )
 
 
+@pytest.mark.skip_with_lcore
 def test_query_with_model_but_not_provider() -> None:
     """Check the endpoint with POST HTTP method for model specified, but no provider."""
     with metrics_utils.RestAPICallCounterChecker(
@@ -518,6 +543,7 @@ def test_query_with_model_but_not_provider() -> None:
         )
 
 
+@pytest.mark.skip_with_lcore
 def test_query_with_unknown_provider() -> None:
     """Check the endpoint with POST HTTP method for unknown provider specified."""
     # retrieve currently selected model
@@ -554,6 +580,7 @@ def test_query_with_unknown_provider() -> None:
         )
 
 
+@pytest.mark.skip_with_lcore
 def test_query_with_unknown_model() -> None:
     """Check the endpoint with POST HTTP method for unknown model specified."""
     # retrieve currently selected provider

tests/e2e/test_user_feedback.py

@@ -6,6 +6,7 @@
 
 import pytest
 import requests
+import os
 
 from tests.e2e.utils import cluster as cluster_utils
 from tests.e2e.utils import response as response_utils
@@ -27,7 +28,10 @@ def test_feedback_can_post_with_wrong_token():
         timeout=test_api.BASIC_ENDPOINTS_TIMEOUT,
         headers={"Authorization": "Bearer wrong-token"},
     )
-    assert response.status_code == requests.codes.forbidden
+    if os.getenv("LCORE", "False").lower() not in ("true", "1", "t"):
+        assert response.status_code == requests.codes.forbidden
+    else:
+        assert response.status_code == requests.codes.unauthorized
 
 
 @pytest.mark.data_export

tests/e2e/utils/metrics.py

@@ -92,12 +92,10 @@ def get_metric_labels(lines, info_node_name, value=None) -> dict:
     # info node was not found
     return {}
 
-import ipdb
+
 def get_enabled_model_and_provider(client):
     """Read configured model and provider from metrics."""
 
-    ipdb.set_trace()
-    
     if os.getenv("LCORE", 'False').lower() in ('true', '1', 't'):
         response = client.get("/v1/models", timeout=BASIC_ENDPOINTS_TIMEOUT)
         assert response.status_code == requests.codes.ok