""" Tests for OpenAI Responses API client implementation. The Responses API is designed specifically for GPT-5 and provides: - Chain-of-thought preservation between conversation turns - Reduced reasoning token generation through context reuse - Improved cache hit rates and lower latency - Better integration with GPT-5 reasoning features These tests validate the Responses API client implementation, parameter handling, and integration with AutoGen frameworks. """ from typing import Any, Dict, cast from unittest.mock import AsyncMock, patch import pytest from autogen_core import CancellationToken from autogen_core.models import CreateResult from autogen_ext.models.openai import ( AzureOpenAIResponsesAPIClient, OpenAIResponsesAPIClient, ) from autogen_ext.models.openai._responses_client import ( ResponsesAPICreateParams, ) from test_gpt5_features import TestCodeExecutorTool class TestResponsesAPIClientInitialization: """Test Responses API client initialization and configuration.""" def test_openai_responses_client_creation(self) -> None: """Test OpenAI Responses API client can be created.""" with patch("autogen_ext.models.openai._responses_client._openai_client_from_config") as mock: mock.return_value = AsyncMock() client = OpenAIResponsesAPIClient(model="gpt-5", api_key="test-key") # Access through public info() for type safety assert client.info()["family"] == "GPT_5" def test_azure_responses_client_creation(self) -> None: """Test Azure OpenAI Responses API client can be created.""" with patch("autogen_ext.models.openai._responses_client._azure_openai_client_from_config") as mock: mock.return_value = AsyncMock() client = AzureOpenAIResponsesAPIClient( model="gpt-5", azure_endpoint="https://test.openai.azure.com/", azure_deployment="gpt-5-deployment", api_version="2024-06-01", api_key="test-key", ) assert client.info()["family"] == "GPT_5" def test_invalid_model_raises_error(self) -> None: """Test that invalid model names raise appropriate errors.""" with patch("autogen_ext.models.openai._responses_client._openai_client_from_config") as mock: mock.return_value = AsyncMock() with pytest.raises(ValueError, match="model_info is required"): OpenAIResponsesAPIClient(model="invalid-model", api_key="test-key") class TestResponsesAPIParameterHandling: """Test Responses API specific parameter handling.""" @pytest.fixture def mock_openai_client(self) -> Any: with patch("autogen_ext.models.openai._responses_client._openai_client_from_config") as mock: mock_client = AsyncMock() mock_client.responses.create = AsyncMock() mock.return_value = mock_client yield mock_client @pytest.fixture def client(self, mock_openai_client: Any) -> OpenAIResponsesAPIClient: return OpenAIResponsesAPIClient(model="gpt-5", api_key="test-key") def test_process_create_args_basic(self, client: OpenAIResponsesAPIClient) -> None: """Test basic parameter processing for Responses API.""" params = client._OpenAIResponsesAPIClient__process_create_args( # type: ignore[attr-defined] input="Test input", tools=[], tool_choice="auto", extra_create_args={}, reasoning_effort="medium", verbosity="high", preambles=True, ) assert isinstance(params, ResponsesAPICreateParams) assert params.input == "Test input" assert params.create_args["input"] == "Test input" assert params.create_args["reasoning"]["effort"] == "medium" assert params.create_args["text"]["verbosity"] == "high" assert params.create_args["preambles"] is True def test_process_create_args_with_cot_preservation(self, client: OpenAIResponsesAPIClient) -> None: """Test chain-of-thought preservation parameters.""" params = client._OpenAIResponsesAPIClient__process_create_args( # type: ignore[attr-defined] input="Follow-up question", tools=[], tool_choice="auto", extra_create_args={}, previous_response_id="resp-123", reasoning_items=[{"type": "reasoning", "content": "Previous reasoning"}], ) # mypy/pyright: create_args is a dict[str, Any] create_args: Dict[str, Any] = params.create_args assert create_args.get("previous_response_id") == "resp-123" assert create_args.get("reasoning_items") == [{"type": "reasoning", "content": "Previous reasoning"}] def test_invalid_extra_args_rejected(self, client: OpenAIResponsesAPIClient) -> None: """Test that invalid extra arguments are rejected.""" with pytest.raises(ValueError, match="Extra create args are invalid for Responses API"): client._OpenAIResponsesAPIClient__process_create_args( # type: ignore[attr-defined] input="Test", tools=[], tool_choice="auto", extra_create_args={"invalid_param": "value"}, # Not allowed in Responses API ) def test_default_reasoning_effort(self, client: OpenAIResponsesAPIClient) -> None: """Test default reasoning effort is set when not specified.""" params = client._OpenAIResponsesAPIClient__process_create_args( # type: ignore[attr-defined] input="Test input", tools=[], tool_choice="auto", extra_create_args={} ) # Should default to medium reasoning effort create_args: Dict[str, Any] = params.create_args reasoning: Dict[str, Any] = cast(Dict[str, Any], create_args.get("reasoning", {})) assert reasoning.get("effort") == "medium" class TestResponsesAPICallHandling: """Test actual API call handling and response processing.""" @pytest.fixture def mock_openai_client(self) -> Any: with patch("autogen_ext.models.openai._responses_client._openai_client_from_config") as mock: mock_client = AsyncMock() mock_client.responses.create = AsyncMock() mock.return_value = mock_client yield mock_client @pytest.fixture def client(self, mock_openai_client: Any) -> OpenAIResponsesAPIClient: return OpenAIResponsesAPIClient(model="gpt-5", api_key="test-key") async def test_basic_text_response(self, client: OpenAIResponsesAPIClient, mock_openai_client: Any) -> None: """Test processing of basic text response.""" mock_response = { "id": "resp-123", "choices": [{"message": {"content": "This is a test response"}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 15, "completion_tokens": 25}, } mock_openai_client.responses.create.return_value = mock_response result = await client.create(input="Test question") assert isinstance(result, CreateResult) assert result.content == "This is a test response" assert result.finish_reason == "stop" assert result.usage.prompt_tokens == 15 assert result.usage.completion_tokens == 25 assert hasattr(result, "response_id") assert result.response_id == "resp-123" # type: ignore async def test_response_with_reasoning(self, client: OpenAIResponsesAPIClient, mock_openai_client: Any) -> None: """Test processing response with reasoning items.""" mock_response = { "id": "resp-124", "choices": [{"message": {"content": "Final answer after reasoning"}, "finish_reason": "stop"}], "reasoning_items": [ {"type": "reasoning", "content": "First, I need to consider..."}, {"type": "reasoning", "content": "Then, I should analyze..."}, {"type": "reasoning", "content": "Finally, the conclusion is..."}, ], "usage": {"prompt_tokens": 30, "completion_tokens": 50}, } mock_openai_client.responses.create.return_value = mock_response result = await client.create(input="Complex reasoning question", reasoning_effort="high") assert result.content == "Final answer after reasoning" assert result.thought is not None assert "First, I need to consider..." in result.thought assert "Then, I should analyze..." in result.thought assert "Finally, the conclusion is..." in result.thought async def test_custom_tool_call_response(self, client: OpenAIResponsesAPIClient, mock_openai_client: Any) -> None: """Test processing response with custom tool calls.""" from test_gpt5_features import TestCodeExecutorTool code_tool = TestCodeExecutorTool() mock_response = { "id": "resp-125", "choices": [ { "message": { "content": "I'll execute this Python code for you.", "tool_calls": [ { "id": "call-789", "custom": { "name": "code_exec", "input": "print('Hello from GPT-5!')\nresult = 2 + 2\nprint(f'2 + 2 = {result}')", }, } ], }, "finish_reason": "tool_calls", } ], "usage": {"prompt_tokens": 25, "completion_tokens": 35}, } mock_openai_client.responses.create.return_value = mock_response result = await client.create(input="Run this Python code to do basic math", tools=[code_tool], preambles=True) assert isinstance(result.content, list) assert len(result.content) == 1 tool_call = result.content[0] assert tool_call.name == "code_exec" assert "print('Hello from GPT-5!')" in tool_call.arguments assert result.thought == "I'll execute this Python code for you." assert str(result.finish_reason) == "tool_calls" async def test_cot_preservation_call(self, client: OpenAIResponsesAPIClient, mock_openai_client: Any) -> None: """Test call with chain-of-thought preservation.""" # First call mock_response1 = { "id": "resp-100", "choices": [{"message": {"content": "Initial response"}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 20, "completion_tokens": 30}, "reasoning_items": [{"type": "reasoning", "content": "Initial reasoning"}], } mock_openai_client.responses.create.return_value = mock_response1 result1 = await client.create(input="First question", reasoning_effort="high") # Second call with preserved context mock_response2 = { "id": "resp-101", "choices": [{"message": {"content": "Follow-up response"}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 10, "completion_tokens": 20}, # Lower tokens due to context reuse } mock_openai_client.responses.create.return_value = mock_response2 result2 = await client.create( input="Follow-up question", previous_response_id=result1.response_id, # type: ignore reasoning_effort="low", ) # Verify parameters were passed correctly call_kwargs = mock_openai_client.responses.create.call_args[1] assert call_kwargs["previous_response_id"] == "resp-100" assert call_kwargs["reasoning"]["effort"] == "low" # Verify lower token usage due to context reuse assert result2.usage.prompt_tokens < result1.usage.prompt_tokens class TestResponsesAPIErrorHandling: """Test error handling in Responses API client.""" @pytest.fixture def mock_openai_client(self) -> Any: with patch("autogen_ext.models.openai._responses_client._openai_client_from_config") as mock: mock_client = AsyncMock() mock_client.responses.create = AsyncMock() mock.return_value = mock_client yield mock_client @pytest.fixture def client(self, mock_openai_client: Any) -> OpenAIResponsesAPIClient: return OpenAIResponsesAPIClient(model="gpt-5", api_key="test-key") async def test_api_error_propagation(self, client: OpenAIResponsesAPIClient, mock_openai_client: Any) -> None: """Test that API errors are properly propagated.""" from openai import APIError # Instantiate with minimal required args for latest SDK mock_openai_client.responses.create.side_effect = APIError(message="Test API error") # type: ignore[call-arg] with pytest.raises(APIError, match="Test API error"): await client.create(input="Test input") async def test_cancellation_token_support(self, client: OpenAIResponsesAPIClient, mock_openai_client: Any) -> None: """Test cancellation token is properly handled.""" cancellation_token = CancellationToken() # Mock a successful response mock_response = { "id": "resp-999", "choices": [{"message": {"content": "Response"}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 5, "completion_tokens": 10}, } mock_openai_client.responses.create.return_value = mock_response result = await client.create(input="Test with cancellation", cancellation_token=cancellation_token) assert result.content == "Response" # Verify cancellation token was linked to the future # (This is tested implicitly by successful completion) async def test_malformed_response_handling(self, client: OpenAIResponsesAPIClient, mock_openai_client: Any) -> None: """Test handling of malformed API responses.""" # Response missing required fields mock_response = { "id": "resp-bad" # Missing choices, usage, etc. } mock_openai_client.responses.create.return_value = mock_response result = await client.create(input="Test malformed response") # Should handle gracefully with defaults assert result.content == "" assert result.usage.prompt_tokens == 0 assert result.usage.completion_tokens == 0 class TestResponsesAPIIntegration: """Test integration scenarios for Responses API.""" @pytest.fixture def mock_openai_client(self) -> Any: with patch("autogen_ext.models.openai._responses_client._openai_client_from_config") as mock: mock_client = AsyncMock() mock_client.responses.create = AsyncMock() mock.return_value = mock_client yield mock_client @pytest.fixture def client(self, mock_openai_client: Any) -> OpenAIResponsesAPIClient: return OpenAIResponsesAPIClient(model="gpt-5", api_key="test-key") async def test_multi_turn_conversation_simulation( self, client: OpenAIResponsesAPIClient, mock_openai_client: Any ) -> None: """Simulate a realistic multi-turn conversation with GPT-5.""" # Turn 1: Initial complex question mock_openai_client.responses.create.return_value = { "id": "resp-001", "choices": [ {"message": {"content": "Let me break down quantum computing fundamentals..."}, "finish_reason": "stop"} ], "reasoning_items": [ {"type": "reasoning", "content": "This is a complex topic requiring careful explanation..."} ], "usage": {"prompt_tokens": 50, "completion_tokens": 200}, } result1 = await client.create( input="Explain quantum computing to someone with a physics background", reasoning_effort="high", verbosity="high", ) # Turn 2: Follow-up question with context reuse mock_openai_client.responses.create.return_value = { "id": "resp-002", "choices": [ { "message": {"content": "Building on quantum fundamentals, quantum algorithms..."}, "finish_reason": "stop", } ], "usage": {"prompt_tokens": 30, "completion_tokens": 150}, # Lower due to context } result2 = await client.create( input="How do quantum algorithms leverage these principles?", previous_response_id=result1.response_id, # type: ignore reasoning_effort="medium", # Less reasoning needed due to context ) # Turn 3: Specific implementation request mock_openai_client.responses.create.return_value = { "id": "resp-003", "choices": [ { "message": { "content": "I'll provide a simple quantum algorithm implementation.", "tool_calls": [ { "id": "call-001", "custom": { "name": "code_exec", "input": "# Simple quantum circuit\nfrom qiskit import QuantumCircuit\nqc = QuantumCircuit(2)\nqc.h(0)\nqc.cx(0, 1)\nprint(qc)", }, } ], }, "finish_reason": "tool_calls", } ], "usage": {"prompt_tokens": 25, "completion_tokens": 100}, } code_tool = TestCodeExecutorTool() result3 = await client.create( input="Show me a simple quantum circuit implementation", previous_response_id=result2.response_id, # type: ignore tools=[code_tool], reasoning_effort="minimal", # Very little reasoning needed preambles=True, ) # Verify the conversation flow assert "quantum computing fundamentals" in result1.content assert result1.thought is not None assert "quantum algorithms" in result2.content assert result2.usage.prompt_tokens < result1.usage.prompt_tokens assert isinstance(result3.content, list) assert result3.content[0].name == "code_exec" assert "QuantumCircuit" in result3.content[0].arguments assert result3.thought == "I'll provide a simple quantum algorithm implementation." async def test_usage_tracking(self, client: OpenAIResponsesAPIClient, mock_openai_client: Any) -> None: """Test token usage tracking across multiple calls.""" # Multiple API calls with different usage call_responses = [ { "id": "r1", "choices": [{"message": {"content": "Response 1"}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 10, "completion_tokens": 20}, }, { "id": "r2", "choices": [{"message": {"content": "Response 2"}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 15, "completion_tokens": 25}, }, { "id": "r3", "choices": [{"message": {"content": "Response 3"}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 5, "completion_tokens": 15}, }, ] for i, response in enumerate(call_responses): mock_openai_client.responses.create.return_value = response await client.create(input=f"Test input {i+1}") # Check cumulative usage total_usage = client.total_usage() actual_usage = client.actual_usage() assert total_usage.prompt_tokens == 30 # 10 + 15 + 5 assert total_usage.completion_tokens == 60 # 20 + 25 + 15 assert actual_usage.prompt_tokens == 30 assert actual_usage.completion_tokens == 60 if __name__ == "__main__": pytest.main([__file__, "-v"])