Browse Source

updated code for ci validations

pull/6917/head
tejas-dharani 11 months ago
parent
commit
f229ce758f
5 changed files with 110 additions and 86 deletions
  1. +38
    -44
      python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py
  2. +33
    -13
      python/packages/autogen-ext/src/autogen_ext/models/openai/_responses_client.py
  3. +24
    -15
      python/packages/autogen-ext/tests/models/test_gpt5_features.py
  4. +10
    -9
      python/packages/autogen-ext/tests/models/test_openai_model_client.py
  5. +5
    -5
      python/packages/autogen-ext/tests/models/test_responses_api_client.py

+ 38
- 44
python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py View File

@@ -273,6 +273,36 @@ def _add_usage(usage1: RequestUsage, usage2: RequestUsage) -> RequestUsage:
)


def _build_custom_tool_param_from_schema(custom_schema: Dict[str, Any]) -> Dict[str, Any]:
"""Build an OpenAI ChatCompletionToolParam for a GPT-5 custom tool schema.

The input schema is expected to be a mapping with at least "name" and optional
"description" and "format" (for grammar or other formats).
"""
custom_tool_param: Dict[str, Any] = {
"type": "custom",
"custom": {
"name": custom_schema["name"],
"description": custom_schema.get("description", ""),
},
}
if "format" in custom_schema:
format_config = custom_schema["format"]
# Support grammar format as well as opaque format payloads
format_type = cast(Dict[str, Any], format_config).get("type") if isinstance(format_config, dict) else None
if format_type == "grammar":
syntax = cast(Dict[str, Any], format_config).get("syntax")
definition = cast(Dict[str, Any], format_config).get("definition")
if syntax and definition:
custom_tool_param["custom"]["format"] = {
"type": "grammar",
"grammar": {"type": syntax, "grammar": definition},
}
else:
custom_tool_param["custom"]["format"] = format_config
return custom_tool_param


def convert_tools(
tools: Sequence[Tool | ToolSchema | CustomTool | CustomToolSchema],
) -> List[ChatCompletionToolParam]:
@@ -280,58 +310,22 @@ def convert_tools(
for tool in tools:
if isinstance(tool, CustomTool):
# GPT-5 Custom Tool - format according to OpenAI API spec
custom_schema = tool.schema
custom_tool_param: Dict[str, Any] = {
"type": "custom",
"custom": {
"name": custom_schema["name"],
"description": custom_schema.get("description", ""),
},
}
if "format" in custom_schema:
format_config = custom_schema["format"]
format_type = format_config.get("type")
if format_type == "grammar":
syntax = format_config.get("syntax")
definition = format_config.get("definition")
if syntax and definition:
custom_tool_param["custom"]["format"] = {
"type": "grammar",
"grammar": {"type": syntax, "grammar": definition},
}
else:
custom_tool_param["custom"]["format"] = format_config
custom_schema = cast(Dict[str, Any], tool.schema)
custom_tool_param = _build_custom_tool_param_from_schema(custom_schema)
result.append(cast(ChatCompletionToolParam, custom_tool_param))
elif isinstance(tool, dict) and "format" in tool:
# Custom tool schema dict
custom_tool_param: Dict[str, Any] = {
"type": "custom",
"custom": {
"name": tool["name"],
"description": tool.get("description", ""),
},
}
if "format" in tool:
format_config = tool["format"]
format_type = format_config.get("type")
if format_type == "grammar":
syntax = format_config.get("syntax")
definition = format_config.get("definition")
if syntax and definition:
custom_tool_param["custom"]["format"] = {
"type": "grammar",
"grammar": {"type": syntax, "grammar": definition},
}
else:
custom_tool_param["custom"]["format"] = format_config
# Custom tool schema dict (explicit schema)
custom_schema = cast(Dict[str, Any], tool)
custom_tool_param = _build_custom_tool_param_from_schema(custom_schema)
result.append(cast(ChatCompletionToolParam, custom_tool_param))
else:
# Standard function tool
tool_schema: ToolSchema
if isinstance(tool, Tool):
tool_schema = tool.schema
else:
assert isinstance(tool, dict)
tool_schema = tool
# At this point, this must be a function ToolSchema (not a CustomToolSchema)
tool_schema = cast(ToolSchema, tool)

result.append(
ChatCompletionToolParam(


+ 33
- 13
python/packages/autogen-ext/src/autogen_ext/models/openai/_responses_client.py View File

@@ -123,8 +123,8 @@ from typing_extensions import Unpack

from .._utils.normalize_stop_reason import normalize_stop_reason
from . import _model_info
from autogen_core import EVENT_LOGGER_NAME
from ._openai_client import (
EVENT_LOGGER_NAME,
convert_tools,
normalize_name,
)
@@ -502,11 +502,11 @@ class BaseOpenAIResponsesAPIClient:
if message_dict.get("content"):
thought = cast(str, message_dict["content"])

finish_reason = "tool_calls"
finish_reason_tools: Optional[str] = "tool_calls"
else:
# Text response
content = cast(str, message_dict.get("content", ""))
finish_reason = cast(Optional[str], choice.get("finish_reason", "stop"))
finish_reason: Optional[str] = cast(Optional[str], choice.get("finish_reason", "stop"))

# Extract reasoning if available
reasoning_items_data: Optional[List[Dict[str, Any]]] = result.get("reasoning_items") # type: ignore[assignment]
@@ -519,6 +519,26 @@ class BaseOpenAIResponsesAPIClient:
if reasoning_texts:
thought = "\n".join(reasoning_texts)

# Build CreateResult
if (locals().get("finish_reason_tools") or "") == "tool_calls":
# The model requested tool calls
create_result = CreateResult(
finish_reason=normalize_stop_reason("tool_calls"),
content=cast(List[FunctionCall], content),
usage=usage,
cached=False,
thought=thought,
)
else:
# Plain text response
create_result = CreateResult(
finish_reason=normalize_stop_reason(finish_reason),
content=str(content),
usage=usage,
cached=False,
thought=thought,
)

else:
# Fallback for direct content
content = str(result.get("content", ""))
@@ -528,23 +548,23 @@ class BaseOpenAIResponsesAPIClient:
if "reasoning" in result:
thought = str(result["reasoning"]) # best effort

response = CreateResult(
finish_reason=normalize_stop_reason(finish_reason),
content=content,
usage=usage,
cached=bool(result.get("cached", False)),
logprobs=None, # Responses API may not provide logprobs
thought=thought,
)
# Build CreateResult
create_result = CreateResult(
finish_reason=normalize_stop_reason(finish_reason),
content=str(content),
usage=usage,
cached=False,
thought=thought,
)

# Store response ID for potential future use
if "id" in result:
response.response_id = cast(str, result["id"]) # type: ignore
create_result.response_id = cast(str, result["id"]) # type: ignore

self._total_usage = _add_usage(self._total_usage, usage)
self._actual_usage = _add_usage(self._actual_usage, usage)

return response
return create_result

async def close(self) -> None:
"""Close the underlying client."""


+ 24
- 15
python/packages/autogen-ext/tests/models/test_gpt5_features.py View File

@@ -45,7 +45,7 @@ class CodeExecResult(BaseModel):
class TestCodeExecutorTool(BaseCustomTool[CodeExecResult]):
"""Test implementation of GPT-5 custom tool for code execution."""

def __init__(self):
def __init__(self) -> None:
super().__init__(
return_type=CodeExecResult,
name="code_exec",
@@ -63,7 +63,7 @@ class SQLResult(BaseModel):
class TestSQLTool(BaseCustomTool[SQLResult]):
"""Test implementation of GPT-5 custom tool with grammar constraints."""

def __init__(self):
def __init__(self) -> None:
sql_grammar: CustomToolFormat = {
"type": "grammar",
"syntax": "lark",
@@ -139,11 +139,11 @@ class TestCustomToolsIntegration:

assert schema["name"] == "sql_query"
assert "format" in schema
fmt = schema.get("format")
assert fmt is not None and isinstance(fmt, dict)
assert fmt.get("type") == "grammar"
assert fmt.get("syntax") == "lark"
assert isinstance(fmt.get("definition"), str) and "SELECT" in fmt.get("definition", "")
fmt_any = schema.get("format")
assert isinstance(fmt_any, dict)
assert fmt_any.get("type") == "grammar"
assert fmt_any.get("syntax") == "lark"
assert isinstance(fmt_any.get("definition"), str) and "SELECT" in fmt_any.get("definition", "")

def test_convert_custom_tools(self) -> None:
"""Test conversion of custom tools to OpenAI API format."""
@@ -155,13 +155,13 @@ class TestCustomToolsIntegration:
assert len(converted) == 2

# Check code tool conversion
code_tool_param = next(t for t in converted if t.get("custom", {}).get("name") == "code_exec")
assert code_tool_param["type"] == "custom"
code_tool_param = next(cast(Dict[str, Any], t) for t in converted if cast(Dict[str, Any], t).get("custom", {}).get("name") == "code_exec")
assert str(code_tool_param.get("type")) == "custom"
assert "format" not in code_tool_param.get("custom", {})

# Check SQL tool conversion with grammar
sql_tool_param = next(t for t in converted if t.get("custom", {}).get("name") == "sql_query")
assert sql_tool_param["type"] == "custom"
sql_tool_param = next(cast(Dict[str, Any], t) for t in converted if cast(Dict[str, Any], t).get("custom", {}).get("name") == "sql_query")
assert str(sql_tool_param.get("type")) == "custom"
assert "format" in sql_tool_param.get("custom", {})
assert sql_tool_param.get("custom", {}).get("format", {}).get("type") == "grammar"

@@ -337,8 +337,15 @@ class TestAllowedToolsFeature:
exec_tool = FunctionTool(dangerous_exec, description="Code executor")
code_tool = TestCodeExecutorTool()

all_tools = [calc_tool, exec_tool, code_tool]
safe_tools = [calc_tool] # Only allow calculator
from autogen_core.tools import Tool as _Tool, ToolSchema as _ToolSchema
from autogen_core.tools import CustomTool as _CustomTool, CustomToolSchema as _CustomToolSchema

all_tools: List[_Tool | _ToolSchema | _CustomTool | _CustomToolSchema] = [
cast(_Tool, calc_tool),
cast(_Tool, exec_tool),
cast(_CustomTool, code_tool),
]
safe_tools: List[_Tool | _CustomTool | str] = [cast(_Tool, calc_tool)] # Only allow calculator

mock_response = ChatCompletion(
id="test-id",
@@ -536,9 +543,11 @@ class TestGPT5IntegrationScenarios:
)
mock_openai_client.chat.completions.create.return_value = mock_response

# Tools typed to expected union for create
tools_param = [code_tool, sql_tool]
result = await client.create(
messages=[UserMessage(content="Analyze this fibonacci implementation and run it for n=10", source="user")],
tools=[code_tool, sql_tool],
tools=tools_param,
reasoning_effort="medium", # type: ignore[arg-type]
verbosity="low", # type: ignore[arg-type]
preambles=True,
@@ -610,7 +619,7 @@ class TestGPT5IntegrationScenarios:


@pytest.mark.asyncio
async def test_gpt5_error_handling():
async def test_gpt5_error_handling() -> None:
"""Test proper error handling for GPT-5 specific scenarios."""

# Test invalid reasoning effort


+ 10
- 9
python/packages/autogen-ext/tests/models/test_openai_model_client.py View File

@@ -2,7 +2,7 @@ import asyncio
import json
import logging
import os
from typing import Annotated, Any, AsyncGenerator, Dict, List, Literal, Tuple, TypeVar
from typing import Annotated, Any, AsyncGenerator, Dict, List, Literal, Tuple, TypeVar, get_args
from unittest.mock import AsyncMock, MagicMock

import httpx
@@ -3268,14 +3268,14 @@ async def test_openai_tool_choice_validation_error_integration() -> None:


# GPT-5 model tests
def test_gpt5_model_resolution():
def test_gpt5_model_resolution() -> None:
"""Test that GPT-5 models resolve correctly."""
assert resolve_model("gpt-5") == "gpt-5-2025-08-07"
assert resolve_model("gpt-5-mini") == "gpt-5-mini-2025-08-07"
assert resolve_model("gpt-5-nano") == "gpt-5-nano-2025-08-07"


def test_gpt5_model_info():
def test_gpt5_model_info() -> None:
"""Test that GPT-5 models have correct capabilities."""
from autogen_ext.models.openai._model_info import get_info

@@ -3294,7 +3294,7 @@ def test_gpt5_model_info():
assert gpt5_nano_info["family"] == ModelFamily.GPT_5_NANO


def test_gpt5_client_creation():
def test_gpt5_client_creation() -> None:
"""Test that GPT-5 client can be created with new parameters."""
client = OpenAIChatCompletionClient(
model="gpt-5",
@@ -3304,7 +3304,7 @@ def test_gpt5_client_creation():


@pytest.mark.asyncio
async def test_gpt5_reasoning_effort_parameter():
async def test_gpt5_reasoning_effort_parameter() -> None:
"""Test that reasoning_effort parameter is properly handled."""
# Mock the OpenAI client to avoid actual API calls
import unittest.mock
@@ -3348,16 +3348,17 @@ async def test_gpt5_reasoning_effort_parameter():
assert call_args.kwargs["verbosity"] == "low"


def test_gpt5_model_families():
def test_gpt5_model_families() -> None:
"""Test that GPT-5 model families are properly defined."""
assert ModelFamily.GPT_5 == "gpt-5"
assert ModelFamily.GPT_5_MINI == "gpt-5-mini"
assert ModelFamily.GPT_5_NANO == "gpt-5-nano"

# Check that they're included in the ANY type
assert "gpt-5" in ModelFamily.ANY.__args__
assert "gpt-5-mini" in ModelFamily.ANY.__args__
assert "gpt-5-nano" in ModelFamily.ANY.__args__
any_args = get_args(ModelFamily.ANY)
assert "gpt-5" in any_args
assert "gpt-5-mini" in any_args
assert "gpt-5-nano" in any_args


# TODO: add integration tests for Azure OpenAI using AAD token.

+ 5
- 5
python/packages/autogen-ext/tests/models/test_responses_api_client.py View File

@@ -63,7 +63,7 @@ class TestResponsesAPIParameterHandling:
"""Test Responses API specific parameter handling."""

@pytest.fixture
def mock_openai_client(self):
def mock_openai_client(self) -> Any:
with patch("autogen_ext.models.openai._responses_client._openai_client_from_config") as mock:
mock_client = AsyncMock()
mock_client.responses.create = AsyncMock()
@@ -135,7 +135,7 @@ class TestResponsesAPICallHandling:
"""Test actual API call handling and response processing."""

@pytest.fixture
def mock_openai_client(self):
def mock_openai_client(self) -> Any:
with patch("autogen_ext.models.openai._responses_client._openai_client_from_config") as mock:
mock_client = AsyncMock()
mock_client.responses.create = AsyncMock()
@@ -225,7 +225,7 @@ class TestResponsesAPICallHandling:
assert tool_call.name == "code_exec"
assert "print('Hello from GPT-5!')" in tool_call.arguments
assert result.thought == "I'll execute this Python code for you."
assert result.finish_reason == "tool_calls"
assert str(result.finish_reason) == "tool_calls"

async def test_cot_preservation_call(self, client: OpenAIResponsesAPIClient, mock_openai_client: Any) -> None:
"""Test call with chain-of-thought preservation."""
@@ -267,7 +267,7 @@ class TestResponsesAPIErrorHandling:
"""Test error handling in Responses API client."""

@pytest.fixture
def mock_openai_client(self):
def mock_openai_client(self) -> Any:
with patch("autogen_ext.models.openai._responses_client._openai_client_from_config") as mock:
mock_client = AsyncMock()
mock_client.responses.create = AsyncMock()
@@ -327,7 +327,7 @@ class TestResponsesAPIIntegration:
"""Test integration scenarios for Responses API."""

@pytest.fixture
def mock_openai_client(self):
def mock_openai_client(self) -> Any:
with patch("autogen_ext.models.openai._responses_client._openai_client_from_config") as mock:
mock_client = AsyncMock()
mock_client.responses.create = AsyncMock()


Loading…
Cancel
Save