autogen/test/agentchat/test_groupchat.py

#!/usr/bin/env python3 -m pytest

import builtins
import io
import json
import logging
from typing import Any, Dict, List, Optional
from unittest import TestCase, mock

import pytest
from test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST

import autogen
from autogen import Agent, AssistantAgent, GroupChat, GroupChatManager
from autogen.exception_utils import AgentNameConflict, UndefinedNextAgent


def test_func_call_groupchat():
    agent1 = autogen.ConversableAgent(
        "alice",
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is alice speaking.",
    )
    agent2 = autogen.ConversableAgent(
        "bob",
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is bob speaking.",
        function_map={"test_func": lambda x: x},
    )
    groupchat = autogen.GroupChat(agents=[agent1, agent2], messages=[], max_round=3)
    group_chat_manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=False)
    agent2.initiate_chat(group_chat_manager, message={"function_call": {"name": "test_func", "arguments": '{"x": 1}'}})

    assert len(groupchat.messages) == 3
    assert (
        groupchat.messages[-2]["role"] == "function"
        and groupchat.messages[-2]["name"] == "test_func"
        and groupchat.messages[-2]["content"] == "1"
    )
    assert groupchat.messages[-1]["name"] == "alice"

    agent3 = autogen.ConversableAgent(
        "carol",
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is carol speaking.",
        function_map={"test_func": lambda x: x + 1},
    )
    groupchat = autogen.GroupChat(agents=[agent1, agent2, agent3], messages=[], max_round=3)
    group_chat_manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=False)
    agent3.initiate_chat(group_chat_manager, message={"function_call": {"name": "test_func", "arguments": '{"x": 1}'}})

    assert (
        groupchat.messages[-2]["role"] == "function"
        and groupchat.messages[-2]["name"] == "test_func"
        and groupchat.messages[-2]["content"] == "1"
    )
    assert groupchat.messages[-1]["name"] == "carol"

    agent2.initiate_chat(group_chat_manager, message={"function_call": {"name": "func", "arguments": '{"x": 1}'}})


def test_chat_manager():
    agent1 = autogen.ConversableAgent(
        "alice",
        max_consecutive_auto_reply=2,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is alice speaking.",
    )
    agent2 = autogen.ConversableAgent(
        "bob",
        max_consecutive_auto_reply=2,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is bob speaking.",
    )
    groupchat = autogen.GroupChat(agents=[agent1, agent2], messages=[], max_round=2)
    group_chat_manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=False)
    agent1.initiate_chat(group_chat_manager, message="hello")

    assert len(agent1.chat_messages[group_chat_manager]) == 2
    assert len(groupchat.messages) == 2

    group_chat_manager.reset()
    assert len(groupchat.messages) == 0
    agent1.reset()
    agent2.reset()
    agent2.initiate_chat(group_chat_manager, message="hello")
    assert len(groupchat.messages) == 2

    with pytest.raises(ValueError):
        agent2.initiate_chat(group_chat_manager, message={"function_call": {"name": "func", "arguments": '{"x": 1}'}})


def _test_selection_method(method: str):
    agent1 = autogen.ConversableAgent(
        "alice",
        max_consecutive_auto_reply=10,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is alice speaking.",
    )
    agent2 = autogen.ConversableAgent(
        "bob",
        max_consecutive_auto_reply=10,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is bob speaking.",
    )
    agent3 = autogen.ConversableAgent(
        "charlie",
        max_consecutive_auto_reply=10,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is charlie speaking.",
    )

    groupchat = autogen.GroupChat(
        agents=[agent1, agent2, agent3],
        messages=[],
        max_round=6,
        speaker_selection_method=method,
        allow_repeat_speaker=False if method == "manual" else True,
    )
    group_chat_manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=False)

    if method == "round_robin":
        agent1.initiate_chat(group_chat_manager, message="This is alice speaking.")
        assert len(agent1.chat_messages[group_chat_manager]) == 6
        assert len(groupchat.messages) == 6
        assert [msg["content"] for msg in agent1.chat_messages[group_chat_manager]] == [
            "This is alice speaking.",
            "This is bob speaking.",
            "This is charlie speaking.",
        ] * 2
    elif method == "auto":
        agent1.initiate_chat(group_chat_manager, message="This is alice speaking.")
        assert len(agent1.chat_messages[group_chat_manager]) == 6
        assert len(groupchat.messages) == 6
    elif method == "random":
        agent1.initiate_chat(group_chat_manager, message="This is alice speaking.")
        assert len(agent1.chat_messages[group_chat_manager]) == 6
        assert len(groupchat.messages) == 6
    elif method == "manual":
        for user_input in ["", "q", "x", "1", "10"]:
            with mock.patch.object(builtins, "input", lambda _: user_input):
                group_chat_manager.reset()
                agent1.reset()
                agent2.reset()
                agent3.reset()
                agent1.initiate_chat(group_chat_manager, message="This is alice speaking.")
                if user_input == "1":
                    assert len(agent1.chat_messages[group_chat_manager]) == 6
                    assert len(groupchat.messages) == 6
                    assert [msg["content"] for msg in agent1.chat_messages[group_chat_manager]] == [
                        "This is alice speaking.",
                        "This is bob speaking.",
                        "This is alice speaking.",
                        "This is bob speaking.",
                        "This is alice speaking.",
                        "This is bob speaking.",
                    ]
                else:
                    assert len(agent1.chat_messages[group_chat_manager]) == 6
                    assert len(groupchat.messages) == 6
    elif method == "wrong":
        with pytest.raises(ValueError):
            agent1.initiate_chat(group_chat_manager, message="This is alice speaking.")


def test_speaker_selection_method():
    for method in ["auto", "round_robin", "random", "manual", "wrong", "RounD_roBin"]:
        _test_selection_method(method)


def _test_n_agents_less_than_3(method):
    agent1 = autogen.ConversableAgent(
        "alice",
        max_consecutive_auto_reply=10,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is alice speaking.",
    )
    agent2 = autogen.ConversableAgent(
        "bob",
        max_consecutive_auto_reply=10,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is bob speaking.",
    )
    # test two agents
    groupchat = autogen.GroupChat(
        agents=[agent1, agent2],
        messages=[],
        max_round=6,
        speaker_selection_method=method,
        allow_repeat_speaker=[agent1, agent2] if method == "random" else False,
    )
    group_chat_manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=False)
    agent1.initiate_chat(group_chat_manager, message="This is alice speaking.")
    assert len(agent1.chat_messages[group_chat_manager]) == 6
    assert len(groupchat.messages) == 6
    if method != "random" or method.lower() == "round_robin":
        assert [msg["content"] for msg in agent1.chat_messages[group_chat_manager]] == [
            "This is alice speaking.",
            "This is bob speaking.",
        ] * 3

    # test zero agent
    with pytest.raises(ValueError):
        groupchat = autogen.GroupChat(
            agents=[], messages=[], max_round=6, speaker_selection_method="round_robin", allow_repeat_speaker=False
        )
        group_chat_manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=False)
        agent1.initiate_chat(group_chat_manager, message="This is alice speaking.")


def test_invalid_allow_repeat_speaker():
    agent1 = autogen.ConversableAgent(
        "alice",
        max_consecutive_auto_reply=10,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is alice speaking.",
    )
    agent2 = autogen.ConversableAgent(
        "bob",
        max_consecutive_auto_reply=10,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is bob speaking.",
    )
    # test invalid allow_repeat_speaker
    with pytest.raises(ValueError) as e:
        autogen.GroupChat(
            agents=[agent1, agent2],
            messages=[],
            max_round=6,
            speaker_selection_method="round_robin",
            allow_repeat_speaker={},
        )
    assert str(e.value) == "GroupChat allow_repeat_speaker should be a bool or a list of Agents.", e.value


def test_n_agents_less_than_3():
    for method in ["auto", "round_robin", "random", "RounD_roBin"]:
        _test_n_agents_less_than_3(method)


def test_plugin():
    # Give another Agent class ability to manage group chat
    agent1 = autogen.ConversableAgent(
        "alice",
        max_consecutive_auto_reply=2,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is alice speaking.",
    )
    agent2 = autogen.ConversableAgent(
        "bob",
        max_consecutive_auto_reply=2,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is bob speaking.",
    )
    groupchat = autogen.GroupChat(agents=[agent1, agent2], messages=[], max_round=2)
    group_chat_manager = autogen.ConversableAgent(name="deputy_manager", llm_config=False)
    group_chat_manager.register_reply(
        autogen.Agent,
        reply_func=autogen.GroupChatManager.run_chat,
        config=groupchat,
        reset_config=autogen.GroupChat.reset,
    )
    agent1.initiate_chat(group_chat_manager, message="hello")

    assert len(agent1.chat_messages[group_chat_manager]) == 2
    assert len(groupchat.messages) == 2


def test_agent_mentions():
    agent1 = autogen.ConversableAgent(
        "alice",
        max_consecutive_auto_reply=2,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is alice speaking.",
    )
    agent2 = autogen.ConversableAgent(
        "bob",
        max_consecutive_auto_reply=2,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is bob speaking.",
    )
    agent3 = autogen.ConversableAgent(
        "sam",
        max_consecutive_auto_reply=2,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is sam speaking.",
    )
    groupchat = autogen.GroupChat(agents=[agent1, agent2, agent3], messages=[], max_round=2)

    # Basic counting
    assert json.dumps(groupchat._mentioned_agents("", [agent1, agent2, agent3]), sort_keys=True) == "{}"
    assert json.dumps(groupchat._mentioned_agents("alice", [agent1, agent2, agent3]), sort_keys=True) == '{"alice": 1}'
    assert (
        json.dumps(groupchat._mentioned_agents("alice bob alice", [agent1, agent2, agent3]), sort_keys=True)
        == '{"alice": 2, "bob": 1}'
    )
    assert (
        json.dumps(groupchat._mentioned_agents("alice bob alice sam", [agent1, agent2, agent3]), sort_keys=True)
        == '{"alice": 2, "bob": 1, "sam": 1}'
    )
    assert (
        json.dumps(groupchat._mentioned_agents("alice bob alice sam robert", [agent1, agent2, agent3]), sort_keys=True)
        == '{"alice": 2, "bob": 1, "sam": 1}'
    )

    # Substring
    assert (
        json.dumps(groupchat._mentioned_agents("sam samantha basam asami", [agent1, agent2, agent3]), sort_keys=True)
        == '{"sam": 1}'
    )

    # Word boundaries
    assert (
        json.dumps(groupchat._mentioned_agents("alice! .alice. .alice", [agent1, agent2, agent3]), sort_keys=True)
        == '{"alice": 3}'
    )

    # Special characters in agent names
    agent4 = autogen.ConversableAgent(
        ".*",
        max_consecutive_auto_reply=2,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="Match everything.",
    )

    groupchat = autogen.GroupChat(agents=[agent1, agent2, agent3, agent4], messages=[], max_round=2)
    assert (
        json.dumps(
            groupchat._mentioned_agents("alice bob alice sam robert .*", [agent1, agent2, agent3, agent4]),
            sort_keys=True,
        )
        == '{".*": 1, "alice": 2, "bob": 1, "sam": 1}'
    )


def test_termination():
    agent1 = autogen.ConversableAgent(
        "alice",
        max_consecutive_auto_reply=10,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is alice speaking.",
    )
    agent2 = autogen.ConversableAgent(
        "bob",
        max_consecutive_auto_reply=10,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is bob speaking.",
    )
    agent3 = autogen.ConversableAgent(
        "sam",
        max_consecutive_auto_reply=10,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is sam speaking. TERMINATE",
    )

    # Test empty is_termination_msg function
    groupchat = autogen.GroupChat(
        agents=[agent1, agent2, agent3], messages=[], speaker_selection_method="round_robin", max_round=10
    )

    group_chat_manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=False, is_termination_msg=None)

    agent1.initiate_chat(group_chat_manager, message="'None' is_termination_msg function.")
    assert len(groupchat.messages) == 10

    # Test user-provided is_termination_msg function
    agent1.reset()
    agent2.reset()
    agent3.reset()

    groupchat = autogen.GroupChat(
        agents=[agent1, agent2, agent3], messages=[], speaker_selection_method="round_robin", max_round=10
    )

    group_chat_manager = autogen.GroupChatManager(
        groupchat=groupchat,
        llm_config=False,
        is_termination_msg=lambda x: x.get("content", "").rstrip().find("TERMINATE") >= 0,
    )

    agent1.initiate_chat(group_chat_manager, message="User-provided is_termination_msg function.")
    assert len(groupchat.messages) == 3


def test_next_agent():
    def create_agent(name: str) -> autogen.ConversableAgent:
        return autogen.ConversableAgent(
            name,
            max_consecutive_auto_reply=10,
            human_input_mode="NEVER",
            llm_config=False,
            default_auto_reply=f"This is {name} speaking.",
        )

    agent1 = create_agent("alice")
    agent2 = create_agent("bob")
    agent3 = create_agent("sam")
    agent4 = create_agent("sally")
    agent5 = create_agent("samantha")
    agent6 = create_agent("robert")

    # Test empty is_termination_msg function
    groupchat = autogen.GroupChat(
        agents=[agent1, agent2, agent3], messages=[], speaker_selection_method="round_robin", max_round=10
    )

    assert groupchat.next_agent(agent1, [agent1, agent2, agent3]) == agent2
    assert groupchat.next_agent(agent2, [agent1, agent2, agent3]) == agent3
    assert groupchat.next_agent(agent3, [agent1, agent2, agent3]) == agent1

    assert groupchat.next_agent(agent1) == agent2
    assert groupchat.next_agent(agent2) == agent3
    assert groupchat.next_agent(agent3) == agent1

    assert groupchat.next_agent(agent1, [agent1, agent3]) == agent3
    assert groupchat.next_agent(agent3, [agent1, agent3]) == agent1

    assert groupchat.next_agent(agent2, [agent1, agent3]) == agent3
    assert groupchat.next_agent(agent4, [agent1, agent3]) == agent1
    assert groupchat.next_agent(agent4, [agent1, agent2, agent3]) == agent1

    with pytest.raises(UndefinedNextAgent):
        groupchat.next_agent(agent4, [agent5, agent6])


def test_send_intros():
    agent1 = autogen.ConversableAgent(
        "alice",
        description="The first agent.",
        max_consecutive_auto_reply=10,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is alice speaking. TERMINATE",
    )
    agent2 = autogen.ConversableAgent(
        "bob",
        description="The second agent.",
        max_consecutive_auto_reply=10,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is bob speaking. TERMINATE",
    )
    agent3 = autogen.ConversableAgent(
        "sam",
        description="The third agent.",
        max_consecutive_auto_reply=10,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is sam speaking. TERMINATE",
    )
    agent4 = autogen.ConversableAgent(
        "sally",
        description="The fourth agent.",
        max_consecutive_auto_reply=10,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is sally speaking. TERMINATE",
    )

    # Test empty is_termination_msg function
    groupchat = autogen.GroupChat(
        agents=[agent1, agent2, agent3],
        messages=[],
        speaker_selection_method="round_robin",
        max_round=10,
        send_introductions=True,
    )

    intro = groupchat.introductions_msg()
    assert "The first agent." in intro
    assert "The second agent." in intro
    assert "The third agent." in intro
    assert "The fourth agent." not in intro

    intro = groupchat.introductions_msg([agent1, agent2, agent4])
    assert "The first agent." in intro
    assert "The second agent." in intro
    assert "The third agent." not in intro
    assert "The fourth agent." in intro

    groupchat = autogen.GroupChat(
        agents=[agent1, agent2, agent3],
        messages=[],
        speaker_selection_method="round_robin",
        max_round=10,
        send_introductions=True,
    )

    group_chat_manager = autogen.GroupChatManager(
        groupchat=groupchat,
        llm_config=False,
        is_termination_msg=lambda x: x.get("content", "").rstrip().find("TERMINATE") >= 0,
    )

    group_chat_manager.initiate_chat(group_chat_manager, message="The initiating message.")
    for a in [agent1, agent2, agent3]:
        messages = agent1.chat_messages[group_chat_manager]
        assert len(messages) == 3
        assert "The first agent." in messages[0]["content"]
        assert "The second agent." in messages[0]["content"]
        assert "The third agent." in messages[0]["content"]
        assert "The initiating message." == messages[1]["content"]
        assert messages[2]["content"] == agent1._default_auto_reply

    # Reset and start again
    agent1.reset()
    agent2.reset()
    agent3.reset()
    agent4.reset()

    # Check the default (no introductions)
    groupchat2 = autogen.GroupChat(
        agents=[agent1, agent2, agent3], messages=[], speaker_selection_method="round_robin", max_round=10
    )

    group_chat_manager2 = autogen.GroupChatManager(
        groupchat=groupchat2,
        llm_config=False,
        is_termination_msg=lambda x: x.get("content", "").rstrip().find("TERMINATE") >= 0,
    )

    group_chat_manager2.initiate_chat(group_chat_manager2, message="The initiating message.")
    for a in [agent1, agent2, agent3]:
        messages = agent1.chat_messages[group_chat_manager2]
        assert len(messages) == 2
        assert "The initiating message." == messages[0]["content"]
        assert messages[1]["content"] == agent1._default_auto_reply


def test_selection_helpers():
    agent1 = autogen.ConversableAgent(
        "alice",
        max_consecutive_auto_reply=10,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is alice speaking.",
        description="Alice is an AI agent.",
    )
    agent2 = autogen.ConversableAgent(
        "bob",
        max_consecutive_auto_reply=10,
        human_input_mode="NEVER",
        llm_config=False,
        description="Bob is an AI agent.",
    )
    agent3 = autogen.ConversableAgent(
        "sam",
        max_consecutive_auto_reply=10,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is sam speaking.",
        system_message="Sam is an AI agent.",
    )

    # Test empty is_termination_msg function
    groupchat = autogen.GroupChat(
        agents=[agent1, agent2, agent3], messages=[], speaker_selection_method="round_robin", max_round=10
    )

    select_speaker_msg = groupchat.select_speaker_msg()
    select_speaker_prompt = groupchat.select_speaker_prompt()

    assert "Alice is an AI agent." in select_speaker_msg
    assert "Bob is an AI agent." in select_speaker_msg
    assert "Sam is an AI agent." in select_speaker_msg
    assert str(["Alice", "Bob", "Sam"]).lower() in select_speaker_prompt.lower()

    with mock.patch.object(builtins, "input", lambda _: "1"):
        groupchat.manual_select_speaker()


def test_init_default_parameters():
    agents = [autogen.ConversableAgent(name=f"Agent{i}", llm_config=False) for i in range(3)]
    group_chat = GroupChat(agents=agents, messages=[], max_round=3)
    for agent in agents:
        assert set([a.name for a in group_chat.allowed_speaker_transitions_dict[agent]]) == set(
            [a.name for a in agents]
        )


def test_graph_parameters():
    agents = [autogen.ConversableAgent(name=f"Agent{i}", llm_config=False) for i in range(3)]
    with pytest.raises(ValueError):
        GroupChat(
            agents=agents,
            messages=[],
            max_round=3,
            allowed_or_disallowed_speaker_transitions={agents[0]: [agents[1]], agents[1]: [agents[2]]},
        )
    with pytest.raises(ValueError):
        GroupChat(
            agents=agents,
            messages=[],
            max_round=3,
            allow_repeat_speaker=False,  # should be None
            allowed_or_disallowed_speaker_transitions={agents[0]: [agents[1]], agents[1]: [agents[2]]},
        )

    with pytest.raises(ValueError):
        GroupChat(
            agents=agents,
            messages=[],
            max_round=3,
            allow_repeat_speaker=None,
            allowed_or_disallowed_speaker_transitions={agents[0]: [agents[1]], agents[1]: [agents[2]]},
            speaker_transitions_type="a",
        )

    group_chat = GroupChat(
        agents=agents,
        messages=[],
        max_round=3,
        allowed_or_disallowed_speaker_transitions={agents[0]: [agents[1]], agents[1]: [agents[2]]},
        speaker_transitions_type="allowed",
    )
    assert "Agent0" in group_chat.agent_names


def test_graceful_exit_before_max_round():
    agent1 = autogen.ConversableAgent(
        "alice",
        max_consecutive_auto_reply=10,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is alice speaking.",
    )
    agent2 = autogen.ConversableAgent(
        "bob",
        max_consecutive_auto_reply=10,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is bob speaking.",
    )
    agent3 = autogen.ConversableAgent(
        "sam",
        max_consecutive_auto_reply=10,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is sam speaking.",
    )

    # This speaker_transitions limits the transition to be only from agent1 to agent2, and from agent2 to agent3 and end.
    allowed_or_disallowed_speaker_transitions = {agent1: [agent2], agent2: [agent3]}

    # Test empty is_termination_msg function
    groupchat = autogen.GroupChat(
        agents=[agent1, agent2, agent3],
        messages=[],
        speaker_selection_method="round_robin",
        max_round=10,
        allow_repeat_speaker=None,
        allowed_or_disallowed_speaker_transitions=allowed_or_disallowed_speaker_transitions,
        speaker_transitions_type="allowed",
    )

    group_chat_manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=False, is_termination_msg=None)

    agent1.initiate_chat(group_chat_manager, message="")

    # Note that 3 is much lower than 10 (max_round), so the conversation should end before 10 rounds.
    assert len(groupchat.messages) == 3


def test_clear_agents_history():
    agent1 = autogen.ConversableAgent(
        "alice",
        max_consecutive_auto_reply=10,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is alice speaking.",
    )
    agent2 = autogen.ConversableAgent(
        "bob",
        max_consecutive_auto_reply=10,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is bob speaking.",
    )
    agent3 = autogen.ConversableAgent(
        "sam",
        max_consecutive_auto_reply=10,
        human_input_mode="ALWAYS",
        llm_config=False,
    )
    groupchat = autogen.GroupChat(agents=[agent1, agent2, agent3], messages=[], max_round=3, enable_clear_history=True)
    group_chat_manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=False)

    # testing pure "clear history" statement
    with mock.patch.object(builtins, "input", lambda _: "clear history. How you doing?"):
        res = agent1.initiate_chat(group_chat_manager, message="hello", summary_method="last_msg")
    agent1_history = list(agent1._oai_messages.values())[0]
    agent2_history = list(agent2._oai_messages.values())[0]
    assert agent1_history == [{"content": "How you doing?", "name": "sam", "role": "user"}]
    assert agent2_history == [{"content": "How you doing?", "name": "sam", "role": "user"}]
    assert groupchat.messages == [{"content": "How you doing?", "name": "sam", "role": "user"}]
    print("Chat summary", res.summary)
    print("Chat cost", res.cost)
    print("Chat history", res.chat_history)

    # testing clear history for defined agent
    with mock.patch.object(builtins, "input", lambda _: "clear history bob. How you doing?"):
        agent1.initiate_chat(group_chat_manager, message="hello")
    agent1_history = list(agent1._oai_messages.values())[0]
    agent2_history = list(agent2._oai_messages.values())[0]
    assert agent1_history == [
        {"content": "hello", "role": "assistant"},
        {"content": "This is bob speaking.", "name": "bob", "role": "user"},
        {"content": "How you doing?", "name": "sam", "role": "user"},
    ]
    assert agent2_history == [{"content": "How you doing?", "name": "sam", "role": "user"}]
    assert groupchat.messages == [
        {"content": "hello", "role": "user", "name": "alice"},
        {"content": "This is bob speaking.", "name": "bob", "role": "user"},
        {"content": "How you doing?", "name": "sam", "role": "user"},
    ]

    # testing clear history with defined nr of messages to preserve
    with mock.patch.object(builtins, "input", lambda _: "clear history 1. How you doing?"):
        agent1.initiate_chat(group_chat_manager, message="hello")
    agent1_history = list(agent1._oai_messages.values())[0]
    agent2_history = list(agent2._oai_messages.values())[0]
    assert agent1_history == [
        {"content": "This is bob speaking.", "name": "bob", "role": "user"},
        {"content": "How you doing?", "name": "sam", "role": "user"},
    ]
    assert agent2_history == [
        {"content": "This is bob speaking.", "role": "assistant"},
        {"content": "How you doing?", "name": "sam", "role": "user"},
    ]
    assert groupchat.messages == [
        {"content": "This is bob speaking.", "role": "user", "name": "bob"},
        {"content": "How you doing?", "role": "user", "name": "sam"},
    ]

    # testing clear history with defined agent and nr of messages to preserve
    with mock.patch.object(builtins, "input", lambda _: "clear history bob 1. How you doing?"):
        agent1.initiate_chat(group_chat_manager, message="hello")
    agent1_history = list(agent1._oai_messages.values())[0]
    agent2_history = list(agent2._oai_messages.values())[0]
    assert agent1_history == [
        {"content": "hello", "role": "assistant"},
        {"content": "This is bob speaking.", "name": "bob", "role": "user"},
        {"content": "How you doing?", "name": "sam", "role": "user"},
    ]
    assert agent2_history == [
        {"content": "This is bob speaking.", "role": "assistant"},
        {"content": "How you doing?", "name": "sam", "role": "user"},
    ]
    assert groupchat.messages == [
        {"content": "hello", "name": "alice", "role": "user"},
        {"content": "This is bob speaking.", "name": "bob", "role": "user"},
        {"content": "How you doing?", "name": "sam", "role": "user"},
    ]

    # testing saving tool_call message when clear history going to remove it leaving only tool_response message
    agent1.reset()
    agent2.reset()
    agent3.reset()
    # we want to broadcast the message only in the preparation.
    groupchat = autogen.GroupChat(agents=[agent1, agent2, agent3], messages=[], max_round=1, enable_clear_history=True)
    group_chat_manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=False)
    # We want to trigger the broadcast of group chat manager, which requires `request_reply` to be set to True.
    agent1.send("dummy message", group_chat_manager, request_reply=True)
    agent1.send(
        {
            "content": None,
            "role": "assistant",
            "function_call": None,
            "tool_calls": [
                {"id": "call_test_id", "function": {"arguments": "", "name": "test_tool"}, "type": "function"}
            ],
        },
        group_chat_manager,
        request_reply=True,
    )
    agent1.send(
        {
            "role": "tool",
            "tool_responses": [{"tool_call_id": "call_emulated", "role": "tool", "content": "example tool response"}],
            "content": "example tool response",
        },
        group_chat_manager,
        request_reply=True,
    )
    # increase max_round to 3
    groupchat.max_round = 3
    group_chat_manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=False)
    with mock.patch.object(builtins, "input", lambda _: "clear history alice 1. How you doing?"):
        agent1.initiate_chat(group_chat_manager, message="hello", clear_history=False)

    agent1_history = list(agent1._oai_messages.values())[0]
    assert agent1_history == [
        {
            "tool_calls": [
                {"id": "call_test_id", "function": {"arguments": "", "name": "test_tool"}, "type": "function"},
            ],
            "content": None,
            "role": "assistant",
        },
        {
            "content": "example tool response",
            "tool_responses": [{"tool_call_id": "call_emulated", "role": "tool", "content": "example tool response"}],
            "role": "tool",
        },
    ]

    # testing clear history called from tool response
    agent1.reset()
    agent2.reset()
    agent3.reset()
    agent2 = autogen.ConversableAgent(
        "bob",
        max_consecutive_auto_reply=10,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply={
            "role": "tool",
            "tool_responses": [{"tool_call_id": "call_emulated", "role": "tool", "content": "USER INTERRUPTED"}],
            "content": "Clear history. How you doing?",
        },
    )
    groupchat = autogen.GroupChat(agents=[agent1, agent2, agent3], messages=[], max_round=1, enable_clear_history=True)
    group_chat_manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=False)
    agent1.send("dummy message", group_chat_manager, request_reply=True)
    agent1.send(
        {
            "content": None,
            "role": "assistant",
            "function_call": None,
            "tool_calls": [
                {"id": "call_test_id", "function": {"arguments": "", "name": "test_tool"}, "type": "function"}
            ],
        },
        group_chat_manager,
        request_reply=True,
    )
    groupchat.max_round = 2
    group_chat_manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=False)

    agent1.initiate_chat(group_chat_manager, message="hello")
    agent1_history = list(agent1._oai_messages.values())[0]
    assert agent1_history == [
        {
            "tool_calls": [
                {"id": "call_test_id", "function": {"arguments": "", "name": "test_tool"}, "type": "function"},
            ],
            "content": None,
            "role": "assistant",
        },
    ]


def test_get_agent_by_name():
    def agent(name: str) -> autogen.ConversableAgent:
        return autogen.ConversableAgent(
            name=name,
            max_consecutive_auto_reply=10,
            human_input_mode="NEVER",
            llm_config=False,
        )

    def team(members: List[autogen.Agent], name: str) -> autogen.Agent:
        gc = autogen.GroupChat(agents=members, messages=[])

        return autogen.GroupChatManager(groupchat=gc, name=name, llm_config=False)

    team_member1 = agent("team1_member1")
    team_member2 = agent("team1_member2")
    team_dup_member1 = agent("team1_member1")
    team_dup_member2 = agent("team1_member2")

    user = agent("user")
    team1 = team([team_member1, team_member2], "team1")
    team1_duplicate = team([team_dup_member1, team_dup_member2], "team1")

    gc = autogen.GroupChat(agents=[user, team1, team1_duplicate], messages=[])

    # Testing default arguments
    assert gc.agent_by_name("user") == user
    assert gc.agent_by_name("team1") == team1 or gc.agent_by_name("team1") == team1_duplicate

    # Testing recursive search
    assert gc.agent_by_name("user", recursive=True) == user
    assert (
        gc.agent_by_name("team1_member1", recursive=True) == team_member1
        or gc.agent_by_name("team1_member1", recursive=True) == team_dup_member1
    )

    # Get agent that does not exist
    assert gc.agent_by_name("team2") is None
    assert gc.agent_by_name("team2", recursive=True) is None
    assert gc.agent_by_name("team2", raise_on_name_conflict=True) is None
    assert gc.agent_by_name("team2", recursive=True, raise_on_name_conflict=True) is None

    # Testing naming conflict
    with pytest.raises(AgentNameConflict):
        gc.agent_by_name("team1", raise_on_name_conflict=True)

    # Testing name conflict with recursive search
    with pytest.raises(AgentNameConflict):
        gc.agent_by_name("team1_member1", recursive=True, raise_on_name_conflict=True)


def test_get_nested_agents_in_groupchat():
    def agent(name: str) -> autogen.ConversableAgent:
        return autogen.ConversableAgent(
            name=name,
            max_consecutive_auto_reply=10,
            human_input_mode="NEVER",
            llm_config=False,
        )

    def team(name: str) -> autogen.ConversableAgent:
        member1 = agent(f"member1_{name}")
        member2 = agent(f"member2_{name}")

        gc = autogen.GroupChat(agents=[member1, member2], messages=[])

        return autogen.GroupChatManager(groupchat=gc, name=name, llm_config=False)

    user = agent("user")
    team1 = team("team1")
    team2 = team("team2")

    gc = autogen.GroupChat(agents=[user, team1, team2], messages=[])

    agents = gc.nested_agents()
    assert len(agents) == 7


def test_nested_teams_chat():
    """Tests chat capabilities of nested teams"""
    team1_msg = {"content": "Hello from team 1"}
    team2_msg = {"content": "Hello from team 2"}

    def agent(name: str, auto_reply: Optional[Dict[str, Any]] = None) -> autogen.ConversableAgent:
        return autogen.ConversableAgent(
            name=name,
            max_consecutive_auto_reply=10,
            human_input_mode="NEVER",
            llm_config=False,
            default_auto_reply=auto_reply,
        )

    def team(name: str, auto_reply: Optional[Dict[str, Any]] = None) -> autogen.ConversableAgent:
        member1 = agent(f"member1_{name}", auto_reply=auto_reply)
        member2 = agent(f"member2_{name}", auto_reply=auto_reply)

        gc = autogen.GroupChat(agents=[member1, member2], messages=[])

        return autogen.GroupChatManager(groupchat=gc, name=name, llm_config=False)

    def chat(gc_manager: autogen.GroupChatManager):
        team1_member1 = gc_manager.groupchat.agent_by_name("member1_team1", recursive=True)
        team2_member2 = gc_manager.groupchat.agent_by_name("member2_team2", recursive=True)

        assert team1_member1 is not None
        assert team2_member2 is not None

        team1_member1.send(team1_msg, team2_member2, request_reply=True)

    user = agent("user")
    team1 = team("team1", auto_reply=team1_msg)
    team2 = team("team2", auto_reply=team2_msg)

    gc = autogen.GroupChat(agents=[user, team1, team2], messages=[])
    gc_manager = autogen.GroupChatManager(groupchat=gc, llm_config=False)

    chat(gc_manager)

    team1_member1 = gc.agent_by_name("member1_team1", recursive=True)
    team2_member2 = gc.agent_by_name("member2_team2", recursive=True)

    assert team1_member1 and team2_member2

    msg = team1_member1.chat_messages[team2_member2][0]
    reply = team1_member1.chat_messages[team2_member2][1]

    assert msg["content"] == team1_msg["content"]
    assert reply["content"] == team2_msg["content"]


def test_custom_speaker_selection():
    a1 = autogen.UserProxyAgent(
        name="a1",
        default_auto_reply="This is a1 speaking.",
        human_input_mode="NEVER",
        code_execution_config={},
    )

    a2 = autogen.UserProxyAgent(
        name="a2",
        default_auto_reply="This is a2 speaking.",
        human_input_mode="NEVER",
        code_execution_config={},
    )

    a3 = autogen.UserProxyAgent(
        name="a3",
        default_auto_reply="TERMINATE",
        human_input_mode="NEVER",
        code_execution_config={},
    )

    def custom_speaker_selection_func(last_speaker: Agent, groupchat: GroupChat) -> Agent:
        """Define a customized speaker selection function.
        A recommended way is to define a transition for each speaker using the groupchat allowed_or_disallowed_speaker_transitions parameter.
        """
        if last_speaker is a1:
            return a2
        elif last_speaker is a2:
            return a3

    groupchat = autogen.GroupChat(
        agents=[a1, a2, a3],
        messages=[],
        max_round=20,
        speaker_selection_method=custom_speaker_selection_func,
    )
    manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=False)

    result = a1.initiate_chat(manager, message="Hello, this is a1 speaking.")
    assert len(result.chat_history) == 3


def test_custom_speaker_selection_with_transition_graph():
    """
    In this test, although speaker_selection_method is defined, the speaker transitions are also defined.
    There are 26 agents here, a to z.
    The speaker transitions are defined such that the agents can transition to the next alphabet.
    In addition, because we want the transition order to be a,u,t,o,g,e,n, we also define the speaker transitions for these agents.
    The speaker_selection_method is defined to return the next agent in the expected sequence.
    """

    # For loop that creates UserProxyAgent with names from a to z
    agents = [
        autogen.UserProxyAgent(
            name=chr(97 + i),
            default_auto_reply=f"My name is {chr(97 + i)}",
            human_input_mode="NEVER",
            code_execution_config={},
        )
        for i in range(26)
    ]

    # Initiate allowed speaker transitions
    allowed_or_disallowed_speaker_transitions = {}

    # Each agent can transition to the next alphabet as a baseline
    # Key is Agent, value is a list of Agents that the key Agent can transition to
    for i in range(25):
        allowed_or_disallowed_speaker_transitions[agents[i]] = [agents[i + 1]]

    # The test is to make sure that the agent sequence is a,u,t,o,g,e,n, so we need to add those transitions
    expected_sequence = ["a", "u", "t", "o", "g", "e", "n"]
    current_agent = None
    previous_agent = None

    for char in expected_sequence:
        # convert char to i so that we can use chr(97+i)
        current_agent = agents[ord(char) - 97]
        if previous_agent is not None:
            # Add transition
            allowed_or_disallowed_speaker_transitions[previous_agent].append(current_agent)
        previous_agent = current_agent

    def custom_speaker_selection_func(last_speaker: Agent, groupchat: GroupChat) -> Optional[Agent]:
        """
        Define a customized speaker selection function.
        """
        expected_sequence = ["a", "u", "t", "o", "g", "e", "n"]

        last_speaker_char = last_speaker.name
        # Find the index of last_speaker_char in the expected_sequence
        last_speaker_index = expected_sequence.index(last_speaker_char)
        # Return the next agent in the expected sequence
        if last_speaker_index == len(expected_sequence) - 1:
            return None  # terminate the conversation
        else:
            next_agent = agents[ord(expected_sequence[last_speaker_index + 1]) - 97]
            return next_agent

    groupchat = autogen.GroupChat(
        agents=agents,
        messages=[],
        max_round=20,
        speaker_selection_method=custom_speaker_selection_func,
        allowed_or_disallowed_speaker_transitions=allowed_or_disallowed_speaker_transitions,
        speaker_transitions_type="allowed",
    )
    manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=False)

    results = agents[0].initiate_chat(manager, message="My name is a")
    actual_sequence = []

    # Append to actual_sequence using results.chat_history[idx]['content'][-1]
    for idx in range(len(results.chat_history)):
        actual_sequence.append(results.chat_history[idx]["content"][-1])  # append the last character of the content

    assert expected_sequence == actual_sequence


def test_custom_speaker_selection_overrides_transition_graph():
    """
    In this test, team A engineer can transition to team A executor and team B engineer, but team B engineer cannot transition to team A executor.
    The expected behaviour is that the custom speaker selection function will override the constraints of the graph.
    """

    # For loop that creates UserProxyAgent with names from a to z
    agents = [
        autogen.UserProxyAgent(
            name="teamA_engineer",
            default_auto_reply="My name is teamA_engineer",
            human_input_mode="NEVER",
            code_execution_config={},
        ),
        autogen.UserProxyAgent(
            name="teamA_executor",
            default_auto_reply="My name is teamA_executor",
            human_input_mode="NEVER",
            code_execution_config={},
        ),
        autogen.UserProxyAgent(
            name="teamB_engineer",
            default_auto_reply="My name is teamB_engineer",
            human_input_mode="NEVER",
            code_execution_config={},
        ),
    ]

    allowed_or_disallowed_speaker_transitions = {}

    # teamA_engineer can transition to teamA_executor and teamB_engineer
    # teamB_engineer can transition to no one
    allowed_or_disallowed_speaker_transitions[agents[0]] = [agents[1], agents[2]]

    def custom_speaker_selection_func(last_speaker: Agent, groupchat: GroupChat) -> Optional[Agent]:
        if last_speaker.name == "teamA_engineer":
            return agents[2]  # Goto teamB_engineer
        elif last_speaker.name == "teamB_engineer":
            return agents[1]  # Goto teamA_executor and contradict the graph

    groupchat = autogen.GroupChat(
        agents=agents,
        messages=[],
        max_round=20,
        speaker_selection_method=custom_speaker_selection_func,
        allowed_or_disallowed_speaker_transitions=allowed_or_disallowed_speaker_transitions,
        speaker_transitions_type="allowed",
    )
    manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=False)
    results = agents[0].initiate_chat(manager, message="My name is teamA_engineer")

    speakers = []
    for idx in range(len(results.chat_history)):
        speakers.append(results.chat_history[idx].get("name"))

    assert "teamA_executor" in speakers


def test_role_for_select_speaker_messages():
    agent1 = autogen.ConversableAgent(
        "alice",
        max_consecutive_auto_reply=10,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is alice speaking.",
    )
    agent2 = autogen.ConversableAgent(
        "bob",
        max_consecutive_auto_reply=10,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is bob speaking.",
    )

    groupchat = autogen.GroupChat(
        agents=[agent1, agent2],
        messages=[{"role": "user", "content": "Let's have a chat!"}],
        max_round=3,
        role_for_select_speaker_messages="system",
    )

    # Replicate the _auto_select_speaker nested chat.

    # Agent for checking the response from the speaker_select_agent
    checking_agent = autogen.ConversableAgent("checking_agent")

    # Agent for selecting a single agent name from the response
    speaker_selection_agent = autogen.ConversableAgent(
        "speaker_selection_agent",
        llm_config=None,
        human_input_mode="NEVER",  # Suppresses some extra terminal outputs, outputs will be handled by select_speaker_auto_verbose
    )

    # The role_for_select_speaker_message is put into the initiate_chat of the nested two-way chat
    # into a message attribute called 'override_role'. This is evaluated in Conversable Agent's _append_oai_message function
    # e.g.: message={'content':self.select_speaker_prompt(agents),'override_role':self.role_for_select_speaker_messages},
    message = {"content": "A prompt goes here.", "override_role": groupchat.role_for_select_speaker_messages}
    checking_agent._append_oai_message(message, "assistant", speaker_selection_agent)

    # Test default is "system"
    assert len(checking_agent.chat_messages) == 1
    assert checking_agent.chat_messages[speaker_selection_agent][-1]["role"] == "system"

    # Test as "user"
    groupchat.role_for_select_speaker_messages = "user"
    message = {"content": "A prompt goes here.", "override_role": groupchat.role_for_select_speaker_messages}
    checking_agent._append_oai_message(message, "assistant", speaker_selection_agent)

    assert len(checking_agent.chat_messages) == 1
    assert checking_agent.chat_messages[speaker_selection_agent][-1]["role"] == "user"

    # Test as something unusual
    groupchat.role_for_select_speaker_messages = "SockS"
    message = {"content": "A prompt goes here.", "override_role": groupchat.role_for_select_speaker_messages}
    checking_agent._append_oai_message(message, "assistant", speaker_selection_agent)

    assert len(checking_agent.chat_messages) == 1
    assert checking_agent.chat_messages[speaker_selection_agent][-1]["role"] == "SockS"

    # Test empty string and None isn't accepted

    # Test with empty strings
    with pytest.raises(ValueError) as e:
        groupchat = autogen.GroupChat(
            agents=[agent1, agent2],
            messages=[{"role": "user", "content": "Let's have a chat!"}],
            max_round=3,
            role_for_select_speaker_messages="",
        )
    assert "role_for_select_speaker_messages cannot be empty or None." in str(e.value)

    with pytest.raises(ValueError) as e:
        groupchat = autogen.GroupChat(
            agents=[agent1, agent2],
            messages=[{"role": "user", "content": "Let's have a chat!"}],
            max_round=3,
            role_for_select_speaker_messages=None,
        )
    assert "role_for_select_speaker_messages cannot be empty or None." in str(e.value)


def test_select_speaker_message_and_prompt_templates():
    """
    In this test, two agents are part of a group chat which has customized select speaker message and select speaker prompt templates. Both valid and empty string values will be used.
    The expected behaviour is that the customized speaker selection message and prompts will override the default values or throw exceptions if empty.
    """

    agent1 = autogen.ConversableAgent(
        "Alice",
        description="A wonderful employee named Alice.",
        human_input_mode="NEVER",
        llm_config=False,
    )
    agent2 = autogen.ConversableAgent(
        "Bob",
        description="An amazing employee named Bob.",
        human_input_mode="NEVER",
        llm_config=False,
    )

    # Customised message, this is always the first message in the context
    custom_msg = """You are the CEO of a niche organisation creating small software tools for the healthcare sector with a small team of specialists. Call them in sequence.
    The job roles and responsibilities are:
    {roles}
    You must select only from {agentlist}."""

    # Customised prompt, this is always the last message in the context
    custom_prompt = """Read the above conversation.
    Then select the next job role from {agentlist} to take action.
    RETURN ONLY THE NAME OF THE NEXT ROLE."""

    # Test empty is_termination_msg function
    groupchat = autogen.GroupChat(
        agents=[agent1, agent2],
        messages=[],
        speaker_selection_method="auto",
        max_round=10,
        select_speaker_message_template=custom_msg,
        select_speaker_prompt_template=custom_prompt,
    )

    # Test with valid strings, checking for the correct string and roles / agentlist to be included

    assert groupchat.select_speaker_msg() == custom_msg.replace(
        "{roles}", "Alice: A wonderful employee named Alice.\nBob: An amazing employee named Bob."
    ).replace("{agentlist}", "['Alice', 'Bob']")

    assert groupchat.select_speaker_prompt() == custom_prompt.replace("{agentlist}", "['Alice', 'Bob']")

    # Test with empty strings
    with pytest.raises(ValueError, match="select_speaker_message_template cannot be empty or None."):
        groupchat = autogen.GroupChat(
            agents=[agent1, agent2],
            messages=[],
            speaker_selection_method="auto",
            max_round=10,
            select_speaker_message_template="",
            select_speaker_prompt_template="Not empty.",
        )

    # Will not throw an exception, prompt can be empty/None (empty is converted to None)
    groupchat = autogen.GroupChat(
        agents=[agent1, agent2],
        messages=[],
        speaker_selection_method="auto",
        max_round=10,
        select_speaker_message_template="Not empty.",
        select_speaker_prompt_template="",
    )

    assert groupchat.select_speaker_prompt_template is None

    # Test with None
    with pytest.raises(ValueError, match="select_speaker_message_template cannot be empty or None."):
        groupchat = autogen.GroupChat(
            agents=[agent1, agent2],
            messages=[],
            speaker_selection_method="auto",
            max_round=10,
            select_speaker_message_template=None,
            select_speaker_prompt_template="Not empty.",
        )

    # Will not throw an exception, prompt can be empty/None (empty is converted to None)
    groupchat = autogen.GroupChat(
        agents=[agent1, agent2],
        messages=[],
        speaker_selection_method="auto",
        max_round=10,
        select_speaker_message_template="Not empty.",
        select_speaker_prompt_template=None,
    )

    assert groupchat.select_speaker_prompt_template is None


def test_speaker_selection_agent_name_match():
    """
    In this test a group chat, with auto speaker selection, the speaker name match
    function is tested against the extended name match regex.
    """

    user_proxy = autogen.UserProxyAgent(
        name="User_proxy",
        system_message="A human admin.",
        code_execution_config=False,
        human_input_mode="NEVER",
    )
    storywriter = autogen.AssistantAgent(
        name="Story_writer",
        system_message="An ideas person.",
        llm_config=None,
    )
    pm = autogen.AssistantAgent(
        name="Product_manager",
        system_message="Great in evaluating story ideas.",
        llm_config=None,
    )

    all_agents = [user_proxy, storywriter, pm]
    groupchat = autogen.GroupChat(agents=all_agents, messages=[], max_round=8, speaker_selection_method="auto")

    # Test exact match (unchanged outcome)
    result = groupchat._mentioned_agents(agents=all_agents, message_content="Story_writer")
    assert result == {"Story_writer": 1}

    # Test match with extra text (unchanged outcome)
    result = groupchat._mentioned_agents(
        agents=all_agents,
        message_content="' Story_writer.\n\nHere are three story ideas for Grade 3 kids:\n\n1. **The Adventure of the Magic Garden:** A you...",
    )
    assert result == {"Story_writer": 1}

    # Test match with escaped underscore (new outcome)
    result = groupchat._mentioned_agents(agents=all_agents, message_content="Story\\_writer")
    assert result == {"Story_writer": 1}

    # Test match with space (new outcome)
    result = groupchat._mentioned_agents(agents=all_agents, message_content="Story writer")
    assert result == {"Story_writer": 1}

    # Test match with different casing (unchanged outcome)
    result = groupchat._mentioned_agents(agents=all_agents, message_content="Story_Writer")
    assert result == {}

    # Test match with invalid name (unchanged outcome)
    result = groupchat._mentioned_agents(agents=all_agents, message_content="NoName_Person")
    assert result == {}

    # Test match with no name (unchanged outcome)
    result = groupchat._mentioned_agents(agents=all_agents, message_content="")
    assert result == {}

    # Test match with multiple agents and exact matches (unchanged outcome)
    result = groupchat._mentioned_agents(
        agents=all_agents, message_content="Story_writer will follow the Product_manager."
    )
    assert result == {"Story_writer": 1, "Product_manager": 1}

    # Test match with multiple agents and escaped underscores (new outcome)
    result = groupchat._mentioned_agents(
        agents=all_agents, message_content="Story\\_writer will follow the Product\\_manager."
    )
    assert result == {"Story_writer": 1, "Product_manager": 1}

    # Test match with multiple agents and escaped underscores (new outcome)
    result = groupchat._mentioned_agents(
        agents=all_agents, message_content="Story\\_writer will follow the Product\\_manager."
    )
    assert result == {"Story_writer": 1, "Product_manager": 1}

    # Test match with multiple agents and spaces (new outcome)
    result = groupchat._mentioned_agents(
        agents=all_agents, message_content="Story writer will follow the Product manager."
    )
    assert result == {"Story_writer": 1, "Product_manager": 1}

    # Test match with multiple agents and escaped underscores and spaces mixed (new outcome)
    result = groupchat._mentioned_agents(
        agents=all_agents, message_content="Story writer will follow the Product\\_manager."
    )
    assert result == {"Story_writer": 1, "Product_manager": 1}

    # Test match with multiple agents and incorrect casing (unchanged outcome)
    result = groupchat._mentioned_agents(
        agents=all_agents, message_content="Story Writer will follow the product\\_manager."
    )
    assert result == {}


def test_role_for_reflection_summary():
    llm_config = {"config_list": [{"model": "mock", "api_key": "mock"}]}
    agent1 = autogen.ConversableAgent(
        "alice",
        max_consecutive_auto_reply=10,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is alice speaking.",
    )
    agent2 = autogen.ConversableAgent(
        "bob",
        max_consecutive_auto_reply=10,
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is bob speaking.",
    )
    groupchat = autogen.GroupChat(
        agents=[agent1, agent2], messages=[], max_round=3, speaker_selection_method="round_robin"
    )
    group_chat_manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=llm_config)

    role_name = "user"
    with mock.patch.object(
        autogen.ConversableAgent, "_generate_oai_reply_from_client"
    ) as mock_generate_oai_reply_from_client:
        mock_generate_oai_reply_from_client.return_value = "Mocked summary"

        agent1.initiate_chat(
            group_chat_manager,
            max_turns=2,
            message="hello",
            summary_method="reflection_with_llm",
            summary_args={"summary_role": role_name},
        )

        mock_generate_oai_reply_from_client.assert_called_once()
        args, kwargs = mock_generate_oai_reply_from_client.call_args
        assert kwargs["messages"][-1]["role"] == role_name


def test_speaker_selection_auto_process_result():
    """
    Tests the return result of the 2-agent chat used for speaker selection for the auto method.
    The last message of the messages passed in will contain a pass or fail.
    If passed, the message will contain the name of the correct agent and that agent will be returned.
    If failed, the message will contain the reason for failure for the last attempt and the next
    agent in the sequence will be returned.
    """
    cmo = autogen.ConversableAgent(
        name="Chief_Marketing_Officer",
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is alice speaking.",
    )
    pm = autogen.ConversableAgent(
        name="Product_Manager",
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is bob speaking.",
        function_map={"test_func": lambda x: x},
    )

    agent_list = [cmo, pm]
    groupchat = autogen.GroupChat(agents=agent_list, messages=[], max_round=3)

    chat_result = autogen.ChatResult(
        chat_id=None,
        chat_history=[
            {
                "content": "Let's get this meeting started. First the Product_Manager will create 3 new product ideas.",
                "name": "Chairperson",
                "role": "assistant",
            },
            {"content": "You are an expert at finding the next speaker.", "role": "assistant"},
            {"content": "Product_Manager", "role": "user"},
            {"content": "UPDATED_BELOW", "role": "user"},
        ],
    )

    ### Agent selected successfully
    chat_result.chat_history[3]["content"] = "[AGENT SELECTED]Product_Manager"

    # Product_Manager should be returned
    assert groupchat._process_speaker_selection_result(chat_result, cmo, agent_list) == pm

    ### Agent not selected successfully
    chat_result.chat_history[3][
        "content"
    ] = "[AGENT SELECTION FAILED]Select speaker attempt #3 of 3 failed as it did not include any agent names."

    # The next speaker in the list will be selected, which will be the Product_Manager (as the last speaker is the Chief_Marketing_Officer)
    assert groupchat._process_speaker_selection_result(chat_result, cmo, agent_list) == pm

    ### Invalid result messages, will return the next agent
    chat_result.chat_history[3]["content"] = "This text should not be here."

    # The next speaker in the list will be selected, which will be the Chief_Marketing_Officer (as the last speaker is the Product_Maanger)
    assert groupchat._process_speaker_selection_result(chat_result, pm, agent_list) == cmo


def test_speaker_selection_validate_speaker_name():
    """
    Tests the speaker name validation function used to evaluate the return result of the LLM
    during speaker selection in 'auto' mode.

    Function: _validate_speaker_name

    If a single agent name is returned by the LLM, it will add a relevant message to the chat messages and return True, None
    If multiple agent names are returned and there are attempts left, it will return a message to be used to prompt the LLM to try again
    If multiple agent names are return and there are no attempts left, it will add a relevant message to the chat messages and return True, None
    If no agent names are returned and there are attempts left, it will return a message to be used to prompt the LLM to try again
    If no agent names are returned and there are no attempts left, it will add a relevant message to the chat messages and return True, None

    When returning a message, it will include the 'override_role' key and value to support the GroupChat role_for_select_speaker_messages attribute
    """

    # Group Chat setup
    cmo = autogen.ConversableAgent(
        name="Chief_Marketing_Officer",
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is alice speaking.",
    )
    pm = autogen.ConversableAgent(
        name="Product_Manager",
        human_input_mode="NEVER",
        llm_config=False,
        default_auto_reply="This is bob speaking.",
        function_map={"test_func": lambda x: x},
    )

    agent_list = [cmo, pm]
    agent_list_string = f"{[agent.name for agent in agent_list]}"
    groupchat = autogen.GroupChat(agents=agent_list, messages=[], max_round=3)

    # Speaker Selection 2-agent chat setup

    # Agent for selecting a single agent name from the response
    speaker_selection_agent = autogen.ConversableAgent(
        "speaker_selection_agent",
    )

    # Agent for checking the response from the speaker_select_agent
    checking_agent = autogen.ConversableAgent("checking_agent")

    # Select speaker messages
    select_speaker_messages = [
        {
            "content": "Let's get this meeting started. First the Product_Manager will create 3 new product ideas.",
            "name": "Chairperson",
            "role": "assistant",
        },
        {"content": "You are an expert at finding the next speaker.", "role": "assistant"},
        {"content": "UPDATED_BELOW", "role": "user"},
    ]

    ### Single agent name returned
    attempts_left = 2
    attempt = 1
    select_speaker_messages[-1]["content"] = "Product_Manager is the next to speak"

    result = groupchat._validate_speaker_name(
        recipient=checking_agent,
        messages=select_speaker_messages,
        sender=speaker_selection_agent,
        config=None,
        attempts_left=attempts_left,
        attempt=attempt,
        agents=agent_list,
    )

    assert result == (True, None)
    assert select_speaker_messages[-1]["content"] == "[AGENT SELECTED]Product_Manager"

    select_speaker_messages.pop(-1)  # Remove the last message before the next test

    ### Multiple agent names returned with attempts left
    attempts_left = 2
    attempt = 1
    select_speaker_messages[-1]["content"] = "Product_Manager must speak after the Chief_Marketing_Officer"

    result = groupchat._validate_speaker_name(
        recipient=checking_agent,
        messages=select_speaker_messages,
        sender=speaker_selection_agent,
        config=None,
        attempts_left=attempts_left,
        attempt=attempt,
        agents=agent_list,
    )

    assert result == (
        True,
        {
            "content": groupchat.select_speaker_auto_multiple_template.format(agentlist=agent_list_string),
            "override_role": groupchat.role_for_select_speaker_messages,
        },
    )

    ### Multiple agent names returned with no attempts left
    attempts_left = 0
    attempt = 1
    select_speaker_messages[-1]["content"] = "Product_Manager must speak after the Chief_Marketing_Officer"

    result = groupchat._validate_speaker_name(
        recipient=checking_agent,
        messages=select_speaker_messages,
        sender=speaker_selection_agent,
        config=None,
        attempts_left=attempts_left,
        attempt=attempt,
        agents=agent_list,
    )

    assert result == (True, None)
    assert (
        select_speaker_messages[-1]["content"]
        == f"[AGENT SELECTION FAILED]Select speaker attempt #{attempt} of {attempt + attempts_left} failed as it returned multiple names."
    )

    select_speaker_messages.pop(-1)  # Remove the last message before the next test

    ### No agent names returned with attempts left
    attempts_left = 3
    attempt = 2
    select_speaker_messages[-1]["content"] = "The PM must speak after the CMO"

    result = groupchat._validate_speaker_name(
        recipient=checking_agent,
        messages=select_speaker_messages,
        sender=speaker_selection_agent,
        config=None,
        attempts_left=attempts_left,
        attempt=attempt,
        agents=agent_list,
    )

    assert result == (
        True,
        {
            "content": groupchat.select_speaker_auto_none_template.format(agentlist=agent_list_string),
            "override_role": groupchat.role_for_select_speaker_messages,
        },
    )

    ### Multiple agents returned with no attempts left
    attempts_left = 0
    attempt = 3
    select_speaker_messages[-1]["content"] = "The PM must speak after the CMO"

    result = groupchat._validate_speaker_name(
        recipient=checking_agent,
        messages=select_speaker_messages,
        sender=speaker_selection_agent,
        config=None,
        attempts_left=attempts_left,
        attempt=attempt,
        agents=agent_list,
    )

    assert result == (True, None)
    assert (
        select_speaker_messages[-1]["content"]
        == f"[AGENT SELECTION FAILED]Select speaker attempt #{attempt} of {attempt + attempts_left} failed as it did not include any agent names."
    )


def test_select_speaker_auto_messages():
    """
    In this test, two agents are part of a group chat which has customized select speaker "auto" multiple and no-name prompt messages. Both valid and empty string values will be used.
    The expected behaviour is that the customized speaker selection "auto" messages will override the default values or throw exceptions if empty.
    """

    agent1 = autogen.ConversableAgent(
        "Alice",
        description="A wonderful employee named Alice.",
        human_input_mode="NEVER",
        llm_config=False,
    )
    agent2 = autogen.ConversableAgent(
        "Bob",
        description="An amazing employee named Bob.",
        human_input_mode="NEVER",
        llm_config=False,
    )

    # Customised message for select speaker auto method where multiple agent names are returned
    custom_multiple_names_msg = "You mentioned multiple names but we need just one. Select the best one. A reminder that the options are {agentlist}."

    # Customised message for select speaker auto method where no agent names are returned
    custom_no_names_msg = "You forgot to select a single names and we need one, and only one. Select the best one. A reminder that the options are {agentlist}."

    # Test empty is_termination_msg function
    groupchat = autogen.GroupChat(
        agents=[agent1, agent2],
        messages=[],
        speaker_selection_method="auto",
        max_round=10,
        select_speaker_auto_multiple_template=custom_multiple_names_msg,
        select_speaker_auto_none_template=custom_no_names_msg,
    )

    # Test using the _validate_speaker_name function, checking for the correct string and agentlist to be included
    agents = [agent1, agent2]

    messages = [{"content": "Alice and Bob should both speak.", "name": "speaker_selector", "role": "user"}]
    assert groupchat._validate_speaker_name(None, messages, None, None, 1, 1, agents) == (
        True,
        {
            "content": custom_multiple_names_msg.replace("{agentlist}", "['Alice', 'Bob']"),
            "override_role": groupchat.role_for_select_speaker_messages,
        },
    )

    messages = [{"content": "Fred should both speak.", "name": "speaker_selector", "role": "user"}]
    assert groupchat._validate_speaker_name(None, messages, None, None, 1, 1, agents) == (
        True,
        {
            "content": custom_no_names_msg.replace("{agentlist}", "['Alice', 'Bob']"),
            "override_role": groupchat.role_for_select_speaker_messages,
        },
    )

    # Test with empty strings
    with pytest.raises(ValueError, match="select_speaker_auto_multiple_template cannot be empty or None."):
        groupchat = autogen.GroupChat(
            agents=[agent1, agent2],
            messages=[],
            speaker_selection_method="auto",
            max_round=10,
            select_speaker_auto_multiple_template="",
        )

    with pytest.raises(ValueError, match="select_speaker_auto_none_template cannot be empty or None."):
        groupchat = autogen.GroupChat(
            agents=[agent1, agent2],
            messages=[],
            speaker_selection_method="auto",
            max_round=10,
            select_speaker_auto_none_template="",
        )

    # Test with None
    with pytest.raises(ValueError, match="select_speaker_auto_multiple_template cannot be empty or None."):
        groupchat = autogen.GroupChat(
            agents=[agent1, agent2],
            messages=[],
            speaker_selection_method="auto",
            max_round=10,
            select_speaker_auto_multiple_template=None,
        )

    with pytest.raises(ValueError, match="select_speaker_auto_none_template cannot be empty or None."):
        groupchat = autogen.GroupChat(
            agents=[agent1, agent2],
            messages=[],
            speaker_selection_method="auto",
            max_round=10,
            select_speaker_auto_none_template=None,
        )


def test_manager_messages_to_string():
    """In this test we test the conversion of messages to a JSON string"""
    messages = [
        {
            "content": "You are an expert at finding the next speaker.",
            "role": "system",
        },
        {
            "content": "Let's get this meeting started. First the Product_Manager will create 3 new product ideas.",
            "name": "Chairperson",
            "role": "assistant",
        },
    ]

    groupchat = GroupChat(messages=messages, agents=[])
    manager = GroupChatManager(groupchat)

    # Convert the messages List[Dict] to a JSON string
    converted_string = manager.messages_to_string(messages)

    # The conversion should match the original messages
    assert json.loads(converted_string) == messages


def test_manager_messages_from_string():
    """In this test we test the conversion of a JSON string of messages to a messages List[Dict]"""
    messages_str = r"""[{"content": "You are an expert at finding the next speaker.", "role": "system"}, {"content": "Let's get this meeting started. First the Product_Manager will create 3 new product ideas.", "name": "Chairperson", "role": "assistant"}]"""

    groupchat = GroupChat(messages=[], agents=[])
    manager = GroupChatManager(groupchat)

    # Convert the messages List[Dict] to a JSON string
    messages = manager.messages_from_string(messages_str)

    # The conversion should match the original messages
    assert messages_str == json.dumps(messages)


def test_manager_resume_functions():
    """Tests functions within the resume chat functionality"""

    # Setup
    coder = AssistantAgent(name="Coder", llm_config=None)
    groupchat = GroupChat(messages=[], agents=[coder])
    manager = GroupChatManager(groupchat)

    # Tests that messages are indeed passed in
    with pytest.raises(Exception):
        manager._valid_resume_messages(messages=[])

    # Tests that the messages passed in match the agents of the group chat
    messages = [
        {
            "content": "You are an expert at finding the next speaker.",
            "role": "system",
        },
        {
            "content": "Let's get this meeting started. First the Product_Manager will create 3 new product ideas.",
            "name": "Chairperson",
            "role": "assistant",
        },
    ]

    # Chairperson does not exist as an agent
    with pytest.raises(Exception):
        manager._valid_resume_messages(messages)

    messages = [
        {
            "content": "You are an expert at finding the next speaker.",
            "role": "system",
        },
        {
            "content": "Let's get this meeting started. First the Product_Manager will create 3 new product ideas.",
            "name": "Coder",
            "role": "assistant",
        },
    ]

    # Coder does exist as an agent, no error
    manager._valid_resume_messages(messages)

    # Tests termination message replacement
    final_msg = (
        "Let's get this meeting started. First the Product_Manager will create 3 new product ideas. TERMINATE this."
    )
    messages = [
        {
            "content": "You are an expert at finding the next speaker.",
            "role": "system",
        },
        {
            "content": final_msg,
            "name": "Coder",
            "role": "assistant",
        },
    ]

    manager._process_resume_termination(remove_termination_string="TERMINATE", messages=messages)

    # TERMINATE should be removed
    assert messages[-1]["content"] == final_msg.replace("TERMINATE", "")

    # Tests termination message replacement with function
    def termination_func(x: str) -> str:
        if "APPROVED" in x:
            x = x.replace("APPROVED", "")
        else:
            x = x.replace("TERMINATE", "")
        return x

    final_msg1 = "Product_Manager has created 3 new product ideas. APPROVED"
    messages1 = [
        {
            "content": "You are an expert at finding the next speaker.",
            "role": "system",
        },
        {
            "content": final_msg1,
            "name": "Coder",
            "role": "assistant",
        },
    ]

    manager._process_resume_termination(remove_termination_string=termination_func, messages=messages1)

    # APPROVED should be removed
    assert messages1[-1]["content"] == final_msg1.replace("APPROVED", "")

    final_msg2 = "Idea has been approved. TERMINATE"
    messages2 = [
        {
            "content": "You are an expert at finding the next speaker.",
            "role": "system",
        },
        {
            "content": final_msg2,
            "name": "Coder",
            "role": "assistant",
        },
    ]

    manager._process_resume_termination(remove_termination_string=termination_func, messages=messages2)

    # TERMINATE should be removed, "approved" should still be present as the termination_func only replaces upper-cased "APPROVED".
    assert messages2[-1]["content"] == final_msg2.replace("TERMINATE", "")
    assert "approved" in messages2[-1]["content"]

    # Check if the termination string doesn't exist there's no replacing of content
    final_msg = (
        "Let's get this meeting started. First the Product_Manager will create 3 new product ideas. TERMINATE this."
    )
    messages = [
        {
            "content": "You are an expert at finding the next speaker.",
            "role": "system",
        },
        {
            "content": final_msg,
            "name": "Coder",
            "role": "assistant",
        },
    ]

    manager._process_resume_termination(remove_termination_string="THE-END", messages=messages)

    # It should not be changed
    assert messages[-1]["content"] == final_msg

    # Test that it warns that the termination condition would match
    manager._is_termination_msg = lambda x: x.get("content", "").find("TERMINATE") >= 0

    # Attach a handler to the logger so we can check the log output
    log_stream = io.StringIO()
    handler = logging.StreamHandler(log_stream)
    logger = logging.getLogger()  # Get the root logger
    logger.addHandler(handler)

    # We should get a warning that TERMINATE is still in the messages
    manager._process_resume_termination(remove_termination_string="THE-END", messages=messages)

    # Get the logged output and check that the warning was provided.
    log_output = log_stream.getvalue()

    assert "WARNING: Last message meets termination criteria and this may terminate the chat." in log_output


def test_manager_resume_returns():
    """Tests the return resume chat functionality"""

    # Test the return agent and message is correct
    coder = AssistantAgent(name="Coder", llm_config=None)
    groupchat = GroupChat(messages=[], agents=[coder])
    manager = GroupChatManager(groupchat)
    messages = [
        {
            "content": "You are an expert at coding.",
            "role": "system",
        },
        {
            "content": "Let's get coding, should I use Python?",
            "name": "Coder",
            "role": "assistant",
        },
    ]

    return_agent, return_message = manager.resume(messages=messages)

    assert return_agent == coder
    assert return_message == messages[-1]

    # Test when no agent provided, the manager will be returned
    messages = [{"content": "You are an expert at coding.", "role": "system", "name": "chat_manager"}]

    return_agent, return_message = manager.resume(messages=messages)

    assert return_agent == manager
    assert return_message == messages[-1]


def test_manager_resume_messages():
    """Tests that the messages passed into resume are the correct format"""

    coder = AssistantAgent(name="Coder", llm_config=None)
    groupchat = GroupChat(messages=[], agents=[coder])
    manager = GroupChatManager(groupchat)
    messages = 1

    # Only acceptable messages types are JSON str and List[Dict]

    # Try a number
    with pytest.raises(Exception):
        return_agent, return_message = manager.resume(messages=messages)

    # Try an empty string
    with pytest.raises(Exception):
        return_agent, return_message = manager.resume(messages="")

    # Try a message starter string, which isn't valid
    with pytest.raises(Exception):
        return_agent, return_message = manager.resume(messages="Let's get this conversation started.")


if __name__ == "__main__":
    # test_func_call_groupchat()
    # test_broadcast()
    # test_chat_manager()
    # test_plugin()
    # test_speaker_selection_method()
    # test_n_agents_less_than_3()
    # test_agent_mentions()
    # test_termination()
    # test_next_agent()
    # test_send_intros()
    # test_invalid_allow_repeat_speaker()
    # test_graceful_exit_before_max_round()
    # test_clear_agents_history()
    # test_custom_speaker_selection_overrides_transition_graph()
    # test_role_for_select_speaker_messages()
    # test_select_speaker_message_and_prompt_templates()
    # test_speaker_selection_agent_name_match()
    # test_role_for_reflection_summary()
    # test_speaker_selection_auto_process_result()
    # test_speaker_selection_validate_speaker_name()
    # test_select_speaker_auto_messages()
    # test_select_speaker_auto_messages()
    # test_manager_messages_to_string()
    # test_manager_messages_from_string()
    test_manager_resume_functions()
    # test_manager_resume_returns()
    # test_manager_resume_messages()
    pass